From b09b81da4dc99f4f820c6622e4e0bd9fe65a1bbd Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 19 Feb 2019 14:49:21 -0600 Subject: [PATCH 01/56] This branch supports grid/slurm (and possibly other remote queueing packages) with the -G and -S options. It does checkpointing for mecat2pw and mecat2cns, allowing restarting of failed jobs. It also allows correction against (in mecat2pw) and of (in mecat2cns) a subset of the given reads with the -R option. The -k option of mecat2cns has been changed to default to zero rather than 10, with assumption that a quicker partitioning is better (and setting -k to zero is now the same as using a negative value, rather than creating an infinite loop). --- .gitignore | 3 + mecat2canu/src/canu_version_update.pl | 0 .../src/overlapErrorAdjustment/findErrors.C | 1 + src/common/alignment.h | 202 +++++- src/common/buffer_line_iterator.cpp | 175 +++-- src/common/buffer_line_iterator.h | 7 +- src/common/fasta_reader.h | 2 + src/common/packed_db.h | 32 +- src/common/split_database.cpp | 240 ++++-- src/common/split_database.h | 7 +- src/mecat2cns/dw.cpp | 248 +++---- src/mecat2cns/dw.h | 5 +- src/mecat2cns/main.cpp | 148 +++- src/mecat2cns/mecat2cns.mk | 7 +- src/mecat2cns/mecat_correction.cpp | 260 +++---- src/mecat2cns/mecat_correction.h | 8 +- src/mecat2cns/options.cpp | 367 +++++----- src/mecat2cns/options.h | 15 +- src/mecat2cns/overlaps_partition.cpp | 339 +++------ src/mecat2cns/overlaps_partition.h | 17 +- src/mecat2cns/overlaps_store.h | 280 ++++--- src/mecat2cns/reads_correction_aux.cpp | 63 +- src/mecat2cns/reads_correction_aux.h | 238 ++++-- src/mecat2cns/reads_correction_can.cpp | 174 ++--- src/mecat2cns/reads_correction_m4.cpp | 153 ++-- src/mecat2pw/pw.cpp | 195 +++-- src/mecat2pw/pw_impl.cpp | 683 ++++++++++-------- src/mecat2pw/pw_impl.h | 62 +- src/mecat2pw/pw_options.cpp | 97 ++- src/mecat2pw/pw_options.h | 10 + 30 files changed, 2332 insertions(+), 1706 deletions(-) mode change 100644 => 100755 mecat2canu/src/canu_version_update.pl diff --git a/.gitignore b/.gitignore index cd2946a..9808873 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ $RECYCLE.BIN/ Network Trash Folder Temporary Items .apdisk + +# Manual backup files +*.orig diff --git a/mecat2canu/src/canu_version_update.pl b/mecat2canu/src/canu_version_update.pl old mode 100644 new mode 100755 diff --git a/mecat2canu/src/overlapErrorAdjustment/findErrors.C b/mecat2canu/src/overlapErrorAdjustment/findErrors.C index 825579a..2331ed1 100644 --- a/mecat2canu/src/overlapErrorAdjustment/findErrors.C +++ b/mecat2canu/src/overlapErrorAdjustment/findErrors.C @@ -86,6 +86,7 @@ Extract_Needed_Frags(feParameters *G, uint32 ii = 0; // Index into reads arrays uint32 fi = G->olaps[lastOlap].b_iid; // Actual ID we're extracting + if (hiID < fi) return; assert(loID <= fi); fprintf(stderr, "Extract_Needed_Frags()-- Loading used reads between "F_U32" and "F_U32".\n", diff --git a/src/common/alignment.h b/src/common/alignment.h index c0792f1..4046d3a 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -185,30 +185,170 @@ m4_to_candidate(const M4Record& m4, ExtensionCandidate& ec) ec.score = m4vscore(m4); } -struct M5Record -{ +class M5Record { + public: idx_t qid; // 1) qname - idx_t qsize; // 2) qlength - idx_t qstart; // 3) qstart + idx_t qsize; // 2) qlength + idx_t qstart; // 3) qstart idx_t qend; // 4) qend - int qdir; // 5) qstrand + int qdir; // 5) qstrand idx_t sid; // 6) sname idx_t ssize; // 7) slength idx_t sstart; // 8) sstart idx_t send; // 9) send - int sdir; // 10) sstrand - int score; // 11) score - int mat; // 12) match - int mis; // 13) mismatch - int ins; // 14) insertion - int dels; // 15) deletion - int mapq; // 16) mapQ - char* pm_q; // 17) aligned query - char* pm_p; // 18) aligned pattern - char* pm_s; // 19) aligned subject + int sdir; // 10) sstrand + int score; // 11) score + int mat; // 12) match + int mis; // 13) mismatch + int ins; // 14) insertion + int dels; // 15) deletion + int mapq; // 16) mapQ + char* pm_q; // 17) aligned query + char* pm_p; // 18) aligned pattern + char* pm_s; // 19) aligned subject double ident; // 20) identity percentage idx_t qext; idx_t sext; + public: + M5Record() : pm_q(0), pm_p(0), pm_s(0) { } + explicit M5Record(const idx_t n) : pm_q(new char[n]), pm_p(new char[n]), pm_s(new char[n]) { } + ~M5Record() { + delete[] pm_q; + delete[] pm_p; + delete[] pm_s; + } + idx_t& m5qid() { + return qid; + } + const idx_t& m5qid() const { + return qid; + } + idx_t& m5qsize() { + return qsize; + } + const idx_t& m5qsize() const { + return qsize; + } + idx_t& m5qoff() { + return qstart; + } + const idx_t& m5qoff() const { + return qstart; + } + idx_t& m5qend() { + return qend; + } + const idx_t& m5qend() const { + return qend; + } + int& m5qdir() { + return qdir; + } + const int& m5qdir() const { + return qdir; + } + idx_t& m5sid() { + return sid; + } + const idx_t& m5sid() const { + return sid; + } + idx_t& m5ssize() { + return ssize; + } + const idx_t& m5ssize() const { + return ssize; + } + idx_t& m5soff() { + return sstart; + } + const idx_t& m5soff() const { + return sstart; + } + idx_t& m5send() { + return send; + } + const idx_t& m5send() const { + return send; + } + int& m5sdir() { + return sdir; + } + const int& m5sdir() const { + return sdir; + } + int& m5score() { + return score; + } + const int& m5score() const { + return score; + } + int& m5mat() { + return mat; + } + const int& m5mat() const { + return mat; + } + int& m5mis() { + return mis; + } + const int& m5mis() const { + return mis; + } + int& m5ins() { + return ins; + } + const int& m5ins() const { + return ins; + } + int& m5dels() { + return dels; + } + const int& m5dels() const { + return dels; + } + int& m5mapq() { + return mapq; + } + const int& m5mapq() const { + return mapq; + } + char*& m5qaln() { + return pm_q; + } + const char* m5qaln() const { + return pm_q; + } + char*& m5pat() { + return pm_p; + } + const char* m5pat() const { + return pm_p; + } + char*& m5saln() { + return pm_s; + } + const char* m5saln() const { + return pm_s; + } + double& m5ident() { + return ident; + } + const double& m5ident() const { + return ident; + } + idx_t& m5qext() { + return qext; + } + const idx_t& m5qext() const { + return qext; + } + idx_t& m5sext() { + return sext; + } + const idx_t& m5sext() const { + return sext; + } }; #define m5qid(m) ((m).qid) @@ -238,38 +378,28 @@ void PrintM5Record(std::ostream& out, const M5Record& m5, const int printAln); void InitM5Record(M5Record& m5); void DestroyM5Record(M5Record& m5); -inline M5Record* NewM5Record(idx_t maxAlnSize) -{ - M5Record* m5 = new M5Record; - m5qaln(*m5) = new char[maxAlnSize]; - m5saln(*m5) = new char[maxAlnSize]; - m5pat(*m5) = new char[maxAlnSize]; - - return m5; +inline M5Record* NewM5Record(const idx_t maxAlnSize) { + return new M5Record(maxAlnSize); } -inline M5Record* DeleteM5Record(M5Record* m5) -{ - if (!m5) return NULL; - delete[] m5qaln(*m5); - delete[] m5saln(*m5); - delete[] m5pat(*m5); - delete m5; +inline M5Record* DeleteM5Record(M5Record* const m5) { + if (m5) { + delete m5; + } return NULL; } -inline int M5RecordOvlpSize(const M5Record& m) -{ - int oq = m5qend(m) - m5qoff(m); - int os = m5send(m) - m5soff(m); +inline int M5RecordOvlpSize(const M5Record& m) { + const int oq(m5qend(m) - m5qoff(m)); + const int os(m5send(m) - m5soff(m)); return std::max(oq, os); } //struct Overlap //{ -// index_t qid, qoff, qend, qsize, qext; +// idx_t qid, qoff, qend, qsize, qext; // int qdir; -// index_t sid, soff, send, ssize, sext; +// idx_t sid, soff, send, ssize, sext; // int sdir; //}; diff --git a/src/common/buffer_line_iterator.cpp b/src/common/buffer_line_iterator.cpp index ce3399f..d9857da 100644 --- a/src/common/buffer_line_iterator.cpp +++ b/src/common/buffer_line_iterator.cpp @@ -9,6 +9,7 @@ BufferLineReader::BufferLineReader(const char* file_name) done_ = false; unget_line_ = false; line_number_ = 0; + eol_length_ = 1; x_read_buffer(); } @@ -19,103 +20,94 @@ bool BufferLineReader::eof() const } -bool BufferLineReader::operator++() -{ - ++line_number_; - - if (unget_line_) - { - unget_line_ = false; - return true; - } - #define buffer_read_ret (!(done_ && line_.size() == 0)) - line_.clear(); - const idx_t start = cur_; - const idx_t end = buf_sz_; - for (idx_t p = start; p < end; ++p) - { - const int c = buf_[p]; - if (c == '\n') - { - line_.push_back(buf_ + start, p - start); - cur_ = ++p; - if (p == end) - x_read_buffer(); - return buffer_read_ret; - } - else if (c == '\r') - { - line_.push_back(buf_ + start, p - start); - cur_ = ++p; - if (p == end) - { - if (x_read_buffer()) - { - p = cur_; - if (buf_[p] == '\n') - cur_ = p + 1; - } - return buffer_read_ret; - } - if (buf_[p] != '\n') return buffer_read_ret; - cur_ = ++p; - if (p == end) - x_read_buffer(); - return buffer_read_ret; - } - } - - x_load_long(); - return buffer_read_ret; +bool BufferLineReader::operator++() { + ++line_number_; + if (unget_line_) { + unget_line_ = false; + return true; + } + line_.clear(); + const idx_t start = cur_; + const idx_t end = buf_sz_; + for (idx_t p = start; p < end; ++p) { + const int c = buf_[p]; + if (c == '\n') { + line_.push_back(buf_ + start, p - start); + cur_ = ++p; + if (p == end) { + x_read_buffer(); + } + return buffer_read_ret; + } else if (c == '\r') { + line_.push_back(buf_ + start, p - start); + cur_ = ++p; + if (p == end) { + if (x_read_buffer()) { + p = cur_; + if (buf_[p] == '\n') { + cur_ = p + 1; + eol_length_ = 2; + } + } + return buffer_read_ret; + } + if (buf_[p] != '\n') { + return buffer_read_ret; + } + cur_ = ++p; + eol_length_ = 2; + if (p == end) { + x_read_buffer(); + } + return buffer_read_ret; + } + } + x_load_long(); + return buffer_read_ret; } - -void BufferLineReader::x_load_long() -{ - idx_t start = cur_; - idx_t end = buf_sz_; - line_.push_back(buf_ + start, end - start); - while (x_read_buffer()) - { - start = cur_; - end = buf_sz_; - for (idx_t p = start; p < end; ++p) - { - const int c = buf_[p]; - if (c == '\r' || c == '\n') - { - line_.push_back(buf_ + start, p - start); - if (++p == end) - { - if (x_read_buffer()) - { - p = cur_; - end = buf_sz_; - if (p < end && c == '\r' && buf_[p] == '\n') { ++p; cur_ = p; } - } - } - else - { - if (c == '\r' && buf_[p] == '\n') - { - if (++p == end) - { - x_read_buffer(); - p = cur_; - } - } - cur_ = p; - } - return; - } - } - line_.push_back(buf_ + start, end - start); - } +// called when EOL isn't present in the local buffer +void BufferLineReader::x_load_long() { + idx_t start = cur_; + idx_t end = buf_sz_; + line_.push_back(buf_ + start, end - start); + while (x_read_buffer()) { + start = cur_; + end = buf_sz_; + for (idx_t p = start; p < end; ++p) { + const int c = buf_[p]; + if (c == '\r' || c == '\n') { + line_.push_back(buf_ + start, p - start); + if (++p == end) { + if (x_read_buffer()) { + p = cur_; + end = buf_sz_; + if (p < end && c == '\r' && buf_[p] == '\n') { + ++p; + cur_ = p; + eol_length_ = 2; + } + } + } else { + if (c == '\r' && buf_[p] == '\n') { + if (++p == end) { + x_read_buffer(); + p = cur_; + } + eol_length_ = 2; + } + cur_ = p; + } + return; + } + } + line_.push_back(buf_ + start, end - start); + } } - +// take kBufferSize into local buffer bool BufferLineReader::x_read_buffer() { std::streambuf* sb = ins_->rdbuf(); @@ -138,4 +130,3 @@ BufferLineReader::~BufferLineReader() delete ins_; delete[] buf_; } - diff --git a/src/common/buffer_line_iterator.h b/src/common/buffer_line_iterator.h index 778a35b..6e944fa 100644 --- a/src/common/buffer_line_iterator.h +++ b/src/common/buffer_line_iterator.h @@ -18,7 +18,7 @@ class BufferLineReader typedef PODArray OneDataLine; public: - BufferLineReader(const char* file_name); + explicit BufferLineReader(const char* file_name); ~BufferLineReader(); OneDataLine& get_line() { return line_; } @@ -29,8 +29,10 @@ class BufferLineReader --line_number_; unget_line_ = true; } - idx_t line_number() { return line_number_; } idx_t line_number() const { return line_number_; } + std::streampos tellg() const { return ins_->tellg() - (buf_sz_ - cur_) - (unget_line_ ? line_.size() + eol_length_ : 0); } + // note that line_number() is not accurate after this call + void seekg(std::streampos pos) { ins_->seekg(pos); x_read_buffer(); } private: void x_load_long(); @@ -47,6 +49,7 @@ class BufferLineReader OneDataLine line_; bool unget_line_; idx_t line_number_; + int eol_length_; }; #endif // BUFFER_LINE_ITERATOR_H diff --git a/src/common/fasta_reader.h b/src/common/fasta_reader.h index 75b7bc2..8c6ac5d 100644 --- a/src/common/fasta_reader.h +++ b/src/common/fasta_reader.h @@ -13,6 +13,8 @@ class FastaReader public: FastaReader(const char* fasta_file_name) : m_Reader(fasta_file_name) { encode_table = get_dna_encode_table(); } idx_t read_one_seq(Sequence& seq); + std::streampos tellg() const { return m_Reader.tellg(); } + void seekg(std::streampos pos) { m_Reader.seekg(pos); } private: void x_parse_defline(const OneDataLine& line, str_t& header); diff --git a/src/common/packed_db.h b/src/common/packed_db.h index c4e29cf..6360557 100644 --- a/src/common/packed_db.h +++ b/src/common/packed_db.h @@ -25,28 +25,18 @@ class PackedDB idx_t bytes = (max_db_size / 4); safe_calloc(pac, u1_t, bytes); } - - void GetSequence(const index_t id, const bool fwd, char* seq, const index_t size_in_ovlp) - { - const index_t offset = seq_idx[id].offset; - const index_t size = seq_idx[id].size; - r_assert(size == size_in_ovlp); - index_t idx = 0; - if (fwd) - { - for (index_t i = 0; i < size; ++i) - { - uint1 c = get_char(offset + i); - seq[idx++] = c; + + void GetSequence(const idx_t id, const bool fwd, char* const seq, const idx_t size) { + r_assert(size == seq_idx[id].size); + if (fwd) { + const idx_t offset(seq_idx[id].offset); + for (idx_t i(0); i < size; ++i) { + seq[i] = get_char(offset + i); } - } - else - { - for (index_t i = size - 1; i >= 0; --i) - { - uint1 c = get_char(offset + i); - c = 3 - c; - seq[idx++] = c; + } else { + const idx_t offset(seq_idx[id].offset + size - 1); + for (idx_t i(0); i < size; ++i) { + seq[i] = 3 - get_char(offset - i); } } } diff --git a/src/common/split_database.cpp b/src/common/split_database.cpp index ca37c0c..d7ae843 100644 --- a/src/common/split_database.cpp +++ b/src/common/split_database.cpp @@ -1,9 +1,14 @@ #include "split_database.h" +// make off_t 64 bit (from ftello man page) +#define _FILE_OFFSET_BITS 64 + #include #include #include +#include // PATH_MAX +#include // unlink() #include "packed_db.h" #include "fasta_reader.h" @@ -67,18 +72,24 @@ insert_one_offset(offset_list_t* list, const int offset, const int size) ++list->curr; } -volume_t* -new_volume_t(int num_reads, int num_bases) -{ - volume_t* volume = (volume_t*)malloc(sizeof(volume_t)); - volume->num_reads = 0; - volume->curr = 0; - if (num_bases == 0) num_bases = (MCS + MSS); - volume->max_size = num_bases; - idx_t vol_bytes = (num_bases + 3) / 4; - safe_calloc(volume->data, uint8_t, vol_bytes); - volume->offset_list = new_offset_list_t(num_reads); - return volume; +volume_t* new_volume_t(const int num_reads, int num_bases, const int no_allocate) { + volume_t* volume = (volume_t*)malloc(sizeof(volume_t)); + volume->num_reads = 0; + volume->curr = 0; + if (no_allocate) { + volume->max_size = 0; + volume->data = 0; + volume->offset_list = 0; + } else { + if (num_bases == 0) { + num_bases = MCS + MSS; + } + idx_t vol_bytes = (num_bases + 3) / 4; + volume->max_size = vol_bytes * 4; + safe_calloc(volume->data, uint8_t, vol_bytes); + volume->offset_list = new_offset_list_t(num_reads); + } + return volume; } void @@ -87,15 +98,21 @@ clear_volume_t(volume_t* v) assert(v); v->num_reads = 0; v->curr = 0; - v->offset_list->curr = 0; - memset(v->data, 0, v->max_size / 4); + if (v->offset_list) { + v->offset_list->curr = 0; + } + if (v->data) { + memset(v->data, 0, v->max_size / 4); + } } volume_t* delete_volume_t(volume_t* v) { - v->offset_list = delete_offset_list_t(v->offset_list); - free(v->data); + delete_offset_list_t(v->offset_list); + if (v->data) { + free(v->data); + } free(v); return NULL; } @@ -119,7 +136,7 @@ add_one_seq(volume_t* volume, const char* s, const int size) } void -extract_one_seq(volume_t* v, const int id, char* s) +extract_one_seq(const volume_t* v, const int id, char* s) { assert(id < v->num_reads); int offset = v->offset_list->offset_list[id].offset; @@ -152,6 +169,28 @@ dump_volume(const char* vol_name, volume_t* v) fclose(out); } +volume_t* +load_volume_header(const char* vol_name) +{ + int num_reads, num_bases; + FILE* in = fopen(vol_name, "rb"); + if (!in) { LOG(stderr, "failed to open file \'%s\'.", vol_name); exit(1); } + + // 1) number of reads + SAFE_READ(&num_reads, int, 1, in); + // 2) number of bases + SAFE_READ(&num_bases, int, 1, in); + + volume_t* v = new_volume_t(num_reads, num_bases, 1); + v->num_reads = num_reads; + v->curr = num_bases; + // 3) start read id + SAFE_READ(&v->start_read_id, int, 1, in); + + fclose(in); + return v; +} + volume_t* load_volume(const char* vol_name) { @@ -218,51 +257,140 @@ extract_one_seq(ifstream& pac_file, PackedDB::SeqIndex& si, u1_t* buffer, char* seq[i] = '\0'; } -int -split_raw_dataset(const char* reads, const char* wrk_dir) -{ +class SplitState { + public: + int vol, rid; + idx_t num_reads, num_nucls; + FILE *idx_file; + SplitState(const char * const reads, const char * const wrk_dir) : done_(0), rd_pos_(0), fr_(reads) { + generate_idx_file_name(wrk_dir, idx_file_name_); + idx_file = fopen(idx_file_name_, "r"); + if (idx_file) { // number of vols = lines in index file + vol = 0; + char *line; + size_t length(1024); + safe_malloc(line, char, length); + ssize_t i; + while ((i = getline(&line, &length, idx_file)) != -1) { + if (i > 0 && line[i - 1] == '\n') { + --i; + } + if (i > 0 && line[i - 1] == '\r') { + --i; + } + if (i > 0) { + ++vol; + } + } + fclose(idx_file); + done_ = 1; + return; + } + idx_file_name_tmp_ = ckpt_file_name_ = idx_file_name_; + ckpt_file_name_ += ".ckpt"; + idx_file_name_tmp_ += ".tmp"; + std::ifstream ckpt_in(ckpt_file_name_.c_str()); + if (ckpt_in) { + off_t idx_pos; + // using off_t instead of std::streampos for rd_pos to allow + // >> for reading from file + ckpt_in >> vol >> num_reads >> num_nucls >> idx_pos >> rd_pos_; + if (!ckpt_in) { + ERROR("Error reading checkpoint file %s", ckpt_file_name_.c_str()); + } + rid = num_reads; + fr_.seekg(rd_pos_); + idx_file = fopen(idx_file_name_tmp_.c_str(), "r+"); + if (fseeko(idx_file, idx_pos, SEEK_SET) == -1) { + ERROR("Could not restore checkpoint, fseeko failed"); + } + } else { + vol = rid = 0; + num_reads = num_nucls = 0; + idx_file = fopen(idx_file_name_tmp_.c_str(), "w"); + } + if (!idx_file) { + ERROR("Could not open index file: %s", idx_file_name_tmp_.c_str()); + } + } + ~SplitState() { } + idx_t read_one_seq(Sequence& seq) { + // have to get position before reading, as read might not be + // covered by next checkpoint + rd_pos_ = fr_.tellg(); + return fr_.read_one_seq(seq); + } + void checkpoint() { + std::string ckpt_file_name_tmp(ckpt_file_name_ + ".tmp"); + std::ofstream ckpt_out(ckpt_file_name_tmp.c_str()); + if (!ckpt_out) { + LOG(stderr, "Checkpoint failed at %d, could not open: %s", vol, ckpt_file_name_tmp.c_str()); + return; + } + ckpt_out << vol << "\n" << num_reads << "\n" << num_nucls << "\n" << ftello(idx_file) << "\n" << rd_pos_ << "\n"; + if (!ckpt_out) { + LOG(stderr, "Warning: error writing checkpoint at %d", vol); + return; + } + ckpt_out.close(); + if (rename(ckpt_file_name_tmp.c_str(), ckpt_file_name_.c_str()) == -1) { + LOG(stderr, "Checkpoint failed at %d, could not rename: %s", vol, ckpt_file_name_tmp.c_str()); + } + } + void finish() { + fclose(idx_file); + if (rename(idx_file_name_tmp_.c_str(), idx_file_name_) == -1) { + ERROR("Could not rename index file: %s", idx_file_name_tmp_.c_str()); + } + unlink(ckpt_file_name_.c_str()); + } + bool already_done() const { + return done_; + } + private: + bool done_; + off_t rd_pos_; + std::string ckpt_file_name_; // idx_file_name + ".ckpt" + std::string idx_file_name_tmp_; // idx_file_name + ".tmp" + char idx_file_name_[PATH_MAX]; + FastaReader fr_; +}; + +int split_raw_dataset(const char* reads, const char* wrk_dir) { DynamicTimer dtimer(__func__); - volume_t* v = new_volume_t(0, 0); - int vol = 0; - int rid = 0; - char idx_file_name[1024], vol_file_name[1024]; - generate_idx_file_name(wrk_dir, idx_file_name); - FILE* idx_file = fopen(idx_file_name, "w"); - FastaReader fr(reads); + SplitState state(reads, wrk_dir); + if (state.already_done()) { + return state.vol; + } + volume_t* v(new_volume_t(0, 0)); + char vol_file_name[PATH_MAX]; Sequence read; - idx_t num_reads = 0, num_nucls = 0; - while (1) - { - idx_t rsize = fr.read_one_seq(read); - if (rsize == -1) break; - ++num_reads; - num_nucls += rsize; - if (v->curr + rsize + 1 > MCS) - { - v->start_read_id = rid; - rid += v->num_reads; - generate_vol_file_name(wrk_dir, vol++, vol_file_name); - fprintf(idx_file, "%s\n", vol_file_name); + for (;;) { + const idx_t rsize(state.read_one_seq(read)); + if (v->curr + rsize >= MCS || rsize == -1) { + v->start_read_id = state.rid; + state.rid += v->num_reads; + generate_vol_file_name(wrk_dir, state.vol++, vol_file_name); + if (fprintf(state.idx_file, "%s\n", vol_file_name) < 0) { + ERROR("Failed to write to index file"); + } + fflush(state.idx_file); dump_volume(vol_file_name, v); + if (rsize == -1) { + break; + } + state.checkpoint(); clear_volume_t(v); } add_one_seq(v, read.sequence().data(), rsize); + ++state.num_reads; + state.num_nucls += rsize; ++v->curr; } - - if (v->curr > 0) - { - v->start_read_id = rid; - rid += v->num_reads; - generate_vol_file_name(wrk_dir, vol++, vol_file_name); - fprintf(idx_file, "%s\n", vol_file_name); - dump_volume(vol_file_name, v); - clear_volume_t(v); - } - fclose(idx_file); + state.finish(); delete_volume_t(v); - LOG(stderr, "split \'%s\' (%lld reads, %lld nucls) into %d volumes.", reads, (long long)num_reads, (long long)num_nucls, vol); - return vol; + LOG(stderr, "split \'%s\' (%lld reads, %lld nucls) into %d volumes.", reads, (long long)state.num_reads, (long long)state.num_nucls, state.vol); + return state.vol; } void @@ -282,7 +410,7 @@ split_dataset(const char* reads, const char* wrk_dir, int* num_vols) volume_t* v = new_volume_t(0, 0); int vol = 0; int rid = 0; - char idx_file_name[1024], vol_file_name[1024]; + char idx_file_name[PATH_MAX], vol_file_name[PATH_MAX]; generate_idx_file_name(wrk_dir, idx_file_name); FILE* idx_file = fopen(idx_file_name, "w"); PackedDB::SeqIndex si; @@ -375,6 +503,10 @@ load_volume_names(const char* idx_file_name, int num_vols) { volume_names_t* vn = new_volume_names_t(num_vols); FILE* fvn = fopen(idx_file_name, "r"); + // allow testing to see if file exists + if (!fvn && num_vols == 0) { + return vn; + } assert(fvn); char* name; size_t ns = 1024; diff --git a/src/common/split_database.h b/src/common/split_database.h index 62bd70d..54fe41e 100644 --- a/src/common/split_database.h +++ b/src/common/split_database.h @@ -24,7 +24,7 @@ typedef struct { } volume_t; volume_t* -new_volume_t(int num_reads, int num_bases); +new_volume_t(int num_reads, int num_bases, int no_allocate = 0); void clear_volume_t(volume_t* v); @@ -66,6 +66,9 @@ get_read_id_from_offset_list(offset_list_t* list, const int offset); volume_t* load_volume(const char* vol_name); +volume_t* +load_volume_header(const char* vol_name); + void generate_vol_file_name(const char* wrk_dir, int vol, char* vol_file_name); @@ -76,7 +79,7 @@ void split_dataset(const char* reads, const char* wrk_dir, int* num_vols); void -extract_one_seq(volume_t* v, const int id, char* s); +extract_one_seq(const volume_t* v, const int id, char* s); int split_raw_dataset(const char* reads, const char* wrk_dir); diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index ac4899e..03c44bd 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -375,181 +375,159 @@ void dw_in_one_direction(const char* query, const int query_size, const char* ta } } -int dw(const char* query, const int query_size, const int query_start, - const char* target, const int target_size, const int target_start, - int* U, int* V, Alignment* align, DPathData2* d_path, - PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, - double error_rate, const int min_aln_size) -{ - result->init(); - align->init(); - // left extend - dw_in_one_direction(query + query_start - 1, query_start, - target + target_start - 1, target_start, - U, V, align, d_path, aln_path, swp, result, - 0, error_rate); - align->init(); - // right extend - dw_in_one_direction(query + query_start, query_size - query_start, - target + target_start, target_size - target_start, - U, V, align, d_path, aln_path, swp, result, - 1, error_rate); - - // merge the results - int i, j, k, idx = 0; - const char* encode2char = "ACGT-"; - for (k = result->left_store_size - 1, i = 0, j = 0; k > - 1; --k, ++idx) - { - unsigned char ch = result->left_store1[k]; +int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, int* U, int* V, Alignment* align, DPathData2* d_path, PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, double error_rate, const int min_aln_size) { + result->init(); + align->init(); + // left extend + dw_in_one_direction(query + query_start - 1, query_start, target + target_start - 1, target_start, U, V, align, d_path, aln_path, swp, result, 0, error_rate); + align->init(); + // right extend + dw_in_one_direction(query + query_start, query_size - query_start, target + target_start, target_size - target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); + // merge the results + int i, j, k, idx = 0; + const char* encode2char("ACGT-"); + for (k = result->left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { + unsigned char ch(result->left_store1[k]); r_assert(ch >= 0 && ch <= 4); ch = encode2char[ch]; result->out_store1[idx] = ch; - if (ch != '-') ++i; - + if (ch != '-') { + ++i; + } ch = result->left_store2[k]; r_assert(ch >= 0 && ch <= 4); ch = encode2char[ch]; result->out_store2[idx] = ch; - if (ch != '-')++j; - } - result->query_start = query_start - i; - if (result->query_start < 0) - { + if (ch != '-') { + ++j; + } + } + result->query_start = query_start - i; + if (result->query_start < 0) { std::cerr << "query_start = " << query_start << ", i = " << i << "\n"; } r_assert(result->query_start >= 0); - result->target_start = target_start - j; + result->target_start = target_start - j; r_assert(result->target_start >= 0); - for (k = 0, i = 0, j = 0; k < result->right_store_size; ++k, ++idx) - { - - unsigned char ch = result->right_store1[k]; + for (k = 0, i = 0, j = 0; k < result->right_store_size; ++k, ++idx) { + unsigned char ch(result->right_store1[k]); r_assert(ch >= 0 && ch <= 4); ch = encode2char[ch]; result->out_store1[idx] = ch; - if (ch != '-') ++i; - + if (ch != '-') { + ++i; + } ch = result->right_store2[k]; r_assert(ch >= 0 && ch <= 4); ch = encode2char[ch]; result->out_store2[idx] = ch; - if (ch != '-') ++j; - } - result->out_store_size = idx; - result->query_end = query_start + i; - result->target_end = target_start + j; - - if (result->out_store_size >= min_aln_size) - { - int mat = 0, mis = 0, ins = 0, del = 0; - for (j = 0; j < result->out_store_size; ++j) - { - if (result->out_store1[j] == result->out_store2[j]) - { - ++mat; - result->out_match_pattern[j] = '|'; - } - else if (result->out_store1[j] == '-') - { - ++ins; - result->out_match_pattern[j] = '*'; - } - else if (result->out_store2[j] == '-') - { - ++del; - result->out_match_pattern[j] = '*'; - } - else - { - ++mis; - result->out_match_pattern[j] = '*'; - } - } - result->out_store1[result->out_store_size] = '\0'; - result->out_store2[result->out_store_size] = '\0'; - result->out_match_pattern[result->out_store_size] = '\0'; - result->mat = mat; - result->mis = mis; - result->ins = ins; - result->del = del; - result->ident = 100.0 * mat / result->out_store_size; - - return 1; - } - return 0; + if (ch != '-') { + ++j; + } + } + result->out_store_size = idx; + result->query_end = query_start + i; + result->target_end = target_start + j; + if (result->out_store_size < min_aln_size) { + return 0; + } + int mat(0), mis(0), ins(0), del(0); + for (j = 0; j < result->out_store_size; ++j) { + if (result->out_store1[j] == result->out_store2[j]) { + ++mat; + result->out_match_pattern[j] = '|'; + } else if (result->out_store1[j] == '-') { + ++ins; + result->out_match_pattern[j] = '*'; + } else if (result->out_store2[j] == '-') { + ++del; + result->out_match_pattern[j] = '*'; + } else { + ++mis; + result->out_match_pattern[j] = '*'; + } + } + result->out_store1[result->out_store_size] = 0; + result->out_store2[result->out_store_size] = 0; + result->out_match_pattern[result->out_store_size] = 0; + result->mat = mat; + result->mis = mis; + result->ins = ins; + result->del = del; + result->ident = double(100) * mat / result->out_store_size; + return 1; } -bool GetAlignment(const char* query, const int query_start, const int query_size, - const char* target, const int target_start, const int target_size, - DiffRunningData* drd, M5Record& m5, double error_rate, - const int min_aln_size) -{ - int flag = dw(query, query_size, query_start, - target, target_size, target_start, - drd->DynQ, drd->DynT, - drd->align, drd->d_path, - drd->aln_path, drd->result, - &drd->swp, error_rate, min_aln_size); - if (!flag) return false; - - int qrb = 0, qre = 0; - int trb = 0, tre = 0; - int eit = 0, k = 0; - const int consecutive_match_region_size = 4; - for (k = 0; k < drd->result->out_store_size && eit < consecutive_match_region_size; ++k) - { - const char qc = drd->result->out_store1[k]; - const char tc = drd->result->out_store2[k]; - if (qc != '-') ++qrb; - if (tc != '-') ++trb; - if (qc == tc) ++eit; - else eit = 0; +bool GetAlignment(const char* const query, const int query_start, const int query_size, const char* const target, const int target_start, const int target_size, DiffRunningData* const drd, M5Record& m5, const double error_rate, const int min_aln_size) { + if (!dw(query, query_size, query_start, target, target_size, target_start, drd->DynQ, drd->DynT, drd->align, drd->d_path, drd->aln_path, drd->result, &drd->swp, error_rate, min_aln_size)) { + return 0; + } + const int consecutive_match_region_size(4); + // trim starting end of alignment + int qrb(0); // q starting pads + int trb(0); // t starting pads + int eit(0); // matching run length + int k; + for (k = 0; k < drd->result->out_store_size; ++k) { + const char qc(drd->result->out_store1[k]); + const char tc(drd->result->out_store2[k]); + if (qc != '-') { + ++qrb; + } + if (tc != '-') { + ++trb; + } + if (qc != tc) { + eit = 0; + } else if (++eit == consecutive_match_region_size) { + ++k; + break; + } + } + if (eit < consecutive_match_region_size) { // no good match + return 0; } - if (eit < consecutive_match_region_size) return false; - k -= consecutive_match_region_size; qrb -= consecutive_match_region_size; trb -= consecutive_match_region_size; - const int start_aln_id = k; - if (start_aln_id < 0) - { - std::cout << qrb << "\t" << trb << "\t" << eit << "\t" << k << "\n"; - } - - for (k = drd->result->out_store_size - 1, eit = 0; k >= 0 && eit < consecutive_match_region_size; --k) - { - const char qc = drd->result->out_store1[k]; - const char tc = drd->result->out_store2[k]; - if (qc != '-') ++qre; - if (tc != '-') ++tre; - if (qc == tc) ++eit; - else eit = 0; + const int start_aln_id(k - consecutive_match_region_size); + // trim trailing end of alignment + int qre(0); // q ending pads + int tre(0); // t ending pads + for (k = drd->result->out_store_size - 1, eit = 0; start_aln_id < k; --k) { + const char qc(drd->result->out_store1[k]); + const char tc(drd->result->out_store2[k]); + if (qc != '-') { + ++qre; + } + if (tc != '-') { + ++tre; + } + if (qc != tc) { + eit = 0; + } else if (++eit == consecutive_match_region_size) { + --k; + break; + } } - if (eit < consecutive_match_region_size) return false; - k += consecutive_match_region_size; qre -= consecutive_match_region_size; tre -= consecutive_match_region_size; - const int end_aln_id = k + 1; - + const int end_aln_id(k + consecutive_match_region_size + 1); m5qsize(m5) = query_size; m5qoff(m5) = drd->result->query_start + qrb; m5qend(m5) = drd->result->query_end - qre; m5qdir(m5) = FWD; - m5ssize(m5) = target_size; m5soff(m5) = drd->result->target_start + trb; m5send(m5) = drd->result->target_end - tre; m5sdir(m5) = FWD; - - const int aln_size = end_aln_id - start_aln_id; - + const int aln_size(end_aln_id - start_aln_id); memcpy(m5qaln(m5), drd->result->out_store1 + start_aln_id, aln_size); memcpy(m5saln(m5), drd->result->out_store2 + start_aln_id, aln_size); memcpy(m5pat(m5), drd->result->out_match_pattern + start_aln_id, aln_size); m5qaln(m5)[aln_size] = '\0'; m5saln(m5)[aln_size] = '\0'; m5pat(m5)[aln_size] = '\0'; - - return true; + return 1; } } // end namespace ns_banded_sw diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 50470d4..a2a844b 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -178,10 +178,7 @@ int dw(const char* query, const int query_size, const int query_start, PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, double error_rate, const int min_aln_size); -bool GetAlignment(const char* query, const int query_start, const int query_size, - const char* target, const int target_start, const int target_size, - DiffRunningData* drd, M5Record& m5, double error_rate, - const int min_aln_size); +bool GetAlignment(const char* query, int query_start, int query_size, const char* target, int target_start, int target_size, DiffRunningData* drd, M5Record& m5, double error_rate, int min_aln_size); } // end of namespace ns_banded_sw diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 7c4a186..b7f8cb7 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -1,27 +1,145 @@ #include "reads_correction_can.h" #include "reads_correction_m4.h" +#include "overlaps_partition.h" +#include "options.h" -int main(int argc, char** argv) -{ - ReadsCorrectionOptions rco; - int r = parse_arguments(argc, argv, rco); - print_options(rco); - if (r) { - print_usage(argv[0]); +#include +#include +#include +#include +#include // ... unlink() +#include // S_IRUSR, S_IXUSR +#include // chmod() + +static void grid_start(const char* const prog, const ReadsCorrectionOptions &options, const int i) { + // create grid script, have grid run it + ReadsCorrectionOptions new_options(options); + new_options.job_index = i; + new_options.grid_options = NULL; + new_options.grid_options_split = NULL; + std::string name("m2cns."); + if (i == -1) { + name += "split"; + new_options.num_threads = 0; // flag as grid spin-off + } else { + std::ostringstream x; + x << i; + name += x.str(); + } + std::string script_file(name + ".sh"); + // as we might not have write permission, delete it + unlink(script_file.c_str()); + std::ofstream out; + open_fstream(out, script_file.c_str(), std::ios::out); + out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; + if (!out) { + std::cerr << "Error writing to " << script_file << "\n"; exit(1); } - if (rco.print_usage_info) { + close_fstream(out); + chmod(script_file.c_str(), S_IRUSR | S_IXUSR); + std::string cmd(i == -1 && options.grid_options_split ? options.grid_options_split : options.grid_options); + cmd += " " + name + " " + script_file; + assert(system(cmd.c_str()) == 0); +} + +// pass by value so we can modify list to easily avoid checking previously +// found results files +static void wait_for_files(std::list results) { + const std::list::const_iterator end_a(results.end()); + while (!results.empty()) { + sleep(60); + std::list::iterator a(results.begin()); + while (a != end_a) { + if (access(a->c_str(), F_OK) == 0) { + a = results.erase(a); + } else { + ++a; + } + } + } +} + +static void merge_results(const char* const output, const std::list& files) { + std::string out_tmp(output); + out_tmp += ".tmp"; + std::ofstream out(out_tmp.c_str()); + std::list::const_iterator a(files.begin()); + const std::list::const_iterator end_a(files.end()); + for (; a != end_a; ++a) { + std::ifstream in(a->c_str()); + out << in.rdbuf(); + if (!in) { + std::cerr << "Error reading from " << *a << "\n"; + exit(1); + } else if (!out) { + std::cerr << "Error writing to " << out_tmp << "\n"; + exit(1); + } + } + out.close(); + if (rename(out_tmp.c_str(), output) == -1) { + std::cerr << "Could not rename concatenated output file: " << out_tmp << "\n"; + exit(1); + } +} + +int main(int argc, char** argv) { + ReadsCorrectionOptions rco; + if (parse_arguments(argc, argv, rco)) { + print_usage(argv[0]); + exit(1); + } else if (rco.print_usage_info) { print_usage(argv[0]); exit(0); } - - if (rco.input_type == INPUT_TYPE_CAN) - { - return reads_correction_can(rco); + // partition once up front, to allow for grid jobs + if (rco.job_index == -1) { + if (access(rco.corrected_reads, F_OK) == 0) { // full run already done + return 0; + } else if (access("partition.done", F_OK) == 0) { // split already done + } else if (rco.grid_options || rco.grid_options_split) { + grid_start(argv[0], rco, -1); + std::list partition_results; + partition_results.push_back("partition.done"); + wait_for_files(partition_results); + } else { + if (rco.input_type == INPUT_TYPE_CAN) { + partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); + } else { + partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); + } + if (rco.num_threads == 0) { // flag saying we're a grid run + return 0; + } + } } - else - { - return reads_correction_m4(rco); + if (rco.grid_options == NULL) { // single process run + if (rco.input_type == INPUT_TYPE_CAN) { + return reads_correction_can(rco); + } else { + return reads_correction_m4(rco); + } + } else { // grid run + // get number of partitions + std::string idx_file_name; + generate_partition_index_file_name(rco.m4, idx_file_name); + std::vector partition_file_vec; + load_partition_files_info(idx_file_name.c_str(), partition_file_vec); + std::list results; + for (size_t i(0); i != partition_file_vec.size(); ++i) { + std::ostringstream os; + os << rco.corrected_reads << "." << i; + results.push_back(os.str()); + if (access(os.str().c_str(), F_OK) != 0) { + grid_start(argv[0], rco, i); + if (rco.grid_start_delay) { + sleep(rco.grid_start_delay); + } + } + } + wait_for_files(results); + merge_results(rco.corrected_reads, results); } return 0; } diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index c96a2db..c268ae8 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -19,8 +19,9 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_LDFLAGS := -L${TARGET_DIR} -TGT_LDLIBS := -lmecat -TGT_PREREQS := libmecat.a +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 +TGT_LDFLAGS := -L${TARGET_DIR} +TGT_LDLIBS := -lmecat +TGT_PREREQS := libmecat.a SUBMAKEFILES := diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 0fc3942..411629e 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -27,18 +27,18 @@ struct CompareOverlapByOverlapSize { bool operator()(const Overlap& a ,const Overlap& b) { - const index_t ovlp_a = std::max(a.qend - a.qoff, a.send - a.soff); - const index_t ovlp_b = std::max(b.qend - b.qoff, b.send - b.soff); + const idx_t ovlp_a = std::max(a.qend - a.qoff, a.send - a.soff); + const idx_t ovlp_b = std::max(b.qend - b.qoff, b.send - b.soff); return ovlp_a > ovlp_b; } }; void -meap_add_one_aln(const std::string& qaln, const std::string& saln, index_t start_soff, CnsTableItem* cns_table, const char* org_seq) +meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* cns_table, const char* org_seq) { r_assert(qaln.size() == saln.size()); - const index_t aln_size = qaln.size(); - index_t i = 0; + const idx_t aln_size = qaln.size(); + idx_t i = 0; const char kGap = '-'; while (i < aln_size) { @@ -51,7 +51,7 @@ meap_add_one_aln(const std::string& qaln, const std::string& saln, index_t start else { r_assert(s == kGap); - index_t j = i + 1; + idx_t j = i + 1; while (j < aln_size && saln[j] == kGap) ++j; ++cns_table[start_soff - 1].del_cnt; i = j; @@ -155,8 +155,8 @@ get_effective_ranges(std::vector& mranges, std::vector& cns_results, CnsResult& cr, - const index_t beg, - const index_t end, + const idx_t beg, + const idx_t end, std::string& cns_seq) { const size_t MaxSeqSize = 60000; @@ -211,7 +211,7 @@ consensus_worker(CnsTableItem* cns_table, const int read_id, std::vector& cns_results) { - index_t beg = 0, end; + idx_t beg = 0, end; CnsResult cns_result; std::string cns_seq; cns_result.id = read_id; @@ -239,26 +239,26 @@ consensus_worker(CnsTableItem* cns_table, } void -consensus_one_read_m4_pacbio(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid) +consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { - PackedDB& reads = *ctd->reads; - ExtensionCandidate* overlaps = ctd->candidates; - DiffRunningData* drd_s = ctd->drd_s; + PackedDB& reads = ctd.reads; + ExtensionCandidate* overlaps = pctd.candidates; + DiffRunningData* drd_s = pctd.drd_s; DiffRunningData* drd = NULL; - M5Record* m5 = ctd->m5; - CnsAlns& cns_vec = ctd->cns_alns; - std::vector& cns_results = ctd->cns_results; - const index_t read_size = overlaps[sid].ssize; - std::vector& qstr = ctd->query; - std::vector& tstr = ctd->target; + M5Record& m5 = pctd.m5; + CnsAlns& cns_vec = pctd.cns_alns; + std::vector& cns_results = pctd.cns_results; + const idx_t read_size = overlaps[sid].ssize; + std::vector& qstr = pctd.query; + std::vector& tstr = pctd.target; tstr.resize(read_size); reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = ctd->qaln; - std::string& ntstr = ctd->saln; - const int min_align_size = ctd->rco.min_align_size; + std::string& nqstr = pctd.qaln; + std::string& ntstr = pctd.saln; + const int min_align_size = ctd.rco.min_align_size; const int max_added = 60; - index_t L, R; + idx_t L, R; if (eid - sid <= max_added) { L = sid; @@ -271,55 +271,55 @@ consensus_one_read_m4_pacbio(ConsensusThreadData* ctd, const index_t read_id, co std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); } - CnsTableItem* cns_table = ctd->cns_table; + CnsTableItem* cns_table = pctd.cns_table; std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); cns_vec.clear(); - for (index_t i = L; i < R; ++i) + for (idx_t i = L; i < R; ++i) { Overlap& ovlp = overlaps[i]; qstr.resize(ovlp.qsize); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); - index_t qext = ovlp.qext; - index_t sext = ovlp.sext; + idx_t qext = ovlp.qext; + idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, *m5, 0.15, min_align_size); + bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size); if (r) { - normalize_gaps(m5qaln(*m5), m5saln(*m5), strlen(m5qaln(*m5)), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(*m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(*m5), m5send(*m5), nqstr, ntstr); + normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); + cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } } std::vector mranges, eranges; cns_vec.get_mapping_ranges(mranges); - get_effective_ranges(mranges, eranges, read_size, ctd->rco.min_size); + get_effective_ranges(mranges, eranges, read_size, ctd.rco.min_size); - consensus_worker(cns_table, ctd->id_list, cns_vec, nqstr, ntstr, eranges, ctd->rco.min_cov, ctd->rco.min_size, read_id, cns_results); + consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } void -consensus_one_read_m4_nanopore(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid) +consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { - PackedDB& reads = *ctd->reads; - ExtensionCandidate* overlaps = ctd->candidates; - DiffRunningData* drd_s = ctd->drd_s; + PackedDB& reads = ctd.reads; + ExtensionCandidate* overlaps = pctd.candidates; + DiffRunningData* drd_s = pctd.drd_s; DiffRunningData* drd = NULL; - M5Record* m5 = ctd->m5; - CnsAlns& cns_vec = ctd->cns_alns; - std::vector& cns_results = ctd->cns_results; - const index_t read_size = overlaps[sid].ssize; - std::vector& qstr = ctd->query; - std::vector& tstr = ctd->target; + M5Record& m5 = pctd.m5; + CnsAlns& cns_vec = pctd.cns_alns; + std::vector& cns_results = pctd.cns_results; + const idx_t read_size = overlaps[sid].ssize; + std::vector& qstr = pctd.query; + std::vector& tstr = pctd.target; tstr.resize(read_size); reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = ctd->qaln; - std::string& ntstr = ctd->saln; - const int min_align_size = ctd->rco.min_align_size; - const double min_mapping_ratio = ctd->rco.min_mapping_ratio - 0.02; + std::string& nqstr = pctd.qaln; + std::string& ntstr = pctd.saln; + const int min_align_size = ctd.rco.min_align_size; + const double min_mapping_ratio = ctd.rco.min_mapping_ratio - 0.02; - index_t L, R; + idx_t L, R; if (eid - sid <= MAX_CNS_OVLPS) { L = sid; @@ -332,43 +332,33 @@ consensus_one_read_m4_nanopore(ConsensusThreadData* ctd, const index_t read_id, std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); } - CnsTableItem* cns_table = ctd->cns_table; + CnsTableItem* cns_table = pctd.cns_table; std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); cns_vec.clear(); - for (index_t i = L; i < R; ++i) + for (idx_t i = L; i < R; ++i) { Overlap& ovlp = overlaps[i]; qstr.resize(ovlp.qsize); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); - index_t qext = ovlp.qext; - index_t sext = ovlp.sext; + idx_t qext = ovlp.qext; + idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, *m5, 0.20, min_align_size); - if (r && check_ovlp_mapping_range(m5qoff(*m5), m5qend(*m5), ovlp.qsize, m5soff(*m5), m5send(*m5), ovlp.ssize, min_mapping_ratio)) + bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); + if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ovlp.qsize, m5soff(m5), m5send(m5), ovlp.ssize, min_mapping_ratio)) { - normalize_gaps(m5qaln(*m5), m5saln(*m5), strlen(m5qaln(*m5)), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(*m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(*m5), m5send(*m5), nqstr, ntstr); + normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); + cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } } std::vector mranges, eranges; eranges.push_back(MappingRange(0, read_size)); - consensus_worker(cns_table, ctd->id_list, cns_vec, nqstr, ntstr, eranges, ctd->rco.min_cov, ctd->rco.min_size, read_id, cns_results); + consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } -struct CmpExtensionCandidateByScore -{ - bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) - { - if (a.score != b.score) return a.score > b.score; - if (a.qid != b.qid) return a.qid < b.qid; - return a.qext < b.qext; - } -}; - inline bool check_cov_stats(u1_t* cov_stats, int soff, int send) { @@ -385,99 +375,89 @@ check_cov_stats(u1_t* cov_stats, int soff, int send) return false; } -void -consensus_one_read_can_pacbio(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid) -{ - PackedDB& reads = *ctd->reads; - ExtensionCandidate* candidates = ctd->candidates; - DiffRunningData* drd_s = ctd->drd_s; - DiffRunningData* drd = NULL; - M5Record* m5 = ctd->m5; - CnsAlns& cns_vec = ctd->cns_alns; - std::vector& cns_results = ctd->cns_results; - const index_t read_size = candidates[sid].ssize; - std::vector& qstr = ctd->query; - std::vector& tstr = ctd->target; +void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { + PackedDB& reads(ctd.reads); + ExtensionCandidate* candidates(pctd.candidates); + DiffRunningData* const drd_s(pctd.drd_s); + M5Record& m5(pctd.m5); + CnsAlns& cns_vec(pctd.cns_alns); + std::vector& cns_results(pctd.cns_results); + const idx_t read_size(candidates[sid].ssize); + std::vector& qstr(pctd.query); + std::vector& tstr(pctd.target); tstr.resize(read_size); reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = ctd->qaln; - std::string& ntstr = ctd->saln; - const int min_align_size = ctd->rco.min_align_size; - const double min_mapping_ratio = ctd->rco.min_mapping_ratio - 0.02; - const int max_added = 60; - - std::sort(candidates + sid, candidates + eid, CmpExtensionCandidateByScore()); - int num_added = 0; - int num_ext = 0; - const int max_ext = 200; - CnsTableItem* cns_table = ctd->cns_table; + std::string& nqstr(pctd.qaln); + std::string& ntstr(pctd.saln); + const int min_align_size(ctd.rco.min_align_size); + const double min_mapping_ratio(ctd.rco.min_mapping_ratio - 0.02); + int num_added(0); + const int max_added(60); + eid = std::min(eid, sid + 200); // max of 200 extents + CnsTableItem* cns_table(pctd.cns_table); std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; - u1_t* cov_stats = ctd->id_list; + u1_t* cov_stats(pctd.id_list); std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); - for (idx_t i = sid; i < eid && num_added < max_added && num_ext < max_ext; ++i) - { - ++num_ext; - ExtensionCandidate& ec = candidates[i]; + for (idx_t i(sid); i < eid && num_added < max_added; ++i) { + ExtensionCandidate& ec(candidates[i]); r_assert(ec.sdir == FWD); - if (used_ids.find(ec.qid) != used_ids.end()) continue; + if (used_ids.find(ec.qid) != used_ids.end()) { + continue; + } qstr.resize(ec.qsize); reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); - index_t qext = ec.qext; - index_t sext = ec.sext; - if (ec.qdir == REV) qext = ec.qsize - 1 - qext; - drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, *m5, 0.15, min_align_size); - if (r && check_ovlp_mapping_range(m5qoff(*m5), m5qend(*m5), ec.qsize, m5soff(*m5), m5send(*m5), ec.ssize, min_mapping_ratio)) - { - if (check_cov_stats(cov_stats, m5soff(*m5), m5send(*m5))) - { + const idx_t sext(ec.sext); + idx_t qext(ec.qext); + if (ec.qdir == REV) { + qext = ec.qsize - 1 - qext; + } + const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); + if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) { + if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5qaln(*m5), m5saln(*m5), strlen(m5qaln(*m5)), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(*m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(*m5), m5send(*m5), nqstr, ntstr); + normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); + cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } } } - std::vector mranges, eranges; cns_vec.get_mapping_ranges(mranges); - get_effective_ranges(mranges, eranges, read_size, ctd->rco.min_size); - - consensus_worker(cns_table, ctd->id_list, cns_vec, nqstr, ntstr, eranges, ctd->rco.min_cov, ctd->rco.min_size, read_id, cns_results); + get_effective_ranges(mranges, eranges, read_size, ctd.rco.min_size); + consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } void -consensus_one_read_can_nanopore(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid) +consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { - PackedDB& reads = *ctd->reads; - ExtensionCandidate* candidates = ctd->candidates; - DiffRunningData* drd_s = ctd->drd_s; + PackedDB& reads = ctd.reads; + ExtensionCandidate* candidates = pctd.candidates; + DiffRunningData* drd_s = pctd.drd_s; DiffRunningData* drd = NULL; - M5Record* m5 = ctd->m5; - CnsAlns& cns_vec = ctd->cns_alns; - std::vector& cns_results = ctd->cns_results; - const index_t read_size = candidates[sid].ssize; - std::vector& qstr = ctd->query; - std::vector& tstr = ctd->target; + M5Record& m5 = pctd.m5; + CnsAlns& cns_vec = pctd.cns_alns; + std::vector& cns_results = pctd.cns_results; + const idx_t read_size = candidates[sid].ssize; + std::vector& qstr = pctd.query; + std::vector& tstr = pctd.target; tstr.resize(read_size); reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = ctd->qaln; - std::string& ntstr = ctd->saln; - const int min_align_size = ctd->rco.min_align_size; - const double min_mapping_ratio = ctd->rco.min_mapping_ratio - 0.02; + std::string& nqstr = pctd.qaln; + std::string& ntstr = pctd.saln; + const int min_align_size = ctd.rco.min_align_size; + const double min_mapping_ratio = ctd.rco.min_mapping_ratio - 0.02; - std::sort(candidates + sid, candidates + eid, CmpExtensionCandidateByScore()); int num_added = 0; int num_ext = 0; const int max_ext = 200; - CnsTableItem* cns_table = ctd->cns_table; + CnsTableItem* cns_table = pctd.cns_table; std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; - u1_t* cov_stats = ctd->id_list; + u1_t* cov_stats = pctd.id_list; std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); for (idx_t i = sid; i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { @@ -487,20 +467,20 @@ consensus_one_read_can_nanopore(ConsensusThreadData* ctd, const index_t read_id, if (used_ids.find(ec.qid) != used_ids.end()) continue; qstr.resize(ec.qsize); reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); - index_t qext = ec.qext; - index_t sext = ec.sext; + idx_t qext = ec.qext; + idx_t sext = ec.sext; if (ec.qdir == REV) qext = ec.qsize - 1 - qext; drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, *m5, 0.20, min_align_size); - if (r && check_ovlp_mapping_range(m5qoff(*m5), m5qend(*m5), ec.qsize, m5soff(*m5), m5send(*m5), ec.ssize, min_mapping_ratio)) + bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); + if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) { - if (check_cov_stats(cov_stats, m5soff(*m5), m5send(*m5))) + if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5qaln(*m5), m5saln(*m5), strlen(m5qaln(*m5)), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(*m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(*m5), m5send(*m5), nqstr, ntstr); + normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); + cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } } } @@ -508,7 +488,7 @@ consensus_one_read_can_nanopore(ConsensusThreadData* ctd, const index_t read_id, std::vector mranges, eranges; eranges.push_back(MappingRange(0, read_size)); - consensus_worker(cns_table, ctd->id_list, cns_vec, nqstr, ntstr, eranges, ctd->rco.min_cov, ctd->rco.min_size, read_id, cns_results); + consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } } // namespace ns_meap_cns { diff --git a/src/mecat2cns/mecat_correction.h b/src/mecat2cns/mecat_correction.h index c00d463..e2204ad 100644 --- a/src/mecat2cns/mecat_correction.h +++ b/src/mecat2cns/mecat_correction.h @@ -6,16 +6,16 @@ namespace ns_meap_cns { void -consensus_one_read_m4_pacbio(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid); +consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); void -consensus_one_read_m4_nanopore(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid); +consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); void -consensus_one_read_can_pacbio(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid); +consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); void -consensus_one_read_can_nanopore(ConsensusThreadData* ctd, const index_t read_id, const index_t sid, const index_t eid); +consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); } // namespace ns_meap_cns diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index 1ddaee1..caf0cd3 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -4,31 +4,31 @@ #include #include +#include +#include -using namespace std; - -static int input_type_pacbio = 1; +static int input_type_pacbio = 1; static int num_threads_pacbio = 1; -static index_t batch_size_pacbio = 100000; -static double mapping_ratio_pacbio = 0.9; +static idx_t batch_size_pacbio = 100000; +static double mapping_ratio_pacbio = 0.9; static int align_size_pacbio = 2000; -static int cov_pacbio = 6; -static int min_size_pacbio = 5000; +static int cov_pacbio = 6; +static int min_size_pacbio = 5000; static bool print_usage_pacbio = false; -static int tech_pacbio = TECH_PACBIO; -static int num_partition_files = 10; +static int tech_pacbio = TECH_PACBIO; -static int input_type_nanopore = 1; -static int num_threads_nanopore = 1; -static index_t batch_size_nanopore = 100000; -static double mapping_ratio_nanopore = 0.4; -static int align_size_nanopore = 400; -static int cov_nanopore = 6; -static int min_size_nanopore = 2000; -static bool print_usage_nanopore = false; -static int tech_nanopore = TECH_NANOPORE; +static int input_type_nanopore = 1; +static int num_threads_nanopore = 1; +static idx_t batch_size_nanopore = 100000; +static double mapping_ratio_nanopore = 0.4; +static int align_size_nanopore = 400; +static int cov_nanopore = 6; +static int min_size_nanopore = 2000; +static bool print_usage_nanopore = false; +static int tech_nanopore = TECH_NANOPORE; static int default_tech = TECH_PACBIO; +static int num_partition_files = 0; static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -40,97 +40,126 @@ static const char min_size_n = 'l'; static const char usage_n = 'h'; static const char tech_n = 'x'; static const char num_partition_files_n = 'k'; +static const char grid_options_n = 'G'; +static const char grid_options_split_n = 'S'; +static const char job_index_n = 'I'; +static const char reads_to_correct_n = 'R'; +static const char grid_start_delay_n = 'D'; void print_pacbio_default_options() { - cerr << '-' << input_type_n << ' ' << input_type_pacbio - << ' ' - << '-' << num_threads_n << ' ' << num_threads_pacbio - << ' ' - << '-' << batch_size_n << ' ' << batch_size_pacbio - << ' ' - << '-' << mapping_ratio_n << ' ' << mapping_ratio_pacbio - << ' ' - << '-' << align_size_n << ' ' << align_size_pacbio - << ' ' - << '-' << cov_n << ' ' << cov_pacbio - << ' ' - << '-' << min_size_n << ' ' << min_size_pacbio - << ' ' - << '-' << num_partition_files_n << ' ' << num_partition_files + std::cerr << "-" << input_type_n << " " << input_type_pacbio + << " -" << num_threads_n << " " << num_threads_pacbio + << " -" << batch_size_n << " " << batch_size_pacbio + << " -" << mapping_ratio_n << " " << mapping_ratio_pacbio + << " -" << align_size_n << " " << align_size_pacbio + << " -" << cov_n << " " << cov_pacbio << " " + << " -" << min_size_n << " " << min_size_pacbio << "\n"; } void print_nanopore_default_options() { - cerr << '-' << input_type_n << ' ' << input_type_nanopore - << ' ' - << '-' << num_threads_n << ' ' << num_threads_nanopore - << ' ' - << '-' << batch_size_n << ' ' << batch_size_nanopore - << ' ' - << '-' << mapping_ratio_n << ' ' << mapping_ratio_nanopore - << ' ' - << '-' << align_size_n << ' ' << align_size_nanopore - << ' ' - << '-' << cov_n << ' ' << cov_nanopore - << ' ' - << '-' << min_size_n << ' ' << min_size_nanopore - << ' ' - << '-' << num_partition_files_n << ' ' << num_partition_files + std::cerr << "-" << input_type_n << " " << input_type_nanopore + << " -" << num_threads_n << " " << num_threads_nanopore + << " -" << batch_size_n << " " << batch_size_nanopore + << " -" << mapping_ratio_n << " " << mapping_ratio_nanopore + << " -" << align_size_n << " " << align_size_nanopore + << " -" << cov_n << " " << cov_nanopore + << " -" << min_size_n << " " << min_size_nanopore << "\n"; } -void -print_usage(const char* prog) +// given options, recreate arguments from the command line +std::string +make_options(const ConsensusOptions& options) { - cerr << "USAGE:\n" - << prog << ' ' << "[options]" << ' ' << "input" << ' ' << "reads" << ' ' << "output" << "\n"; - cerr << "\n" << "OPTIONS:" << "\n"; - - cerr << "-" << tech_n << " <0/1>\t" << "sequencing platform: 0 = PACBIO, 1 = NANOPORE" << "\n" - << "\t\t" << "default: 0" << "\n"; - - cerr << "-" << input_type_n << " <0/1>\t" << "input type: 0 = candidte, 1 = m4" << "\n"; - - cerr << "-" << num_threads_n << " \t" << "number of threads (CPUs)" << "\n"; - - cerr << "-" << batch_size_n << " \t" << "batch size that the reads will be partitioned" << "\n"; - - cerr << "-" << mapping_ratio_n << " \t" << "minimum mapping ratio" << "\n"; - - cerr << "-" << align_size_n << " \t" << "minimum overlap size" << "\n"; - - cerr << "-" << cov_n << " \t" << "minimum coverage under consideration" << "\n"; - - cerr << "-" << min_size_n << " \t" << "minimum length of corrected sequence" << "\n"; - - cerr << "-" << num_partition_files_n << " \t" - << "number of partition files when partitioning overlap results" - << " (if < 0, then it will be set to system limit value)" - << "\n"; - - cerr << "-" << usage_n << "\t\t" << "print usage info." << "\n"; - - cerr << "\n" - << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; + std::ostringstream cmd; + cmd << " -" << input_type_n << " " << (options.input_type == INPUT_TYPE_CAN ? 0 : 1); + if (options.num_threads > -1) { + cmd << " -" << num_threads_n << " " << options.num_threads; + } + if (options.batch_size > 0) { + cmd << " -" << batch_size_n << " " << options.batch_size; + } + if (options.min_mapping_ratio >= 0) { + cmd << " -" << mapping_ratio_n << " " << options.min_mapping_ratio; + } + if (options.min_align_size >= 0) { + cmd << " -" << align_size_n << " " << options.min_align_size; + } + if (options.min_cov >= 0) { + cmd << " -" << cov_n << " " << options.min_cov; + } + if (options.min_size >= 0) { + cmd << " -" << min_size_n << " " << options.min_size; + } + if (options.num_partition_files > 0) { + cmd << " -" << num_partition_files_n << " " << options.num_partition_files; + } + if (options.grid_options != NULL) { + cmd << " -" << grid_options_n << " \"" << options.grid_options << "\""; + } + if (options.grid_options_split != NULL) { + cmd << " -" << grid_options_split_n << " \"" << options.grid_options_split << "\""; + } + if (options.job_index != -1) { + cmd << " -" << job_index_n << " " << options.job_index; + } + if (options.reads_to_correct) { + cmd << " -" << reads_to_correct_n << " " << options.reads_to_correct; + } + if (options.grid_start_delay) { + cmd << " -" << grid_start_delay_n << " " << options.grid_start_delay; + } + cmd << " " << options.m4; + cmd << " " << options.reads; + cmd << " " << options.corrected_reads; + return cmd.str(); +} + +void print_usage(const char* prog) { + std::cerr << "USAGE:\n" + << prog << " [options] input reads output\n" + << "\n" + << "OPTIONS:\n" + << "-" << tech_n << " <0/1>\tsequencing platform: 0 = PACBIO, 1 = NANOPORE\n" + << "\t\tdefault: 0\n" + << "-" << input_type_n << " <0/1>\tinput type: 0 = candidate, 1 = m4\n" + << "-" << num_threads_n << " \tnumber of threads (CPUs)\n" + << "-" << batch_size_n << " \tbatch size that the reads will be partitioned\n" + << "-" << mapping_ratio_n << " \tminimum mapping ratio\n" + << "-" << align_size_n << " \tminimum overlap size\n" + << "-" << cov_n << " \tminimum coverage under consideration\n" + << "-" << min_size_n << " \tminimum length of corrected sequence\n" + << "-" << num_partition_files_n << " \tnumber of partition files when partitioning overlap results (if 0, then use system limit)\n" + << "-" << grid_options_n << " \toptions for grid submission\n" + << "-" << grid_options_split_n << " \toptions for split grid submission\n" + << "-" << reads_to_correct_n << " \tnumber of reads to correct [all]\n" + << "-" << grid_start_delay_n << " \tseconds to delay between starting grid jobs\n" + << "-" << usage_n << "\t\tprint usage info.\n" + << "\n" + << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; print_pacbio_default_options(); - - cerr << "\n" - << "If 'x' is set to be '1' (nanopore), then the other options have the following default values: \n"; + std::cerr << "\n" + << "If 'x' is set to be '1' (nanopore), then the other options have the following default values: \n"; print_nanopore_default_options(); } -ConsensusOptions -init_consensus_options(int tech) -{ +ConsensusOptions init_consensus_options(const int tech) { ConsensusOptions t; + t.m4 = NULL; + t.reads = NULL; + t.corrected_reads = NULL; + t.grid_options = NULL; + t.grid_options_split = NULL; + t.num_partition_files = num_partition_files; + t.job_index = -1; + t.reads_to_correct = 0; + t.grid_start_delay = 0; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; - t.m4 = NULL; - t.reads = NULL; - t.corrected_reads = NULL; t.num_threads = num_threads_pacbio; t.batch_size = batch_size_pacbio; t.min_mapping_ratio = mapping_ratio_pacbio; @@ -138,13 +167,9 @@ init_consensus_options(int tech) t.min_cov = cov_pacbio; t.min_size = min_size_pacbio; t.print_usage_info = print_usage_pacbio; - t.num_partition_files = num_partition_files; t.tech = tech_pacbio; } else { t.input_type = input_type_nanopore; - t.m4 = NULL; - t.reads = NULL; - t.corrected_reads = NULL; t.num_threads = num_threads_nanopore; t.batch_size = batch_size_nanopore; t.min_mapping_ratio = mapping_ratio_nanopore; @@ -152,60 +177,53 @@ init_consensus_options(int tech) t.min_cov = cov_nanopore; t.min_size = min_size_nanopore; t.print_usage_info = print_usage_nanopore; - t.num_partition_files = num_partition_files; t.tech = tech_nanopore; } - return t; + return t; } -int detect_tech(int argc, char* argv[]) -{ - int t = default_tech; - char tech_nstr[64]; tech_nstr[0] = '-'; tech_nstr[1] = tech_n; tech_nstr[2] = '\0'; - - for (int i = 0; i < argc; ++i) { - if (strcmp(tech_nstr, argv[i]) == 0) { - if (i + 1 == argc) { - fprintf(stderr, "argument to option '%c' is missing.\n", tech_n); - t = -1; - } - cout << tech_nstr << "\n"; - if (argv[i + 1][0] == '0') { - t = TECH_PACBIO; - } else if (argv[i + 1][0] == '1') { - t = TECH_NANOPORE; - } else { - fprintf(stderr, "invalid argument to option '%c': %s\n", tech_n, argv[i + 1]); - t = -1; - } - break; - } - } - - return t; +int detect_tech(int argc, char* argv[]) { + int t = default_tech; + char tech_nstr[64]; tech_nstr[0] = '-'; tech_nstr[1] = tech_n; tech_nstr[2] = '\0'; + for (int i = 0; i < argc; ++i) { + if (strcmp(tech_nstr, argv[i]) == 0) { + if (i + 1 == argc) { + fprintf(stderr, "argument to option '%c' is missing.\n", tech_n); + t = -1; + } + std::cout << tech_nstr << "\n"; + if (argv[i + 1][0] == '0') { + t = TECH_PACBIO; + } else if (argv[i + 1][0] == '1') { + t = TECH_NANOPORE; + } else { + fprintf(stderr, "invalid argument to option '%c': %s\n", tech_n, argv[i + 1]); + t = -1; + } + break; + } + } + return t; } -int -parse_arguments(int argc, char* argv[], ConsensusOptions& t) -{ +int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { bool parse_success = true; int tech = detect_tech(argc, argv); if (tech == -1) { return 1; } t = init_consensus_options(tech); - int opt_char; - char err_char; - opterr = 0; - while((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:k:h")) != -1) { + char err_char; + opterr = 0; + while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:")) != -1) { switch (opt_char) { case input_type_n: - if (optarg[0] == '0') + if (optarg[0] == '0') { t.input_type = INPUT_TYPE_CAN; - else if (optarg[0] == '1') + } else if (optarg[0] == '1') { t.input_type = INPUT_TYPE_M4; - else { + } else { fprintf(stderr, "invalid argument to option '%c': %s\n", input_type_n, optarg); return 1; } @@ -231,73 +249,90 @@ parse_arguments(int argc, char* argv[], ConsensusOptions& t) case usage_n: t.print_usage_info = true; break; + case grid_options_n: + t.grid_options = optarg; + break; + case grid_options_split_n: + t.grid_options_split = optarg; + break; + case job_index_n: + t.job_index = atoi(optarg); + break; + case reads_to_correct_n: + t.reads_to_correct = atoi(optarg); + break; + case grid_start_delay_n: + t.grid_start_delay = atoi(optarg); + break; case tech_n: break; case num_partition_files_n: t.num_partition_files = atoi(optarg); break; case '?': - err_char = (char)optopt; + err_char = (char)optopt; fprintf(stderr, "unrecognised option '%c'\n", err_char); - return 1; - break; - case ':': - err_char = (char)optopt; + return 1; + break; + case ':': + err_char = (char)optopt; fprintf(stderr, "argument to option '%c' is missing.\n", err_char); - return 1; - break; + return 1; + break; } } - - if (t.num_threads <= 0) - { + if (t.num_threads < 0) { std::cerr << "cpu threads must be greater than 0\n"; parse_success = false; } - if (t.batch_size <= 0) - { + if (t.batch_size <= 0) { std::cerr << "batch size must be greater than 0\n"; parse_success = false; } - if (t.min_mapping_ratio < 0.0) - { + if (t.min_mapping_ratio < 0.0) { std::cerr << "mapping ratio must be >= 0.0\n"; parse_success = false; } - if (t.min_cov < 0) - { + if (t.min_cov < 0) { std::cerr << "coverage must be >= 0\n"; parse_success = false; } - if (t.min_cov < 0) - { + if (t.min_size < 0) { std::cerr << "sequence size must be >= 0\n"; parse_success = false; } - - if (argc < 3) return 1; - + if (t.reads_to_correct < 0) { + std::cerr << "number of reads must be >= 0\n"; + parse_success = false; + } + if (t.grid_start_delay < 0) { + std::cerr << "grid start delay must be >= 0\n"; + parse_success = false; + } + if (argc - optind < 3) { + return 1; + } t.m4 = argv[argc - 3]; t.reads = argv[argc - 2]; t.corrected_reads = argv[argc - 1]; - - if (parse_success) return 0; - return 1; + return parse_success ? 0 : 1; } void print_options(ConsensusOptions& t) { - cout << "input_type:\t" << t.input_type << "\n"; - if (t.m4) cout << "reads\t" << t.m4 << "\n"; - if (t.reads) cout << "output\t" << t.reads << "\n"; - if (t.corrected_reads) cout << "m4\t" << t.corrected_reads << "\n"; - cout << "number of threads:\t" << t.num_threads << "\n"; - cout << "batch size:\t" << t.batch_size << "\n"; - cout << "mapping ratio:\t" << t.min_mapping_ratio << "\n"; - cout << "align size:\t" << t.min_align_size << "\n"; - cout << "cov:\t" << t.min_cov << "\n"; - cout << "min size:\t" << t.min_size << "\n"; - cout << "partition files:\t" << t.num_partition_files << "\n"; - cout << "tech:\t" << t.tech << "\n"; + std::cout << "input_type:\t" << t.input_type << "\n"; + if (t.m4) std::cout << "reads\t" << t.m4 << "\n"; + if (t.reads) std::cout << "output\t" << t.reads << "\n"; + if (t.corrected_reads) std::cout << "m4\t" << t.corrected_reads << "\n"; + if (t.grid_options) std::cout << "grid\t" << t.grid_options << "\n"; + if (t.grid_options_split) std::cout << "grid_split\t" << t.grid_options_split << "\n"; + std::cout << "number of threads:\t" << t.num_threads << "\n"; + std::cout << "batch size:\t" << t.batch_size << "\n"; + std::cout << "mapping ratio:\t" << t.min_mapping_ratio << "\n"; + std::cout << "align size:\t" << t.min_align_size << "\n"; + std::cout << "cov:\t" << t.min_cov << "\n"; + std::cout << "min size:\t" << t.min_size << "\n"; + std::cout << "partition files:\t" << t.num_partition_files << "\n"; + std::cout << "tech:\t" << t.tech << "\n"; } diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index 6dac71c..6ed35ae 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -2,6 +2,7 @@ #define OPTIONS_H #include "../common/defs.h" +#include #define INPUT_TYPE_CAN 0 #define INPUT_TYPE_M4 1 @@ -12,15 +13,20 @@ struct ConsensusOptions const char* m4; const char* reads; const char* corrected_reads; + const char* grid_options; + const char* grid_options_split; int num_threads; - index_t batch_size; + idx_t batch_size; double min_mapping_ratio; int min_align_size; int min_cov; - index_t min_size; + idx_t min_size; bool print_usage_info; int tech; - int num_partition_files; + int num_partition_files; + int job_index; + int reads_to_correct; + int grid_start_delay; }; void @@ -32,6 +38,9 @@ parse_arguments(int argc, char* argv[], ConsensusOptions& t); void print_options(ConsensusOptions& t); +std::string +make_options(const ConsensusOptions& t); + typedef ConsensusOptions ReadsCorrectionOptions; #endif // OPTIONS_H diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 9698265..bb359c1 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -5,49 +5,45 @@ #include #include -#include - #include "overlaps_store.h" #include "reads_correction_aux.h" -using namespace std; - #define error_and_exit(msg) { std::cerr << msg << "\n"; abort(); } inline bool check_m4record_mapping_range(const M4Record& m4, const double min_cov_ratio) { - const index_t qm = m4qend(m4) - m4qoff(m4); - const index_t qs = m4qsize(m4) * min_cov_ratio; - const index_t sm = m4send(m4) - m4soff(m4); - const index_t ss = m4ssize(m4) * min_cov_ratio; + const idx_t qm = m4qend(m4) - m4qoff(m4); + const idx_t qs = m4qsize(m4) * min_cov_ratio; + const idx_t sm = m4send(m4) - m4soff(m4); + const idx_t ss = m4ssize(m4) * min_cov_ratio; return qm >= qs || sm >= ss; } inline bool query_is_contained(const M4Record& m4, const double min_cov_ratio) { - const index_t qm = m4qend(m4) - m4qoff(m4); - const index_t qs = m4qsize(m4) * min_cov_ratio; + const idx_t qm = m4qend(m4) - m4qoff(m4); + const idx_t qs = m4qsize(m4) * min_cov_ratio; return qm >= qs; } inline bool subject_is_contained(const M4Record& m4, const double min_cov_ratio) { - const index_t sm = m4send(m4) - m4soff(m4); - const index_t ss = m4ssize(m4) * min_cov_ratio; + const idx_t sm = m4send(m4) - m4soff(m4); + const idx_t ss = m4ssize(m4) * min_cov_ratio; return sm >= ss; } void -get_qualified_m4record_counts(const char* m4_file_name, const double min_cov_ratio, index_t& num_qualified_records, index_t& num_reads) +get_qualified_m4record_counts(const char* m4_file_name, const double min_cov_ratio, idx_t& num_qualified_records, idx_t& num_reads) { std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); num_qualified_records = 0; num_reads = -1; - index_t num_records = 0; + idx_t num_records = 0; M4Record m4; m4qext(m4) = m4sext(m4) = INVALID_IDX; while (in >> m4) @@ -67,52 +63,48 @@ get_qualified_m4record_counts(const char* m4_file_name, const double min_cov_rat ++num_reads; } -void -get_repeat_reads(const char* m4_file_name, const double min_cov_ratio, const index_t num_reads, set& repeat_reads) -{ - const char MaxContained = 100; +void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const idx_t num_reads, std::set& repeat_reads) { + const int MaxContained = 100; + // used to imcrement to a max of MaxContained char cnt_table[MaxContained + 1]; - for (int i = 0; i < MaxContained; ++i) cnt_table[i] = i + 1; + for (int i = 0; i < MaxContained; ++i) { + cnt_table[i] = i + 1; + } cnt_table[MaxContained] = MaxContained; - vector cnts(num_reads, 0); + std::vector cnts(num_reads, 0); std::ifstream in; - open_fstream(in, m4_file_name, std::ios::in); + open_fstream(in, m4_file_name, std::ios::in); M4Record m4; m4qext(m4) = m4sext(m4) = INVALID_IDX; - while (in >> m4) - { - if (query_is_contained(m4, min_cov_ratio)) - { - index_t qid = m4qid(m4); - cnts[qid] = cnt_table[cnts[qid]]; + while (in >> m4) { + if (query_is_contained(m4, min_cov_ratio)) { + const idx_t qid = m4qid(m4); + // increments count + cnts[qid] = cnt_table[int(cnts[qid])]; } - if (subject_is_contained(m4, min_cov_ratio)) - { - index_t sid = m4sid(m4); - cnts[sid] = cnt_table[cnts[sid]]; + if (subject_is_contained(m4, min_cov_ratio)) { + const idx_t sid = m4sid(m4); + // increments count + cnts[sid] = cnt_table[int(cnts[sid])]; } - } - close_fstream(in); - - for(index_t i = 0; i < num_reads; ++i) - if (cnts[i] >= MaxContained) - { - cerr << "repeat read " << i << "\n"; + } + close_fstream(in); + for(idx_t i = 0; i < num_reads; ++i) { + if (cnts[i] >= MaxContained) { + std::cerr << "repeat read " << i << "\n"; repeat_reads.insert(i); } - + } LOG(stderr, "number of repeat reads: %d", (int)repeat_reads.size()); } -void -generate_partition_index_file_name(const char* m4_file_name, std::string& ret) -{ +void generate_partition_index_file_name(const char* m4_file_name, std::string& ret) { ret = m4_file_name; ret += ".partition_files"; } void -generate_partition_file_name(const char* m4_file_name, const index_t part, std::string& ret) +generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret) { ret = m4_file_name; ret += ".part"; @@ -124,8 +116,8 @@ generate_partition_file_name(const char* m4_file_name, const index_t part, std:: idx_t get_num_reads(const char* candidates_file) { - ifstream in; - open_fstream(in, candidates_file, ios::in); + std::ifstream in; + open_fstream(in, candidates_file, std::ios::in); ExtensionCandidate ec; int max_id = -1; while (in >> ec) @@ -137,156 +129,105 @@ get_num_reads(const char* candidates_file) return max_id + 1; } -void -normalise_candidate(ExtensionCandidate& src, ExtensionCandidate& dst, const bool subject_is_target) -{ - if (subject_is_target) - { +static void normalise_candidate(const ExtensionCandidate& src, ExtensionCandidate& dst, const bool subject_is_target) { + if (subject_is_target) { dst = src; - } - else - { + } else { dst.qdir = src.sdir; dst.qid = src.sid; dst.qext = src.sext; dst.qsize = src.ssize; + dst.qoff = src.soff; + dst.qend = src.send; dst.sdir = src.qdir; dst.sid = src.qid; dst.sext = src.qext; dst.ssize = src.qsize; + dst.soff = src.qoff; + dst.send = src.qend; dst.score = src.score; } - - if (dst.sdir == REV) - { + if (dst.sdir == REV) { dst.qdir = REVERSE_STRAND(dst.qdir); dst.sdir = REVERSE_STRAND(dst.sdir); } } -int -fix_file_counts(int num_files) { - if (num_files < 0) { - num_files = sysconf(_SC_OPEN_MAX) - 10; +void partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, const int num_files, idx_t num_reads) { + DynamicTimer dtimer(__func__); + PartitionResultsWriter prw(num_files); + idx_t i(0); + off_t input_pos; + int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); + if (!is_restart) { + prw.num_reads = num_reads ? num_reads : get_num_reads(input); } - return num_files; -} - -void -partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, int num_files) -{ - DynamicTimer dt(__func__); - - num_files = fix_file_counts(num_files); - const idx_t num_reads = get_num_reads(input); - const idx_t num_batches = (num_reads + batch_size - 1) / batch_size; - string idx_file_name; + const idx_t num_batches((prw.num_reads + batch_size - 1) / batch_size); + std::string idx_file_name; generate_partition_index_file_name(input, idx_file_name); - ofstream idx_file; - open_fstream(idx_file, idx_file_name.c_str(), ios::out); - + std::ofstream idx_file; + open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec, nec; - PartitionResultsWriter prw(num_files); - for (idx_t i = 0; i < num_batches; i += num_files) { - const idx_t sfid = i; - const idx_t efid = min(sfid + num_files, num_batches); - const int nf = efid - sfid; - const idx_t Lid = batch_size * sfid; - const idx_t Rid = batch_size * efid; - cout << "Lid = " << Lid - << ", Rid = " << Rid - << "\n"; - ifstream in; - open_fstream(in, input, ios::in); - prw.OpenFiles(sfid, efid, input, generate_partition_file_name); - + // and here we go through the input file num_batches times, + // being limited by the number of open output files we can have + for (; i < num_batches; i += prw.kNumFiles) { + const idx_t sfid(i); + const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + const int nf(efid - sfid); + const idx_t L(batch_size * sfid); + const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + std::ifstream in; + open_fstream(in, input, std::ios::in); + if (is_restart) { + if (!in.seekg(input_pos)) { + ERROR("Input seek failed while restoring checkpoint: %s", input); + } + is_restart = 0; + } else { + prw.OpenFiles(sfid, efid, input, generate_partition_file_name, "partition.done"); + } while (in >> ec) { - if (ec.qsize < min_read_size || ec.ssize < min_read_size) continue; - if (ec.qid >= Lid && ec.qid < Rid) { + if (ec.qsize < min_read_size || ec.ssize < min_read_size) { + continue; + } + // not set by >> + ec.qoff = ec.soff = ec.qend = ec.send = 0; + if (ec.qid >= L && ec.qid < R) { normalise_candidate(ec, nec, false); - prw.WriteOneResult((ec.qid - Lid) / batch_size, ec.qid, nec); + if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { + prw.checkpoint(in.tellg()); + } } - if (ec.sid >= Lid && ec.sid < Rid) - { + if (ec.sid >= L && ec.sid < R) { normalise_candidate(ec, nec, true); - prw.WriteOneResult((ec.sid - Lid) / batch_size, ec.sid, nec); + if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { + prw.checkpoint(in.tellg()); + } } } - for (int k = 0; k < nf; ++k) - { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; + for (int k(0); k < nf; ++k) { + if (prw.max_seq_ids[k] == std::numeric_limits::min()) { + continue; + } idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; - fprintf(stderr, "%s contains reads %d --- %d\n", prw.file_names[k].c_str(), (int)prw.min_seq_ids[k], (int)prw.max_seq_ids[k]); + fprintf(stderr, "%s contains reads %ld --- %ld\n", prw.file_names[k].c_str(), long(prw.min_seq_ids[k]), long(prw.max_seq_ids[k])); } prw.CloseFiles(); } close_fstream(idx_file); + prw.finalize(); } -/* void -partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, const int num_files) +partition_m4records(const char* m4_file_name, const double min_cov_ratio, const idx_t batch_size, const int min_read_size, const int num_files) { DynamicTimer dtimer(__func__); - idx_t num_reads = get_num_reads(input); - const index_t num_batches = (num_reads + batch_size - 1) / batch_size; - std::string idx_file_name; - generate_partition_index_file_name(input, idx_file_name); - std::ofstream idx_file; - open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - - ExtensionCandidate ec, nec; - PartitionResultsWriter prw; - for (index_t i = 0; i < num_batches; i += PartitionResultsWriter::kNumFiles) - { - const index_t sfid = i; - const index_t efid = std::min(sfid + PartitionResultsWriter::kNumFiles, num_batches); - const int nf = efid - sfid; - const index_t L = batch_size * sfid; - const index_t R = batch_size * efid; - std::ifstream in; - open_fstream(in, input, std::ios::in); - prw.OpenFiles(sfid, efid, input, generate_partition_file_name); - - while (in >> ec) - { - if (ec.qsize < min_read_size || ec.ssize < min_read_size) continue; - if (ec.qid >= L && ec.qid < R) - { - normalise_candidate(ec, nec, false); - prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec); - } - if (ec.sid >= L && ec.sid < R) - { - normalise_candidate(ec, nec, true); - prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec); - } - } - - for (int k = 0; k < nf; ++k) - { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; - idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; - fprintf(stderr, "%s contains reads %d --- %d\n", prw.file_names[k].c_str(), (int)prw.min_seq_ids[k], (int)prw.max_seq_ids[k]); - } - } - prw.CloseFiles(); - close_fstream(idx_file); -} -*/ - -/* -void -partition_m4records(const char* m4_file_name, const double min_cov_ratio, const index_t batch_size, const int min_read_size) -{ - DynamicTimer dtimer(__func__); - - index_t num_reads, num_qualified_records; + idx_t num_reads, num_qualified_records; get_qualified_m4record_counts(m4_file_name, min_cov_ratio, num_qualified_records, num_reads); - set repeat_reads; + std::set repeat_reads; //get_repeat_reads(m4_file_name, min_cov_ratio, num_reads, repeat_reads); - const index_t num_batches = (num_reads + batch_size - 1) / batch_size; + const idx_t num_batches = (num_reads + batch_size - 1) / batch_size; std::string idx_file_name; generate_partition_index_file_name(m4_file_name, idx_file_name); std::ofstream idx_file; @@ -294,17 +235,17 @@ partition_m4records(const char* m4_file_name, const double min_cov_ratio, const M4Record m4, nm4; ExtensionCandidate ec; - PartitionResultsWriter prw; - for (index_t i = 0; i < num_batches; i += PartitionResultsWriter::kNumFiles) + PartitionResultsWriter prw(num_files); + for (idx_t i = 0; i < num_batches; i += prw.kNumFiles) { - const index_t sfid = i; - const index_t efid = std::min(sfid + PartitionResultsWriter::kNumFiles, num_batches); + const idx_t sfid = i; + const idx_t efid = std::min(sfid + prw.kNumFiles, num_batches); const int nf = efid - sfid; - const index_t L = batch_size * sfid; - const index_t R = batch_size * efid; + const idx_t L = batch_size * sfid; + const idx_t R = batch_size * efid; std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); - prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name); + prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name, "partition.done"); while (in >> m4) { @@ -330,75 +271,7 @@ partition_m4records(const char* m4_file_name, const double min_cov_ratio, const for (int k = 0; k < nf; ++k) { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; - idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; - fprintf(stderr, "%s contains reads %d --- %d\n", prw.file_names[k].c_str(), (int)prw.min_seq_ids[k], (int)prw.max_seq_ids[k]); - } - - prw.CloseFiles(); - } - close_fstream(idx_file); -} -*/ - -void -partition_m4records(const char* m4_file_name, - const double min_cov_ratio, - const index_t batch_size, - const int min_read_size, - int num_files) -{ - DynamicTimer dtimer(__func__); - - num_files = fix_file_counts(num_files); - index_t num_reads, num_qualified_records; - get_qualified_m4record_counts(m4_file_name, min_cov_ratio, num_qualified_records, num_reads); - set repeat_reads; - const index_t num_batches = (num_reads + batch_size - 1) / batch_size; - std::string idx_file_name; - generate_partition_index_file_name(m4_file_name, idx_file_name); - std::ofstream idx_file; - open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - - M4Record m4, nm4; - ExtensionCandidate ec; - PartitionResultsWriter prw(num_files); - for (index_t i = 0; i < num_batches; i += num_files) - { - const index_t sfid = i; - const index_t efid = std::min(sfid + num_files, num_batches); - const int nf = efid - sfid; - const index_t L = batch_size * sfid; - const index_t R = batch_size * efid; - std::ifstream in; - open_fstream(in, m4_file_name, std::ios::in); - prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name); - - while (in >> m4) - { - if (m4qsize(m4) < min_read_size || m4ssize(m4) < min_read_size) continue; - if (!check_m4record_mapping_range(m4, min_cov_ratio)) continue; - if (repeat_reads.find(m4qid(m4)) != repeat_reads.end() - || - repeat_reads.find(m4sid(m4)) != repeat_reads.end()) continue; - - if (m4qid(m4) >= L && m4qid(m4) < R) - { - normalize_m4record(m4, false, nm4); - m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4qid(m4) - L) / batch_size, m4qid(m4), ec); - } - if (m4sid(m4) >= L && m4sid(m4) < R) - { - normalize_m4record(m4, true, nm4); - m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4sid(m4) - L) / batch_size, m4sid(m4), ec); - } - } - - for (int k = 0; k < nf; ++k) - { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; + if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; fprintf(stderr, "%s contains reads %d --- %d\n", prw.file_names[k].c_str(), (int)prw.min_seq_ids[k], (int)prw.max_seq_ids[k]); } diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 275a972..c74c936 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -9,26 +9,19 @@ void generate_partition_index_file_name(const char* m4_file_name, std::string& ret); void -generate_partition_file_name(const char* m4_file_name, const index_t part, std::string& ret); +generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret); void -partition_m4records(const char* m4_file_name, - const double min_cov_ratio, - const index_t batch_size, - const int min_read_size, - int num_files); +partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t batch_size, int min_read_size, int num_files); void -partition_candidates(const char* input, - const idx_t batch_size, - const int min_read_size, - int num_files); +partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); struct PartitionFileInfo { std::string file_name; - index_t min_seq_id; - index_t max_seq_id; + idx_t min_seq_id; + idx_t max_seq_id; }; void diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index a545982..30e9006 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -4,118 +4,220 @@ #include #include #include +#include +#include +#include #include "../common/defs.h" #include "../common/pod_darr.h" -template -class PartitionResultsWriter -{ -public: - typedef void (*file_name_generator)(const char* prefix, const idx_t id, std::string& name); - -public: - PartitionResultsWriter(const int max_num_files) - { - file_is_open = 0; - num_open_files = 0; - MaxNumFiles = max_num_files; - results = new PODArray[MaxNumFiles]; - files = new std::ofstream[MaxNumFiles]; - file_names = new std::string[MaxNumFiles]; - min_seq_ids = new idx_t[MaxNumFiles]; - max_seq_ids = new idx_t[MaxNumFiles]; - for (int i = 0; i < MaxNumFiles; ++i) results[i].reserve(kStoreSize); +template class PartitionResultsWriter { + public: + typedef void (*file_name_generator)(const char* prefix, idx_t id, std::string& name); + public: + const int kNumFiles; // effective open file limit + int kStoreSize; + int num_open_files; + int num_reads; + PODArray* results; // can't use vector<>, causes memory corruption + std::ofstream* files; // can't use vector<>, non-copyable + std::vector file_names; + std::vector min_seq_ids; + std::vector max_seq_ids; + public: + // can't make kNumFiles static, as sysconf() is run-time only; + // leave room for stdin, stdout, stderr, a few others + explicit PartitionResultsWriter(const int num_files) : kNumFiles(num_files > 0 ? num_files : sysconf(_SC_OPEN_MAX) - 10), kStoreSize(0), num_open_files(0), num_reads(0), results(0), files(0) { } + ~PartitionResultsWriter() { + CloseFiles(); + } + void OpenFiles(const idx_t sfid, const idx_t efid, const std::string& prefix, file_name_generator fng, const std::string& done_file) { + CloseFiles(); + if (efid <= sfid) { + return; + } + done_file_ = done_file; + ckpt_file_ = done_file_ + ".ckpt"; + ckpt_file_tmp_ = ckpt_file_ + ".tmp"; + num_open_files = efid - sfid; + batch_start_ = sfid; + allocate_data(prefix, fng, 0); } - - ~PartitionResultsWriter() - { + void CloseFiles() { + if (num_open_files == 0) { + return; + } + for (int i(0); i < num_open_files; ++i) { // flush buffers + if (results[i].size()) { + write_buffer_to_disk(i); + } + close_fstream(files[i]); + // don't attempt to re-finish finished files + if (!file_names[i].empty()) { + const std::string tmp_file(file_names[i] + ".tmp"); + if (rename(tmp_file.c_str(), file_names[i].c_str()) == -1) { + ERROR("Could not rename %s", tmp_file.c_str()); + } + } + } delete[] results; delete[] files; - delete[] file_names; - delete[] min_seq_ids; - delete[] max_seq_ids; + file_names.clear(); + min_seq_ids.clear(); + max_seq_ids.clear(); + kStoreSize = num_open_files = 0; + results = 0; + files = 0; } - - void OpenFiles(const idx_t sfid, const idx_t efid, const std::string& prefix, file_name_generator fng) - { - if (file_is_open) CloseFiles(); - - const int nf = efid - sfid; - if (nf == 0) return; - for (int i = 0; i < nf; ++i) - { - fng(prefix.data(), i + sfid, file_names[i]); - open_fstream(files[i], file_names[i].c_str(), std::ios::binary); - min_seq_ids[i] = std::numeric_limits::max(); - max_seq_ids[i] = std::numeric_limits::min(); + void finalize() { + // touch done file + std::ofstream done(done_file_.c_str()); + if (!done) { + ERROR("Could not create done file %s", done_file_.c_str()); + } + done.close(); + // remove checkpoint file + unlink(std::string(done_file_ + ".ckpt").c_str()); + } + int WriteOneResult(const int i, const idx_t seq_id, const T& r) { + min_seq_ids[i] = std::min(min_seq_ids[i], seq_id); + max_seq_ids[i] = std::max(max_seq_ids[i], seq_id); + results[i].push_back(r); + if (results[i].size() == kStoreSize) { + write_buffer_to_disk(i); results[i].clear(); - } - num_open_files = nf; - file_is_open = true; + if (is_checkpoint_time()) { + return 1; + } + } + return 0; + } + int restart(const std::string& prefix, file_name_generator fng, const std::string& done_file, idx_t& sfid, off_t& input_pos) { + CloseFiles(); + done_file_ = done_file; + ckpt_file_ = done_file_ + ".ckpt"; + ckpt_file_tmp_ = ckpt_file_ + ".tmp"; + std::ifstream in(ckpt_file_.c_str()); + if (!in) { + return 0; + } + in >> batch_start_ >> num_open_files >> num_reads >> input_pos; + if (!in) { + ERROR("Read error while restoring checkpoint from %s", ckpt_file_.c_str()); + } + allocate_data(prefix, fng, 1); + for (int i(0); i < num_open_files; ++i) { + off_t file_pos; + in >> file_pos >> min_seq_ids[i] >> max_seq_ids[i]; + if (!in) { + ERROR("Read error while restore checkpoint from %s (%d)", ckpt_file_.c_str(), i); + } + // don't need to restart finished files + if (!file_names[i].empty() && !files[i].seekp(file_pos)) { + ERROR("Seek failed while restoring checkpoint from %s (%d)", ckpt_file_.c_str(), i); + } + } + sfid = batch_start_; + return 1; } - - void CloseFiles() - { - if (!file_is_open) return; - for (int i = 0; i < num_open_files; ++i) - { - if (results[i].size()) - { - char* buf = (char*)results[i].data(); - std::streamsize s = sizeof(T) * results[i].size(); - files[i].write(buf, s); + void checkpoint(const off_t input_pos) { + std::ofstream out(ckpt_file_tmp_.c_str()); + if (!out) { + LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp_.c_str()); + return; + } + out << batch_start_ << " " << num_open_files << " " << num_reads << " " << input_pos << "\n"; + if (!out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); + return; + } + // flush buffers + for (int i(0); i < num_open_files; ++i) { + if (results[i].size()) { + write_buffer_to_disk(i); + results[i].clear(); + } + files[i].flush(); + out << off_t(files[i].tellp()) << " " << min_seq_ids[i] << " " << max_seq_ids[i] << "\n"; + if (!out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); + return; } } - for (int i = 0; i < num_open_files; ++i) close_fstream(files[i]); - file_is_open = false; - num_open_files = 0; + out.close(); + if (rename(ckpt_file_tmp_.c_str(), ckpt_file_.c_str()) == -1) { + LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file_.c_str()); + } } - - void WriteOneResult(const int fid, const idx_t seq_id, const T& r) - { - if (fid >= num_open_files) { - std::cout << "fid = " << fid - << ", num_open_files = " - << num_open_files - << "\n"; + private: + idx_t batch_start_ ; + time_t next_checkpoint_time_; + std::string done_file_; + std::string ckpt_file_; + std::string ckpt_file_tmp_; + private: + int is_checkpoint_time() { + const time_t current_time(time(0)); + if (current_time < next_checkpoint_time_ || current_time == static_cast(-1)) { + return 0; + } else { + next_checkpoint_time_ = current_time + 300; + return 1; } - r_assert(fid < num_open_files); - min_seq_ids[fid] = std::min(min_seq_ids[fid], seq_id); - max_seq_ids[fid] = std::max(max_seq_ids[fid], seq_id); - results[fid].push_back(r); - if (results[fid].size() == kStoreSize) - { - char* buf = (char*)results[fid].data(); - std::streamsize s = sizeof(T) * results[fid].size(); - files[fid].write(buf, s); - results[fid].clear(); + } + void allocate_data(const std::string& prefix, file_name_generator fng, const int is_restart) { + // allocate about a gb of memory as buffer, split among num_open_files + kStoreSize = (1 << 30) / sizeof(T) / num_open_files; + results = new PODArray[num_open_files]; + file_names.assign(num_open_files, ""); + min_seq_ids.assign(num_open_files, std::numeric_limits::max()); + max_seq_ids.assign(num_open_files, std::numeric_limits::min()); + files = new std::ofstream[num_open_files]; + for (int i(0); i < num_open_files; ++i) { + fng(prefix.c_str(), i + batch_start_, file_names[i]); + const std::string tmp_file(file_names[i] + ".tmp"); + if (is_restart) { + if (access(file_names[i].c_str(), F_OK) == 0) { + // already finished + file_names[i].clear(); + // use /dev/null to prevent write errors + open_fstream(files[i], "/dev/null", std::ios::binary); + } else { + // don't truncate on restart + open_fstream(files[i], tmp_file.c_str(), std::ios::binary | std::ios::in); + } + } else { + open_fstream(files[i], tmp_file.c_str(), std::ios::binary); + } + if (!files[i]) { + ERROR("Open failed on %s", tmp_file.c_str()); + } + results[i].reserve(kStoreSize); + } + next_checkpoint_time_ = time(0) + 300; + } + void write_buffer_to_disk(const int i) { + // can't use static_cast<> + const char* const buf((char*)results[i].data()); + const std::streamsize s(sizeof(T) * results[i].size()); + if (!files[i].write(buf, s)) { + ERROR("Error writing to %s", file_names[i].c_str()); } } - -public: - int MaxNumFiles; - static const int kStoreSize = 500000; - PODArray* results; - bool file_is_open; - int num_open_files; - std::ofstream* files; - std::string* file_names; - idx_t* min_seq_ids; - idx_t* max_seq_ids; }; -template -T* load_partition_data(const char* path, idx_t& num_results) -{ +template T* load_partition_data(const char* const path, idx_t& num_results) { std::ifstream in; open_fstream(in, path, std::ios::binary); in.seekg(0, std::ios::end); - std::streampos fs = in.tellg(); + const std::streampos fs(in.tellg()); in.seekg(0, std::ios::beg); num_results = fs / sizeof(T); - T* arr = new T[num_results]; - in.read((char*)arr, fs); + T* const arr(new T[num_results]); + in.read((char*)arr, fs); // can't use static_cast<> + if (!in) { + ERROR("Error reading partition data: %s", path); + } close_fstream(in); return arr; } diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 1a26897..939932c 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -1,6 +1,6 @@ #include "reads_correction_aux.h" -void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, std::string& qnorm, std::string& tnorm, const bool push) +void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, std::string& qnorm, std::string& tnorm, const bool push) { qnorm.clear(); tnorm.clear(); @@ -8,7 +8,7 @@ void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, #ifndef NDEBUG int qcnt = 0, tcnt = 0; - for (index_t i = 0; i < aln_size; ++i) + for (idx_t i = 0; i < aln_size; ++i) { const char qc = qstr[i]; const char tc = tstr[i]; @@ -18,7 +18,7 @@ void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, #endif // convert mismatches to indels - for (index_t i = 0; i < aln_size; ++i) + for (idx_t i = 0; i < aln_size; ++i) { const char qc = qstr[i]; const char tc = tstr[i]; @@ -31,14 +31,14 @@ void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, // push gaps to the right, but not pass the end if (push) { - index_t qlen = qnorm.size(); - index_t tlen = tnorm.size(); - for (index_t i = 0; i < qlen - 1; ++i) + idx_t qlen = qnorm.size(); + idx_t tlen = tnorm.size(); + for (idx_t i = 0; i < qlen - 1; ++i) { // push target gaps if (tnorm[i] == kGap) { - index_t j = i; + idx_t j = i; while (1) { const char c = tnorm[++j]; @@ -52,7 +52,7 @@ void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, // push query gaps if (qnorm[i] == kGap) { - index_t j = i; + idx_t j = i; while (1) { const char c = qnorm[++j]; @@ -78,38 +78,19 @@ void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, #endif } -struct CmpExtensionCandidateBySid -{ - bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) - { - return a.sid < b.sid; - } -}; - -void -build_cns_thrd_data_can(ExtensionCandidate* ec_list, - const int nec, - const idx_t min_rid, - const idx_t max_rid, - ReadsCorrectionOptions* prco, - PackedDB* reads, - std::ostream* out, - ConsensusThreadData** ppctd) -{ - const index_t num_reads = max_rid - min_rid + 1; - const int num_threads = prco->num_threads; - const index_t num_reads_per_thread = (num_reads + num_threads - 1) / num_threads; - std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySid()); - idx_t max_id = min_rid; - idx_t i = 0, j; - int tid = 0; - while (i < nec) - { - max_id += num_reads_per_thread; - j = i + 1; - while (j < nec && ec_list[j].sid < max_id) ++j; - ppctd[tid] = new ConsensusThreadData(prco, tid, reads, ec_list + i, j - i, out); - ++tid; - i = j; +void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, const idx_t nec) { + const int n(data.rco.num_threads); + // split by number of ec's, rather than reads, since reads ids are not + // necessarily contiguous and we could get empty lists + for (idx_t i(0), k(0); k < n; ++k) { + const idx_t start(i); + // drop fractions here, as we'll likely add a few more ec's below + i += (nec - i) / (n - k); + if (i < nec) { // include all ec's for the last read + const int final_sid(ec_list[i].sid); + for (++i; i < nec && ec_list[i].sid == final_sid; ++i) { } + } + data.data[k].num_candidates = i - start; + data.data[k].candidates = ec_list + start; } } diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 2dc25b3..1e9e396 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -3,6 +3,7 @@ #include #include +#include #include "dw.h" #include "../common/packed_db.h" @@ -68,110 +69,209 @@ struct CnsAln } }; -class CnsAlns -{ -public: - CnsAlns() - { +class CnsAlns { + public: + CnsAlns() : num_alns_(0) { safe_malloc(cns_alns_, CnsAln, MAX_CNS_OVLPS); - clear(); } - ~CnsAlns() - { + ~CnsAlns() { safe_free(cns_alns_); } - void clear() { num_alns_ = 0; } - int num_alns() { return num_alns_; } - CnsAln* begin() { return cns_alns_; } - CnsAln* end() { return cns_alns_ + num_alns_; } - void add_aln(const int soff, const int send, const std::string& qstr, const std::string& tstr) - { + void clear() { + num_alns_ = 0; + } + int num_alns() const { + return num_alns_; + } + CnsAln* begin() { + return cns_alns_; + } + CnsAln* end() { + return cns_alns_ + num_alns_; + } + void add_aln(const int soff, const int send, const std::string& qstr, const std::string& tstr) { r_assert(qstr.size() == tstr.size()); - CnsAln& a = cns_alns_[num_alns_++]; + CnsAln& a(cns_alns_[num_alns_++]); a.soff = soff; a.send = send; a.aln_idx = 0; a.aln_size = qstr.size(); - memcpy(a.qaln, qstr.data(), qstr.size()); - a.qaln[qstr.size()] = '\0'; - memcpy(a.saln, tstr.data(), tstr.size()); - a.saln[tstr.size()] = '\0'; + memcpy(a.qaln, qstr.c_str(), a.aln_size + 1); + memcpy(a.saln, tstr.c_str(), a.aln_size + 1); } - void get_mapping_ranges(std::vector& ranges) - { + void get_mapping_ranges(std::vector& ranges) const { ranges.clear(); - for (int i = 0; i < num_alns_; ++i) ranges.push_back(MappingRange(cns_alns_[i].soff, cns_alns_[i].send)); + ranges.reserve(num_alns_); + for (int i(0); i < num_alns_; ++i) { + ranges.push_back(MappingRange(cns_alns_[i].soff, cns_alns_[i].send)); + } } - -private: + private: + int num_alns_; CnsAln* cns_alns_; - int num_alns_; }; -#define MAX_CNS_RESULTS 10000 +// 1k seems to work a bit better than 10k - perhaps less time waiting for +// another thread to finish writing? +#define MAX_CNS_RESULTS 1000 -struct ConsensusThreadData -{ - ReadsCorrectionOptions rco; - int thread_id; - PackedDB* reads; +struct CmpExtensionCandidateBySidAndScore { + bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) { + if (a.sid != b.sid) { // primary sort + return a.sid < b.sid; // for splitting up in allocate_ecs() + } else if (a.score != b.score) { // secondary sort + return b.score < a.score; // process best ones first + } else if (a.qid != b.qid) { + return a.qid < b.qid; + } else if (a.qext != b.qext) { + return a.qext < b.qext; + } else if (a.sext != b.sext) { // tertiary sort + return a.sext < b.sext; // make sorting consistent + } else { + return a.qdir < b.qdir; + } + // sdir, qsize, ssize are all the same by this point + // qoff, soff, qend, send are all zero + } +}; + +class ConsensusPerThreadData { + public: + // num_candidates, candidates initialized by allocate_ecs() + // next_candidate initialized by ConsensusThreadData::restart() + idx_t num_candidates, next_candidate; ExtensionCandidate* candidates; - int num_candidates; ns_banded_sw::DiffRunningData* drd_s; ns_banded_sw::DiffRunningData* drd_l; - M5Record* m5; + CnsTableItem cns_table[MAX_SEQ_SIZE]; + uint1 id_list[MAX_SEQ_SIZE]; + M5Record m5; CnsAlns cns_alns; std::vector cns_results; std::vector query; std::vector target; std::string qaln; std::string saln; - CnsTableItem* cns_table; - uint1* id_list; - std::ostream* out; - pthread_mutex_t out_lock; - - ConsensusThreadData(ReadsCorrectionOptions* prco, int tid, PackedDB* r, ExtensionCandidate* ec, int nec, std::ostream* output) - { - rco = (*prco); - thread_id = tid; - reads = r; - candidates = ec; - num_candidates = nec; - drd_s = new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small()); - drd_l = new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_large()); - m5 = NewM5Record(MAX_SEQ_SIZE); - out = output; - + public: + ConsensusPerThreadData() : drd_s(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), drd_l(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_large())), m5(MAX_SEQ_SIZE) { + cns_results.reserve(MAX_CNS_RESULTS); query.reserve(MAX_SEQ_SIZE); target.reserve(MAX_SEQ_SIZE); qaln.reserve(MAX_SEQ_SIZE); saln.reserve(MAX_SEQ_SIZE); - safe_malloc(cns_table, CnsTableItem, MAX_SEQ_SIZE); - safe_malloc(id_list, uint1, MAX_SEQ_SIZE); - pthread_mutex_init(&out_lock, NULL); } - - ~ConsensusThreadData() - { + ~ConsensusPerThreadData() { delete drd_s; delete drd_l; - m5 = DeleteM5Record(m5); - safe_free(cns_table); - safe_free(id_list); } }; -void normalize_gaps(const char* qstr, const char* tstr, const index_t aln_size, std::string& qnorm, std::string& tnorm, const bool push); +class ConsensusThreadData { + public: + ReadsCorrectionOptions& rco; + PackedDB& reads; + std::ostream& out; + pthread_mutex_t out_lock; + // this doesn't work as a vector - all the pointers end up pointing + // to the same values, and eventually it seg faults (probably a + // compiler optimization bug) + ConsensusPerThreadData* data; + public: + ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), data(new ConsensusPerThreadData[prco.num_threads]), last_thread_id_(-1), num_threads_written_(0) { + done_file_ = input_file_name; + done_file_ += ".done"; + ckpt_file_ = input_file_name; + ckpt_file_ += ".ckpt"; + ckpt_file_tmp_ = ckpt_file_ + ".tmp"; + pthread_mutex_init(&out_lock, NULL); + pthread_mutex_init(&id_lock_, NULL); + } + ~ConsensusThreadData() { + delete[] data; + pthread_mutex_destroy(&out_lock); + pthread_mutex_destroy(&id_lock_); + } + int get_thread_id() { + pthread_mutex_lock(&id_lock_); + const int tid(++last_thread_id_); + pthread_mutex_unlock(&id_lock_); + return tid; + } + void write_buffer(const int tid, const idx_t i) { + ConsensusPerThreadData& pdata(data[tid]); + std::vector::const_iterator a(pdata.cns_results.begin()); + const std::vector::const_iterator end_a(pdata.cns_results.end()); + pthread_mutex_lock(&out_lock); + for (; a != end_a; ++a) { + out << ">" << a->id << "_" << a->range[0] << "_" << a->range[1] << "_" << a->seq.size() << "\n" << a->seq << "\n"; + if (!out) { + ERROR("Error writing output"); + } + } + pdata.next_candidate = i; + if (++num_threads_written_ <= rco.num_threads) { + checkpoint(); + num_threads_written_ = 0; + } + pthread_mutex_unlock(&out_lock); + pdata.cns_results.clear(); + } + int restart(off_t& output_pos) { + if (access(ckpt_file_.c_str(), F_OK) != 0) { + for (int i(0); i < rco.num_threads; ++i) { + data[i].next_candidate = 0; + } + return 0; + } + std::ifstream ckpt_in(ckpt_file_.c_str()); + if (!ckpt_in) { + ERROR("Restart failed: could not open checkpoint file: %s", ckpt_file_.c_str()); + } + ckpt_in >> rco.job_index >> output_pos; + if (!ckpt_in) { + ERROR("Restart failed: could not read checkpoint file: %s", ckpt_file_.c_str()); + } + for (int i(0); i < rco.num_threads; ++i) { + ckpt_in >> data[i].next_candidate; + if (!ckpt_in) { + ERROR("Restart failed: could not read checkpoint file: %s", ckpt_file_.c_str()); + } + } + return 1; + } + private: + void checkpoint() { + std::ofstream ckpt_out(ckpt_file_tmp_.c_str()); + if (!ckpt_out) { + LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp_.c_str()); + return; + } + out.flush(); + ckpt_out << rco.job_index << " " << off_t(out.tellp()) << "\n"; + if (!ckpt_out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); + return; + } + for (int i(0); i < rco.num_threads; ++i) { + ckpt_out << data[i].next_candidate << "\n"; + if (!ckpt_out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); + return; + } + } + ckpt_out.close(); + if (rename(ckpt_file_tmp_.c_str(), ckpt_file_.c_str()) == -1) { + LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file_.c_str()); + } + } + private: + int last_thread_id_, num_threads_written_; + pthread_mutex_t id_lock_; + std::string done_file_, ckpt_file_, ckpt_file_tmp_; +}; + +void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, std::string& qnorm, std::string& tnorm, bool push); -void -build_cns_thrd_data_can(ExtensionCandidate* ec_list, - const int nec, - const idx_t min_rid, - const idx_t max_rid, - ReadsCorrectionOptions* prco, - PackedDB* reads, - std::ostream* out, - ConsensusThreadData** ppctd); +void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, idx_t nec); #endif // _READS_CORRECTION_AUX_H diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index c605478..f0df158 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -1,108 +1,120 @@ #include "reads_correction_can.h" #include +#include +#include #include "MECAT_AlnGraphBoost.H" #include "mecat_correction.h" #include "overlaps_partition.h" #include "overlaps_store.h" -using namespace std; - -struct CmpExtensionCandidateBySid -{ - bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) - { - return a.sid < b.sid; - } -}; - -void* -reads_correction_func_can(void* arg) -{ - ConsensusThreadData& cns_data = *static_cast(arg); - ExtensionCandidate* candidates = cns_data.candidates; - const index_t num_candidates = cns_data.num_candidates; - index_t i = 0, j; - while (i < num_candidates) - { - const index_t sid = candidates[i].sid; - j = i + 1; - while (j < num_candidates && candidates[j].sid == sid) ++j; - if (j - i < cns_data.rco.min_cov) { i = j; continue; } - if (candidates[i].ssize < cns_data.rco.min_size * 0.95) { i = j; continue; } - if (cns_data.rco.tech == TECH_PACBIO) { - ns_meap_cns::consensus_one_read_can_pacbio(&cns_data, sid, i, j); +static void* reads_correction_func_can(void* const arg) { + ConsensusThreadData& data(*(static_cast(arg))); + const int tid(data.get_thread_id()); + ConsensusPerThreadData& pdata(data.data[tid]); + const ExtensionCandidate* const candidates(pdata.candidates); + const idx_t min_size(data.rco.min_size * 0.95); + const int tech_is_pacbio(data.rco.tech == TECH_PACBIO ? 1 : 0); + idx_t i(pdata.next_candidate); + while (i < pdata.num_candidates) { + const idx_t start(i); + const idx_t sid(candidates[start].sid); + for (++i; i < pdata.num_candidates && candidates[i].sid == sid; ++i) { } + if (i < start + data.rco.min_cov || candidates[start].ssize < min_size) { + continue; + } + if (tech_is_pacbio) { + ns_meap_cns::consensus_one_read_can_pacbio(data, pdata, sid, start, i); } else { - ns_meap_cns::consensus_one_read_can_nanopore(&cns_data, sid, i, j); + ns_meap_cns::consensus_one_read_can_nanopore(data, pdata, sid, start, i); } - if (cns_data.cns_results.size() >= MAX_CNS_RESULTS) - { - pthread_mutex_lock(&cns_data.out_lock); - for (std::vector::iterator iter = cns_data.cns_results.begin(); iter != cns_data.cns_results.end(); ++iter) - { - (*cns_data.out) << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n"; - std::string& seq = iter->seq; - (*cns_data.out) << seq << "\n"; - } - cns_data.cns_results.clear(); - pthread_mutex_unlock(&cns_data.out_lock); + if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { + data.write_buffer(tid, i); } - i = j; - } - return NULL; + } + data.write_buffer(tid, i); + return NULL; } -void -consensus_one_partition_can(const char* m4_file_name, - const index_t min_read_id, - const index_t max_read_id, - ReadsCorrectionOptions& rco, - PackedDB& reads, - std::ostream& out) -{ - idx_t num_ec; - ExtensionCandidate* ec_list = load_partition_data(m4_file_name, num_ec); - ConsensusThreadData* pctds[rco.num_threads]; - build_cns_thrd_data_can(ec_list, num_ec, min_read_id, max_read_id, &rco, &reads, &out, pctds); - pthread_t thread_ids[rco.num_threads]; - for (int i = 0; i < rco.num_threads; ++i) - pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(pctds[i])); - for (int i = 0; i < rco.num_threads; ++i) - pthread_join(thread_ids[i], NULL); - for (int i = 0; i < rco.num_threads; ++i) - { - std::vector& cns_results = pctds[i]->cns_results; - for (std::vector::iterator iter = cns_results.begin(); iter != cns_results.end(); ++iter) - { - out << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n"; - std::string& seq = iter->seq; - out << seq << "\n"; - } +static void consensus_one_partition_can(const char* const m4_file_name, const idx_t min_read_id, const idx_t max_read_id, ConsensusThreadData& data) { + idx_t nec; + ExtensionCandidate* const ec_list(load_partition_data(m4_file_name, nec)); + std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); + allocate_ecs(data, ec_list, nec); + pthread_t thread_ids[data.rco.num_threads]; + for (int i(0); i < data.rco.num_threads; ++i) { + pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(&data)); + } + for (int i(0); i < data.rco.num_threads; ++i) { + pthread_join(thread_ids[i], NULL); } + delete[] ec_list; +} - delete[] ec_list; - for (int i = 0; i < rco.num_threads; ++i) delete pctds[i]; +static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB& reads) { + const PartitionFileInfo& p(partition_file_vec[rco.job_index]); + std::ostringstream os; + os << rco.corrected_reads << "." << rco.job_index; + const std::string results_file(os.str()); + const std::string results_file_tmp(results_file + ".tmp"); + std::ofstream out; + ConsensusThreadData data(rco, reads, out, results_file.c_str()); + off_t output_pos; + const int is_restart(data.restart(output_pos)); + if (is_restart) { + open_fstream(out, results_file_tmp.c_str(), std::ios::out | std::ios::in); + if (!out.seekp(output_pos)) { + ERROR("Restart failed: output file seek failed: %s", results_file_tmp.c_str()); + } + } else { + open_fstream(out, results_file_tmp.c_str(), std::ios::out); + } + char process_info[1024]; + sprintf(process_info, "processing %s", p.file_name.c_str()); + DynamicTimer dtimer(process_info); + consensus_one_partition_can(p.file_name.c_str(), p.min_seq_id, p.max_seq_id, data); + close_fstream(out); + assert(rename(results_file_tmp.c_str(), results_file.c_str()) == 0); + return 0; } -int reads_correction_can(ReadsCorrectionOptions& rco) -{ - partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files); +int reads_correction_can(ReadsCorrectionOptions& rco) { std::string idx_file_name; generate_partition_index_file_name(rco.m4, idx_file_name); std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; reads.load_fasta_db(rco.reads); - std::ofstream out; - open_fstream(out, rco.corrected_reads, std::ios::out); - char process_info[1024]; - for (std::vector::iterator iter = partition_file_vec.begin(); iter != partition_file_vec.end(); ++iter) - { - sprintf(process_info, "processing %s", iter->file_name.c_str()); - DynamicTimer dtimer(process_info); - consensus_one_partition_can(iter->file_name.c_str(), iter->min_seq_id, iter->max_seq_id, rco, reads, out); + if (rco.job_index != -1) { + return reads_correction_can_p(rco, partition_file_vec, reads); + } else { + rco.job_index = 0; // may get changed by data.restart() below + std::string tmp_file(rco.corrected_reads); + tmp_file += ".tmp"; + std::ofstream out; + ConsensusThreadData data(rco, reads, out, rco.corrected_reads); + off_t output_pos; + const int is_restart(data.restart(output_pos)); + if (is_restart) { + open_fstream(out, rco.corrected_reads, std::ios::out | std::ios::in); + out.seekp(output_pos); + if (!out.seekp(output_pos)) { + ERROR("Restart failed: output file seek failed: %s", rco.corrected_reads); + } + } else { + open_fstream(out, rco.corrected_reads, std::ios::out); + } + char process_info[1024]; + const int job_end(partition_file_vec.size()); + for (; rco.job_index < job_end; ++rco.job_index) { + const PartitionFileInfo& p(partition_file_vec[rco.job_index]); + sprintf(process_info, "processing %s", p.file_name.c_str()); + DynamicTimer dtimer(process_info); + consensus_one_partition_can(p.file_name.c_str(), p.min_seq_id, p.max_seq_id, data); + } + close_fstream(out); + assert(rename(tmp_file.c_str(), rco.corrected_reads) == 0); } - return 0; } diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index be7a5d7..2dc4a03 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -1,98 +1,115 @@ #include "reads_correction_m4.h" #include +#include +#include #include "mecat_correction.h" #include "overlaps_partition.h" #include "overlaps_store.h" -void* -reads_correction_func_m4(void* arg) -{ - ConsensusThreadData& cns_data = *static_cast(arg); - ExtensionCandidate* overlaps = cns_data.candidates; - const index_t num_ovlps = cns_data.num_candidates; - index_t i = 0, j; - while (i < num_ovlps) - { - const index_t sid = overlaps[i].sid; - j = i + 1; - while (j < num_ovlps && overlaps[j].sid == sid) ++j; - if (j - i < cns_data.rco.min_cov) { i = j; continue; } - if (overlaps[i].ssize < cns_data.rco.min_size * 0.95) { i = j; continue; } - if (cns_data.rco.tech == TECH_PACBIO) { - ns_meap_cns::consensus_one_read_m4_pacbio(&cns_data, sid, i, j); +void* reads_correction_func_m4(void* arg) { + ConsensusThreadData& data(*(static_cast(arg))); + const int tid(data.get_thread_id()); + ConsensusPerThreadData& pdata(data.data[tid]); + ExtensionCandidate* const overlaps(pdata.candidates); + const idx_t num_ovlps(pdata.num_candidates); + idx_t i = 0, j; + while (i < num_ovlps) { + const idx_t sid = overlaps[i].sid; + j = i + 1; + while (j < num_ovlps && overlaps[j].sid == sid) { + ++j; + } + if (j - i < data.rco.min_cov) { + i = j; + continue; + } + if (overlaps[i].ssize < data.rco.min_size * 0.95) { + i = j; + continue; + } + if (data.rco.tech == TECH_PACBIO) { + ns_meap_cns::consensus_one_read_m4_pacbio(data, pdata, sid, i, j); } else { - ns_meap_cns::consensus_one_read_m4_nanopore(&cns_data, sid, i, j); + ns_meap_cns::consensus_one_read_m4_nanopore(data, pdata, sid, i, j); } - if (cns_data.cns_results.size() >= MAX_CNS_RESULTS) - { - pthread_mutex_lock(&cns_data.out_lock); - for (std::vector::iterator iter = cns_data.cns_results.begin(); iter != cns_data.cns_results.end(); ++iter) - { - (*cns_data.out) << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n"; - std::string& seq = iter->seq; - (*cns_data.out) << seq << "\n"; + if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { + pthread_mutex_lock(&data.out_lock); + for (std::vector::iterator iter = pdata.cns_results.begin(); iter != pdata.cns_results.end(); ++iter) { + data.out << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n" << iter->seq << "\n"; + if (!data.out) { + ERROR("Error writing output"); + } } - cns_data.cns_results.clear(); - pthread_mutex_unlock(&cns_data.out_lock); + pthread_mutex_unlock(&data.out_lock); + pdata.cns_results.clear(); } - i = j; - } - return NULL; + i = j; + } + return NULL; } -void -consensus_one_partition_m4(const char* m4_file_name, - const index_t min_read_id, - const index_t max_read_id, - ReadsCorrectionOptions& rco, - PackedDB& reads, - std::ostream& out) -{ - idx_t num_ec; - ExtensionCandidate* ec_list = load_partition_data(m4_file_name, num_ec); - ConsensusThreadData* pctds[rco.num_threads]; - build_cns_thrd_data_can(ec_list, num_ec, min_read_id, max_read_id, &rco, &reads, &out, pctds); - pthread_t thread_ids[rco.num_threads]; - for (int i = 0; i < rco.num_threads; ++i) - pthread_create(&thread_ids[i], NULL, reads_correction_func_m4, static_cast(pctds[i])); - for (int i = 0; i < rco.num_threads; ++i) - pthread_join(thread_ids[i], NULL); - for (int i = 0; i < rco.num_threads; ++i) - { - std::vector& cns_results = pctds[i]->cns_results; - for (std::vector::iterator iter = cns_results.begin(); iter != cns_results.end(); ++iter) - { +void consensus_one_partition_m4(const char* m4_file_name, const idx_t min_read_id, const idx_t max_read_id, ReadsCorrectionOptions& rco, PackedDB& reads, std::ostream& out) { + idx_t nec; + ExtensionCandidate* ec_list = load_partition_data(m4_file_name, nec); + std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); + ConsensusThreadData data(rco, reads, out, m4_file_name); + allocate_ecs(data, ec_list, nec); + pthread_t thread_ids[rco.num_threads]; + for (int i = 0; i < rco.num_threads; ++i) { + pthread_create(&thread_ids[i], NULL, reads_correction_func_m4, static_cast(&data)); + } + for (int i = 0; i < rco.num_threads; ++i) { + pthread_join(thread_ids[i], NULL); + } + for (int i = 0; i < rco.num_threads; ++i) { + std::vector& cns_results = data.data[i].cns_results; + for (std::vector::iterator iter = cns_results.begin(); iter != cns_results.end(); ++iter) { out << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n"; std::string& seq = iter->seq; out << seq << "\n"; } } + delete[] ec_list; +} - delete[] ec_list; - for (int i = 0; i < rco.num_threads; ++i) delete pctds[i]; +static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB &reads) { + std::ostringstream os; + os << rco.corrected_reads << "." << rco.job_index; + std::string results_file = os.str(); + std::string working_file = results_file + ".working"; + std::ofstream out; + open_fstream(out, working_file.c_str(), std::ios::out); + const PartitionFileInfo &p = partition_file_vec[rco.job_index]; + char process_info[1024]; + sprintf(process_info, "processing %s", p.file_name.c_str()); + DynamicTimer dtimer(process_info); + consensus_one_partition_m4(p.file_name.c_str(), p.min_seq_id, p.max_seq_id, rco, reads, out); + assert(rename(working_file.c_str(), results_file.c_str()) == 0); + return 0; } -int reads_correction_m4(ReadsCorrectionOptions& rco) -{ - double mapping_ratio = rco.min_mapping_ratio - 0.02; - partition_m4records(rco.m4, mapping_ratio, rco.batch_size, rco.min_size, rco.num_partition_files); +int reads_correction_m4(ReadsCorrectionOptions& rco) { std::string idx_file_name; generate_partition_index_file_name(rco.m4, idx_file_name); std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; reads.load_fasta_db(rco.reads); - std::ofstream out; - open_fstream(out, rco.corrected_reads, std::ios::out); - char process_info[1024]; - for (std::vector::iterator iter = partition_file_vec.begin(); iter != partition_file_vec.end(); ++iter) - { - sprintf(process_info, "processing %s", iter->file_name.c_str()); - DynamicTimer dtimer(process_info); - consensus_one_partition_m4(iter->file_name.c_str(), iter->min_seq_id, iter->max_seq_id, rco, reads, out); + if (rco.job_index != -1) { + return reads_correction_m4_p(rco, partition_file_vec, reads); + } else { + std::ofstream out; + open_fstream(out, rco.corrected_reads, std::ios::out); + char process_info[1024]; + for (std::vector::iterator iter = partition_file_vec.begin(); iter != partition_file_vec.end(); ++iter) + { + sprintf(process_info, "processing %s", iter->file_name.c_str()); + DynamicTimer dtimer(process_info); + consensus_one_partition_m4(iter->file_name.c_str(), iter->min_seq_id, iter->max_seq_id, rco, reads, out); + } + + return 0; } - - return 0; } diff --git a/src/mecat2pw/pw.cpp b/src/mecat2pw/pw.cpp index f3b0763..95e5b6f 100644 --- a/src/mecat2pw/pw.cpp +++ b/src/mecat2pw/pw.cpp @@ -4,82 +4,159 @@ #include #include -#include +#include // ... unlink() +#include // PATH_MAX +#include // S_IRUSR, S_IXUSR +#include // chmod() #include #include +#include -using namespace std; - -void -create_volume_results_name_working(int vid, const char* wrk_dir, string& name) -{ +static void create_volume_results_name(const int vid, const char* const wrk_dir, std::string& name) { name = wrk_dir; - if (name[name.size() - 1] != '/') name += '/'; - ostringstream os; - os << "r_" << vid << ".working"; - name += os.str(); + if (name[name.size() - 1] != '/') { + name += '/'; + } + name += "r_"; + std::ostringstream x; + x << vid; + name += x.str(); } -void -create_volume_results_name_finished(int vid, const char* wrk_dir, string& name) -{ - name = wrk_dir; - if (name[name.size() - 1] != '/') name += '/'; - ostringstream os; - os << "r_" << vid; - name += os.str(); +static void merge_results(const char* const output, const std::list& files) { + std::string out_tmp(output); + out_tmp += ".tmp"; + std::ofstream out(out_tmp.c_str()); + std::list::const_iterator a(files.begin()); + const std::list::const_iterator end_a(files.end()); + for (; a != end_a; ++a) { + std::ifstream in(a->c_str()); + out << in.rdbuf(); + if (!in) { + std::cerr << "Error reading from " << *a << "\n"; + exit(1); + } else if (!out) { + std::cerr << "Error writing to " << out_tmp << "\n"; + exit(1); + } + } + out.close(); + if (rename(out_tmp.c_str(), output) == -1) { + std::cerr << "Could not rename concatenated output file: " << out_tmp << "\n"; + exit(1); + } +} + +static void grid_start(std::string prog, const options_t &options, const int i, const int num_vols) { + // create grid script, have grid run it + options_t new_options = options; + new_options.job_index = i; + new_options.grid_options = NULL; + new_options.grid_options_split = NULL; + new_options.num_vols = num_vols; + std::string name("m2pw."); + if (i == -1) { + name += "split"; + } else { + std::ostringstream x; + x << i; + name += x.str(); + } + std::string script_file(options.wrk_dir); + script_file += "/" + name + ".sh"; + std::ofstream out; + // as we might not have write permission, delete it + unlink(script_file.c_str()); + open_fstream(out, script_file.c_str(), std::ios::out); + out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(&new_options) << "\n"; + if (!out) { + std::cerr << "Error writing to " << script_file << "\n"; + exit(1); + } + close_fstream(out); + chmod(script_file.c_str(), S_IRUSR | S_IXUSR); + std::string cmd(i == -1 && options.grid_options_split ? options.grid_options_split : options.grid_options); + cmd += " " + name + " " + script_file; + assert(system(cmd.c_str()) == 0); } -void -merge_results(const char* output, const char* wrk_dir, const int num_volumes) -{ - string vrn; - for (int i = 0; i < num_volumes; ++i) - { - create_volume_results_name_finished(i, wrk_dir, vrn); - ostringstream cmd; - if (i == 0) cmd << "cat " << vrn << " >" << output; - else cmd << "cat " << vrn << " >> " << output; - assert(system(cmd.str().c_str()) == 0); +// pass by value so we can modify list to easily avoid checking previously +// found results files +static void wait_for_files(std::list results) { + const std::list::const_iterator end_a(results.end()); + while (!results.empty()) { + sleep(60); + std::list::iterator a(results.begin()); + while (a != end_a) { + if (access(a->c_str(), F_OK) == 0) { + a = results.erase(a); + } else { + ++a; + } + } } } -int main(int argc, char* argv[]) -{ - options_t options; - int r = parse_arguments(argc, argv, &options); - if (r) - { +int main(int argc, char* argv[]) { + options_t options; + if (parse_arguments(argc, argv, &options)) { print_usage(argv[0]); return 1; } - - int num_vols = split_raw_dataset(options.reads, options.wrk_dir); - - char vol_idx_file_name[1024]; + char vol_idx_file_name[PATH_MAX]; generate_idx_file_name(options.wrk_dir, vol_idx_file_name); - cout << vol_idx_file_name << "\n"; - volume_names_t* vn = load_volume_names(vol_idx_file_name, 0); + volume_names_t* vn; + if (options.num_vols == 0) { // just do the split + split_raw_dataset(options.reads, options.wrk_dir); + return 0; + } else if (options.grid_options == NULL && options.grid_options_split == NULL) { + } else if (options.num_vols == -1) { // spin off split and wait + // check to see if split is already done + vn = load_volume_names(vol_idx_file_name, 0); + if (vn->num_vols == 0) { + grid_start(argv[0], options, -1, 0); + std::list split_results; + split_results.push_back(vol_idx_file_name); + wait_for_files(split_results); + } + delete_volume_names_t(vn); + } + int num_vols = options.num_vols != -1 ? options.num_vols : split_raw_dataset(options.reads, options.wrk_dir); + vn = load_volume_names(vol_idx_file_name, num_vols); r_assert(num_vols == vn->num_vols); - for (int i = 0; i < vn->num_vols; ++i) - { - string volume_results_name_finished; - create_volume_results_name_finished(i, options.wrk_dir, volume_results_name_finished); - if (access(volume_results_name_finished.c_str(), F_OK) == 0) - { - LOG(stderr, "volume %d has been finished\n", i); - continue; + if (options.job_index != -1) { + std::string volume_results_name; + create_volume_results_name(options.job_index, options.wrk_dir, volume_results_name); + process_one_volume(&options, options.job_index, vn->num_vols, volume_results_name, vn); + return 0; + } + std::cout << vol_idx_file_name << "\n"; + std::list results; + for (int i(0); i < vn->num_vols; ++i) { + if (options.reads_to_correct) { + const char *volume_name(get_vol_name(vn, i)); + volume_t *v(load_volume_header(volume_name)); + if (v->start_read_id > options.reads_to_correct) { + break; + } + delete_volume_t(v); } - string volume_results_name_working; - create_volume_results_name_working(i, options.wrk_dir, volume_results_name_working); - ofstream out; - open_fstream(out, volume_results_name_working.c_str(), ios::out); - process_one_volume(&options, i, vn->num_vols, vn, &out); - close_fstream(out); - assert(rename(volume_results_name_working.c_str(), volume_results_name_finished.c_str()) == 0); + std::string volume_results_name; + create_volume_results_name(i, options.wrk_dir, volume_results_name); + results.push_back(volume_results_name); + if (access(volume_results_name.c_str(), F_OK) == 0) { + LOG(stderr, "volume %d has been finished", i); + } else if (options.grid_options == NULL) { + process_one_volume(&options, i, vn->num_vols, volume_results_name, vn); + } else { + grid_start(argv[0], options, i, vn->num_vols); + } + } + delete_volume_names_t(vn); + if (options.grid_options != NULL) { + wait_for_files(results); } - vn = delete_volume_names_t(vn); - - merge_results(options.output, options.wrk_dir, num_vols); + merge_results(options.output, results); + return 0; } diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index 57a6606..f5d8ce0 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -1,3 +1,6 @@ +// make off_t 64 bit (from ftello man page) +#define _FILE_OFFSET_BITS 64 + #include "../common/split_database.h" #include "pw_options.h" #include "../common/diff_gapalign.h" @@ -7,15 +10,24 @@ #include "pw_impl.h" #include - #include #include #include #include +#include // unlink() -#define MSS MAX_SEQ_SIZE +#define RM 100000 +#define DN 500 +#define BC 10 +#define SM 40 +#define SI 41 +#define CHUNK_SIZE 500 +#define ZV 2000 +#define MUL_ZV(a) ((a)*ZV) +#define DIV_ZV(a) ((a)/ZV) +#define MOD_ZV(a) ((a)%ZV) -static int MAXC = 100; +static int maxc = 100; static int output_gapped_start_point = 1; static int kmer_size = 13; static const double ddfs_cutoff_pacbio = 0.25; @@ -25,46 +37,99 @@ static int min_align_size = 0; static int min_kmer_match = 0; static int min_kmer_dist = 0; -using namespace std; +struct candidate_save { + char chain; + int loc1, loc2, left1, left2, right1, right2, score, num1, num2, readno, readstart; +}; -PWThreadData::PWThreadData(options_t* opt, volume_t* ref, volume_t* rd, ref_index* idx, std::ostream* o) - : options(opt), used_thread_id(0), reference(ref), reads(rd), ridx(idx), out(o), m4_results(NULL), ec_results(NULL), next_processed_id(0) -{ - pthread_mutex_init(&id_lock, NULL); - if (options->task == TASK_SEED) - { - safe_malloc(ec_results, ExtensionCandidate*, options->num_threads); - for (int i = 0; i < options->num_threads; ++i) - safe_malloc(ec_results[i], ExtensionCandidate, kResultListSize); - } - else if (options->task == TASK_ALN) - { - safe_malloc(m4_results, M4Record*, options->num_threads); - for (int i = 0; i < options->num_threads; ++i) - safe_malloc(m4_results[i], M4Record, kResultListSize); +struct Back_List { + int index; + short score, seednum, loczhi[SM], seedno[SM]; +}; + +class SeedingBK { + public: + int* index_list; + short* index_score; + Back_List* database; + int* kmer_ids; + public: + SeedingBK(const int ref_size) { + const int num_segs(ref_size / ZV + 5); + safe_malloc(index_list, int, num_segs); + safe_malloc(index_score, short, num_segs); + safe_malloc(database, Back_List, num_segs); + safe_malloc(kmer_ids, int, MAX_SEQ_SIZE); + for (int i(0); i < num_segs; ++i) { + database[i].score = 0; + database[i].index = -1; + } } - else - { - LOG(stderr, "Task must be either %d or %d, not %d!", TASK_SEED, TASK_ALN, options->task); - abort(); + ~SeedingBK() { + safe_free(index_list); + safe_free(index_score); + safe_free(database); + safe_free(kmer_ids); } - pthread_mutex_init(&result_write_lock, NULL); - pthread_mutex_init(&read_retrieve_lock, NULL); -} +}; -PWThreadData::~PWThreadData() -{ - if (ec_results) - { - for (int i = 0; i < options->num_threads; ++i) safe_free(ec_results[i]); - safe_free(ec_results); +class PWThreadData { + public: + int used_thread_id, next_processed_id; + const options_t* const options; + const volume_t* const reference; + const ref_index* const ridx; + volume_t* reads; + std::ostream* const out; + M4Record** m4_results; + ExtensionCandidate** ec_results; + pthread_mutex_t id_lock, result_write_lock, read_retrieve_lock; + static const int kResultListSize = 10000; + public: + PWThreadData(const options_t* opt, const volume_t* ref, const ref_index* idx, std::ostream* o) : options(opt), reference(ref), ridx(idx), reads(0), out(o) { + if (options->task == TASK_SEED) { + safe_malloc(ec_results, ExtensionCandidate*, options->num_threads); + for (int i(0); i < options->num_threads; ++i) { + safe_malloc(ec_results[i], ExtensionCandidate, kResultListSize); + } + } else { + safe_malloc(m4_results, M4Record*, options->num_threads); + for (int i(0); i < options->num_threads; ++i) { + safe_malloc(m4_results[i], M4Record, kResultListSize); + } + } + pthread_mutex_init(&id_lock, NULL); + pthread_mutex_init(&result_write_lock, NULL); + pthread_mutex_init(&read_retrieve_lock, NULL); } - if (m4_results) - { - for (int i = 0; i < options->num_threads; ++i) safe_free(m4_results[i]); - safe_free(m4_results); + ~PWThreadData() { + finish(); + if (options->task == TASK_SEED) { + for (int i(0); i < options->num_threads; ++i) { + safe_free(ec_results[i]); + } + safe_free(ec_results); + } else { + for (int i(0); i < options->num_threads; ++i) { + safe_free(m4_results[i]); + } + safe_free(m4_results); + } + pthread_mutex_destroy(&id_lock); + pthread_mutex_destroy(&result_write_lock); + pthread_mutex_destroy(&read_retrieve_lock); } -} + void start(const char* const read_name) { + used_thread_id = next_processed_id = 0; + reads = load_volume(read_name); + } + void finish() { + if (reads) { + delete_volume_t(reads); + reads = 0; + } + } +}; void reverse_complement(char* dst, const char* src, const int size) @@ -96,28 +161,6 @@ extract_kmers(const char* s, const int ssize, int* kmer_ids) return num_kmers; } -SeedingBK::SeedingBK(const int ref_size) -{ - const int num_segs = ref_size / ZV + 5; - safe_malloc(index_list, int, num_segs); - safe_malloc(index_score, short, num_segs); - safe_malloc(database, Back_List, num_segs); - safe_malloc(kmer_ids, int, MAX_SEQ_SIZE); - for (int i = 0; i < num_segs; ++i) - { - database[i].score = 0; - database[i].index = -1; - } -} - -SeedingBK::~SeedingBK() -{ - safe_free(index_list); - safe_free(index_score); - safe_free(database); - safe_free(kmer_ids); -} - void insert_loc(Back_List *spr,int loc,int seedn,float len) { int list_loc[SI],list_score[SI],list_seed[SI],i,j,minval,mini; @@ -238,46 +281,48 @@ int find_location(int *t_loc,int *t_seedn,int *t_score,int *loc,int k,int *rep_l else return(0); } -int -seeding(const char* read, const int read_size, ref_index* ridx, SeedingBK* sbk) -{ - int* kmer_ids = sbk->kmer_ids; - int* index_list = sbk->index_list; +static int seeding(const char* read, const int read_size, const ref_index* ridx, SeedingBK& sbk) { + int* kmer_ids = sbk.kmer_ids; + int* index_list = sbk.index_list; int* index_spr = index_list; - short* index_score = sbk->index_score; + short* index_score = sbk.index_score; short* index_ss = index_score; - Back_List* database = sbk->database; - + Back_List* database = sbk.database; int num_kmers = extract_kmers(read, read_size, kmer_ids); - int km; int used_segs = 0; - for (km = 0; km < num_kmers; ++km) - { + for (int km = 0; km < num_kmers; ++km) { int num_seeds = ridx->kmer_counts[kmer_ids[km]]; int* seed_arr = ridx->kmer_starts[kmer_ids[km]]; int sid; int endnum = 0; - for (sid = 0; sid < num_seeds; ++sid) - { + for (sid = 0; sid < num_seeds; ++sid) { int seg_id = seed_arr[sid] / ZV; int seg_off = seed_arr[sid] % ZV; Back_List* spr = database + seg_id; - if (spr->score == 0 || spr->seednum < km + 1) - { + if (spr->score == 0 || spr->seednum < km + 1) { int loc = ++spr->score; - if (loc <= SM) { spr->loczhi[loc - 1] = seg_off; spr->seedno[loc - 1] = km + 1; } - else insert_loc(spr, seg_off, km + 1, BC); + if (loc <= SM) { + spr->loczhi[loc - 1] = seg_off; + spr->seedno[loc - 1] = km + 1; + } else { + insert_loc(spr, seg_off, km + 1, BC); + } int s_k; - if (seg_id > 0) s_k = spr->score + (spr - 1)->score; - else s_k = spr->score; - if (endnum < s_k) endnum = s_k; - if (spr->index == -1) - { + if (seg_id > 0) { + s_k = spr->score + (spr - 1)->score; + } else { + s_k = spr->score; + } + if (endnum < s_k) { + endnum = s_k; + } + if (spr->index == -1) { *(index_spr++) = seg_id; *(index_ss++) = s_k; spr->index = used_segs++; + } else { + index_score[spr->index] = s_k; } - else index_score[spr->index] = s_k; } spr->seednum = km + 1; } @@ -285,21 +330,12 @@ seeding(const char* read, const int read_size, ref_index* ridx, SeedingBK* sbk) return used_segs; } -int -get_candidates(volume_t* ref, - SeedingBK* sbk, - const int num_segs, - const int read_id, - const int read_size, - const char chain, - candidate_save* candidates, - int candidatenum) -{ - int* index_list = sbk->index_list; +static int get_candidates(const volume_t* ref, SeedingBK& sbk, const int num_segs, const int read_id, const int read_size, const char chain, candidate_save* candidates, int candidatenum) { + int* index_list = sbk.index_list; int* index_spr = index_list; - short* index_score = sbk->index_score; + short* index_score = sbk.index_score; short* index_ss = index_score; - Back_List* database = sbk->database; + Back_List* database = sbk.database; const int temp_arr_size = 2 * SM + 10; int temp_list[temp_arr_size],temp_seedn[temp_arr_size],temp_score[temp_arr_size]; candidate_save *candidate_loc = candidates, candidate_temp; @@ -408,7 +444,7 @@ get_candidates(volume_t* ref, for(u_k=*index_spr-1, spr1=spr-1; u_k>=0&&nlb>0; spr1--,--nlb,u_k--)if(spr1->score>0) { start_loc = MUL_ZV(u_k); - int scnt = min((int)spr1->score, SM); + int scnt = std::min((int)spr1->score, SM); for(j=0,s_k=0; j < scnt; j++)if(fabs((loc_list-start_loc-spr1->loczhi[j])/((loc_seed-spr1->seedno[j])*BC*1.0)-1.0)score>0) { start_loc = MUL_ZV(u_k); - int scnt = min((int)spr1->score, SM); + int scnt = std::min((int)spr1->score, SM); for(j=0,s_k=0; j < scnt; j++)if(fabs((start_loc+spr1->loczhi[j]-loc_list)/((spr1->seedno[j]-loc_seed)*BC*1.0)-1.0)=candidatenum||candidate_loc[mid].scorehigh; u_k--)candidate_loc[u_k+1]=candidate_loc[u_k]; + if(candidatenumhigh; u_k--)candidate_loc[u_k+1]=candidate_loc[u_k]; else for(u_k=candidatenum-2; u_k>high; u_k--)candidate_loc[u_k+1]=candidate_loc[u_k]; - if(high+1read_retrieve_lock); Lid = data->next_processed_id; - Rid = Lid + CHUNK_SIZE; - if (Rid > data->reads->num_reads) Rid = data->reads->num_reads; data->next_processed_id += CHUNK_SIZE; pthread_mutex_unlock(&data->read_retrieve_lock); + Rid = Lid + CHUNK_SIZE; + if (Rid > data->reads->num_reads) { + Rid = data->reads->num_reads; + } } -void -pairwise_mapping(PWThreadData* data, int tid) -{ - char *read, *read1, *read2, *subject; - safe_malloc(read1, char, MSS); - safe_malloc(read2, char, MSS); - safe_malloc(subject, char, MSS); - SeedingBK* sbk = new SeedingBK(data->reference->curr); - candidate_save candidates[MAXC]; - int num_candidates = 0; +static void pairwise_mapping(PWThreadData* const data, const int tid) { + char read1[MAX_SEQ_SIZE], read2[MAX_SEQ_SIZE], subject[MAX_SEQ_SIZE]; + SeedingBK sbk(data->reference->curr); + candidate_save candidates[maxc]; M4Record* m4_list = data->m4_results[tid]; int m4_list_size = 0; - M4Record* m4v = new M4Record[MAXC]; + M4Record* m4v = new M4Record[maxc]; int num_m4 = 0; GapAligner* aligner = NULL; if (data->options->tech == TECH_PACBIO) { aligner = new DiffAligner(0); - } else if (data->options->tech == TECH_NANOPORE) { - aligner = new XdropAligner(0); } else { - ERROR("TECH must be either %d or %d", TECH_PACBIO, TECH_NANOPORE); + aligner = new XdropAligner(0); } - - int rid, Lid, Rid; - while (1) - { + for (;;) { + int Lid, Rid; get_next_chunk_reads(data, Lid, Rid); - if (Lid >= data->reads->num_reads) break; - for (rid = Lid; rid < Rid; ++rid) - { - int rsize = data->reads->offset_list->offset_list[rid].size; + if (Lid >= data->reads->num_reads) { + break; + } + for (int rid(Lid); rid < Rid; ++rid) { + const int rsize(data->reads->offset_list->offset_list[rid].size); extract_one_seq(data->reads, rid, read1); reverse_complement(read2, read1, rsize); - int s; - char chain; - num_candidates = 0; - for (s = 0; s < 2; ++s) - { - if (s%2) { chain = 'R'; read = read2; } - else { chain = 'F'; read = read1; } - int num_segs = seeding(read, rsize, data->ridx, sbk); - num_candidates = get_candidates(data->reference, - sbk, - num_segs, - rid + data->reads->start_read_id, - rsize, - chain, - candidates, - num_candidates); - } - - for (s = 0; s < num_candidates; ++s) - { - if (candidates[s].chain == 'F') read = read1; - else read = read2; - extract_one_seq(data->reference, candidates[s].readno - data->reference->start_read_id, subject); - int sstart = candidates[s].loc1; - int qstart = candidates[s].loc2; - if (qstart && sstart) - { + int num_segs = seeding(read1, rsize, data->ridx, sbk); + int num_candidates = get_candidates(data->reference, sbk, num_segs, rid + data->reads->start_read_id, rsize, 'F', candidates, 0); + num_segs = seeding(read2, rsize, data->ridx, sbk); + num_candidates = get_candidates(data->reference, sbk, num_segs, rid + data->reads->start_read_id, rsize, 'R', candidates, num_candidates); + for (int i(0); i < num_candidates; ++i) { + extract_one_seq(data->reference, candidates[i].readno - data->reference->start_read_id, subject); + int sstart = candidates[i].loc1; + int qstart = candidates[i].loc2; + if (qstart && sstart) { qstart += kmer_size / 2; sstart += kmer_size / 2; } - int ssize = data->reference->offset_list->offset_list[candidates[s].readno - data->reference->start_read_id].size; - - int flag = aligner->go(read, qstart, rsize, subject, sstart, ssize, min_align_size); - - if (flag) - { - fill_m4record(aligner, rid + data->reads->start_read_id, - candidates[s].readno, candidates[s].chain, - rsize, ssize, qstart, sstart, candidates[s].score, - m4v + num_m4); + const int ssize(data->reference->offset_list->offset_list[candidates[i].readno - data->reference->start_read_id].size); + if (aligner->go(candidates[i].chain == 'F' ? read1 : read2, qstart, rsize, subject, sstart, ssize, min_align_size)) { + fill_m4record(aligner, rid + data->reads->start_read_id, candidates[i].readno, candidates[i].chain, rsize, ssize, qstart, sstart, candidates[i].score, m4v + num_m4); ++num_m4; } } - append_m4v(m4_list, &m4_list_size, m4v, &num_m4, data->out, &data->result_write_lock); } } - - if (m4_list_size) - { - pthread_mutex_lock(&data->result_write_lock); - print_m4record_list(data->out, m4_list, m4_list_size); - m4_list_size = 0; - pthread_mutex_unlock(&data->result_write_lock); - } - - safe_free(read1); - safe_free(read2); - safe_free(subject); - delete sbk; - delete aligner; - delete[] m4v; + if (m4_list_size) { + pthread_mutex_lock(&data->result_write_lock); + print_m4record_list(data->out, m4_list, m4_list_size); + pthread_mutex_unlock(&data->result_write_lock); + } + delete aligner; + delete[] m4v; } -void -candidate_detect(PWThreadData* data, int tid) -{ - char *read, *read1, *read2, *subject; - safe_malloc(read1, char, MAX_SEQ_SIZE); - safe_malloc(read2, char, MAX_SEQ_SIZE); - safe_malloc(subject, char, MAX_SEQ_SIZE); - SeedingBK* sbk = new SeedingBK(data->reference->curr); - Candidate candidates[MAXC]; - int num_candidates = 0; +static void candidate_detect(PWThreadData* const data, const int tid) { r_assert(data->ec_results); - ExtensionCandidate* eclist = data->ec_results[tid]; - int nec = 0; - ExtensionCandidate ec; - - int rid, Lid, Rid; - while (1) - { + ExtensionCandidate* const eclist(data->ec_results[tid]); + char read1[MAX_SEQ_SIZE], read2[MAX_SEQ_SIZE]; + candidate_save candidates[maxc]; + SeedingBK sbk(data->reference->curr); + const int start_match(data->options->reads_to_correct && data->options->reads_to_correct > data->reads->start_read_id ? data->options->reads_to_correct - data->reads->start_read_id : 0); + const int end_match(data->options->reads_to_correct ? data->options->reads_to_correct : data->reference->start_read_id + data->reference->num_reads); + int nec(0); + for (;;) { + int Lid, Rid; get_next_chunk_reads(data, Lid, Rid); - if (Lid >= data->reads->num_reads) break; - for (rid = Lid; rid < Rid; ++rid) - { - int rsize = data->reads->offset_list->offset_list[rid].size; - if (rsize >= MAX_SEQ_SIZE) { - cout << "rsize = " << rsize << "\t" << MAX_SEQ_SIZE << endl; - abort(); - } - extract_one_seq(data->reads, rid, read1); - reverse_complement(read2, read1, rsize); - int s; - int chain; - num_candidates = 0; - for (s = 0; s < 2; ++s) - { - if (s%2) { chain = REV; read = read2; } - else { chain = FWD; read = read1; } - int num_segs = seeding(read, rsize, data->ridx, sbk); - num_candidates = get_candidates(data->reference, - sbk, - num_segs, - rid + data->reads->start_read_id, - rsize, - chain, - candidates, - num_candidates); + if (Lid >= data->reads->num_reads) { + break; + } else if (Rid <= start_match) { // fast-forward to reads to match against + continue; + } else if (Lid < start_match) { + Lid = start_match; } - - for (s = 0; s < num_candidates; ++s) - { - int qstart = candidates[s].loc2; - int sstart = candidates[s].loc1; - if (qstart && sstart) - { - qstart += kmer_size / 2; - sstart += kmer_size / 2; + for (int rid(Lid); rid < Rid; ++rid) { + const int rsize(data->reads->offset_list->offset_list[rid].size); + if (rsize >= MAX_SEQ_SIZE) { + std::cerr << "rsize = " << rsize << "\t" << MAX_SEQ_SIZE << "\n"; + exit(1); } - int qdir = candidates[s].chain; - int sdir = FWD; - int qid = rid + data->reads->start_read_id; - int sid = candidates[s].readno; - int score = candidates[s].score; - - ec.qid = qid; - ec.qdir = qdir; - ec.qext = qstart; - ec.sid = sid; - ec.sdir = sdir; - ec.sext = sstart; - ec.score = score; - ec.qsize = rsize; - ec.ssize = data->reference->offset_list->offset_list[sid - data->reference->start_read_id].size; - if (ec.qdir == REV) ec.qext = ec.qsize - 1 - ec.qext; - if (ec.sdir == REV) ec.sext = ec.ssize - 1 - ec.sext; - eclist[nec] = ec; - ++nec; - if (nec == PWThreadData::kResultListSize) - { - pthread_mutex_lock(&data->result_write_lock); - for (int i = 0; i < nec; ++i) (*data->out) << eclist[i]; - nec = 0; - pthread_mutex_unlock(&data->result_write_lock); + extract_one_seq(data->reads, rid, read1); + int num_segs = seeding(read1, rsize, data->ridx, sbk); + int num_candidates = get_candidates(data->reference, sbk, num_segs, rid + data->reads->start_read_id, rsize, FWD, candidates, 0); + reverse_complement(read2, read1, rsize); + num_segs = seeding(read2, rsize, data->ridx, sbk); + num_candidates = get_candidates(data->reference, sbk, num_segs, rid + data->reads->start_read_id, rsize, REV, candidates, num_candidates); + for (int j = 0; j < num_candidates; ++j) { + ExtensionCandidate& ec(eclist[nec]); + ec.sid = candidates[j].readno; + // this can happen in the change-over volume + if (ec.sid >= end_match) { // skip non-match reads + continue; + } + ec.qext = candidates[j].loc2; + ec.sext = candidates[j].loc1; + if (ec.qext && ec.sext) { + ec.qext += kmer_size / 2; + ec.sext += kmer_size / 2; + } + ec.qid = rid + data->reads->start_read_id; + ec.qdir = candidates[j].chain; + ec.sdir = FWD; + ec.score = candidates[j].score; + ec.qsize = rsize; + ec.ssize = data->reference->offset_list->offset_list[ec.sid - data->reference->start_read_id].size; + if (ec.qdir == REV) { + ec.qext = ec.qsize - 1 - ec.qext; + } + if (++nec == PWThreadData::kResultListSize) { + pthread_mutex_lock(&data->result_write_lock); + for (int i(0); i < nec; ++i) { + (*data->out) << eclist[i]; + if (!(*data->out)) { + std::cerr << "Error writing output\n"; + exit(1); + } + } + pthread_mutex_unlock(&data->result_write_lock); + nec = 0; + } } } } - } - - if (nec) - { + if (nec) { pthread_mutex_lock(&data->result_write_lock); - for (int i = 0; i < nec; ++i) (*data->out) << eclist[i]; - nec = 0; + for (int i(0); i < nec; ++i) { + (*data->out) << eclist[i]; + if (!(*data->out)) { + std::cerr << "Error writing output\n"; + exit(1); + } + } pthread_mutex_unlock(&data->result_write_lock); } - - safe_free(read1); - safe_free(read2); - safe_free(subject); - delete sbk; } -void* -multi_thread_func(void* p) -{ - PWThreadData* data = (PWThreadData*)p; - int t = 0; +static void* multi_thread_func(void* const p) { + PWThreadData* const data(static_cast(p)); pthread_mutex_lock(&data->id_lock); - t = data->used_thread_id; - ++data->used_thread_id; + const int tid(data->used_thread_id++); pthread_mutex_unlock(&data->id_lock); - r_assert(data->options->task == TASK_SEED || data->options->task == TASK_ALN); - if (data->options->task == TASK_SEED) candidate_detect(data, t); - else pairwise_mapping(data, t); + if (data->options->task == TASK_SEED) { + candidate_detect(data, tid); + } else { + pairwise_mapping(data, tid); + } return NULL; } -void -process_one_volume(options_t* options, const int svid, const int evid, volume_names_t* vn, ostream* out) -{ - MAXC = options->num_candidates; +class ProcessState { + public: + int vid; + std::ofstream out; + ProcessState(const std::string &volume_results_name, const int svid) : ckpt_file_name_(volume_results_name + ".ckpt"), volume_results_name_(volume_results_name), volume_results_name_tmp_(volume_results_name + ".tmp") { + std::ifstream ckpt_in(ckpt_file_name_.c_str()); + if (ckpt_in) { + off_t out_pos_; + ckpt_in >> vid >> out_pos_; + if (!ckpt_in) { + std::cerr << "Error: failed to restore from checkpoint: " << ckpt_file_name_ << "\n"; + exit(1); + } + // set ios_base::in to prevent truncation + out.open(volume_results_name_tmp_.c_str(), std::ios_base::out | std::ios_base::in); + out.seekp(out_pos_); + } else { + vid = svid; + // can't set ios_base::in, as we need to create the file + out.open(volume_results_name_tmp_.c_str()); + } + if (!out) { + std::cerr << "Could not open volume results file: " << volume_results_name_tmp_ << "\n"; + exit(1); + } + } + ~ProcessState() { } + void checkpoint() { + std::string ckpt_file_name_tmp(ckpt_file_name_ + ".tmp"); + std::ofstream ckpt_out(ckpt_file_name_tmp.c_str()); + if (!ckpt_out) { + std::cerr << "Checkpoint failed at volume " << vid << ", could not open " << ckpt_file_name_tmp << "\n"; + return; + } + out.flush(); + ckpt_out << vid << "\n" << off_t(out.tellp()) << "\n"; + if (!ckpt_out) { + std::cerr << "Checkpoint failed at volume " << vid << ", write error\n"; + return; + } + ckpt_out.close(); + if (rename(ckpt_file_name_tmp.c_str(), ckpt_file_name_.c_str()) == -1) { + std::cerr << "Checkpoint failed at volume " << vid << ", could not rename: " << ckpt_file_name_tmp << "\n"; + } + } + void finish() { + out.close(); + if (rename(volume_results_name_tmp_.c_str(), volume_results_name_.c_str()) == -1) { + std::cerr << "Could not rename volume results file: " << volume_results_name_tmp_ << "\n"; + exit(1); + } + unlink(ckpt_file_name_.c_str()); + } + private: + std::string ckpt_file_name_; + std::string volume_results_name_; + std::string volume_results_name_tmp_; +}; + +void process_one_volume(const options_t* options, const int svid, const int evid, const std::string& volume_results_name, volume_names_t* vn) { + const std::string volume_results_name_working(volume_results_name + ".tmp"); + const std::string volume_results_name_checkpoint(volume_results_name + ".ckpt"); + maxc = options->num_candidates; output_gapped_start_point = options->output_gapped_start_point; min_align_size = options->min_align_size; min_kmer_match = options->min_kmer_match; - + r_assert(options->task == TASK_SEED || options->task == TASK_ALN); + r_assert(options->tech == TECH_NANOPORE || options->tech == TECH_PACBIO); if (options->tech == TECH_PACBIO) { ddfs_cutoff = ddfs_cutoff_pacbio; min_kmer_dist = 1800; - } else if (options->tech == TECH_NANOPORE) { + } else { ddfs_cutoff = ddfs_cutoff_nanopore; min_kmer_dist = 400; - } else { - ERROR("TECH must be either %d or %d", TECH_PACBIO, TECH_NANOPORE); } - - const char* ref_name = get_vol_name(vn, svid); - volume_t* ref = load_volume(ref_name); - ref_index* ridx = create_ref_index(ref, kmer_size, options->num_threads); + const char* ref_name(get_vol_name(vn, svid)); + volume_t* ref(load_volume(ref_name)); + ProcessState state(volume_results_name, svid); + if (state.vid == svid && ref->start_read_id + ref->num_reads <= options->reads_to_correct) { + // go forward until we get to a volume to match against + for (++state.vid; state.vid < evid; ++state.vid) { + const char* read_name = get_vol_name(vn, state.vid); + volume_t* read = load_volume_header(read_name); + if (read->start_read_id + read->num_reads > options->reads_to_correct) { + delete_volume_t(read); + break; + } + delete_volume_t(read); + } + } + if (state.vid >= evid) { + return; + } + ref_index* ridx(create_ref_index(ref, kmer_size, options->num_threads)); pthread_t tids[options->num_threads]; - char volume_process_info[1024];; - int vid, tid; - for(vid = svid; vid < evid; ++vid) - { - sprintf(volume_process_info, "process volume %d", vid); + char volume_process_info[1024]; + PWThreadData data(options, ref, ridx, &state.out); + for (;;) { + sprintf(volume_process_info, "process volume %d", state.vid); DynamicTimer dtimer(volume_process_info); - const char* read_name = get_vol_name(vn, vid); - LOG(stderr, "processing %s\n", read_name); - volume_t* read = load_volume(read_name); - PWThreadData* data = new PWThreadData(options, ref, read, ridx, out); - for (tid = 0; tid < options->num_threads; ++tid) - { - int err_code = pthread_create(tids + tid, NULL, multi_thread_func, (void*)data); - if (err_code) - { - LOG(stderr, "Error: return code is %d\n", err_code); - abort(); + const char* const read_name(get_vol_name(vn, state.vid)); + LOG(stderr, "processing %s", read_name); + data.start(read_name); + for (int tid(0); tid < options->num_threads; ++tid) { + int err_code = pthread_create(tids + tid, NULL, multi_thread_func, (void*)&data); + if (err_code) { + LOG(stderr, "Error: return code is %d", err_code); + exit(1); } } - for (tid = 0; tid < options->num_threads; ++tid) pthread_join(tids[tid], NULL); - read = delete_volume_t(read); - delete data; + for (int tid(0); tid < options->num_threads; ++tid) { + pthread_join(tids[tid], NULL); + } + // checkpoint should point to next volume to process + // (don't bother to checkpoint last volume) + if (++state.vid >= evid) { + break; + } + state.checkpoint(); + data.finish(); } - ref = delete_volume_t(ref); - ridx = destroy_ref_index(ridx); + state.finish(); + destroy_ref_index(ridx); + delete_volume_t(ref); } diff --git a/src/mecat2pw/pw_impl.h b/src/mecat2pw/pw_impl.h index 2738be5..f915e0c 100644 --- a/src/mecat2pw/pw_impl.h +++ b/src/mecat2pw/pw_impl.h @@ -1,69 +1,13 @@ #ifndef PW_IMPL_H #define PW_IMPL_H -#include - #include "../common/alignment.h" #include "../common/packed_db.h" #include "../common/lookup_table.h" -#define RM 100000 -#define DN 500 -#define BC 10 -#define SM 40 -#define SI 41 -#define CHUNK_SIZE 500 -#define ZV 2000 -#define MUL_ZV(a) ((a)*ZV) -#define DIV_ZV(a) ((a)/ZV) -#define MOD_ZV(a) ((a)%ZV) - -struct candidate_save -{ - int loc1,loc2,left1,left2,right1,right2,score,num1,num2,readno,readstart; - char chain; -}; - -typedef candidate_save Candidate; - -struct Back_List -{ - short score,loczhi[SM],seedno[SM],seednum; - int index; -}; - -struct PWThreadData -{ - options_t* options; - int used_thread_id; - pthread_mutex_t id_lock; - volume_t* reference; - volume_t* reads; - ref_index* ridx; - std::ostream* out; - M4Record** m4_results; - ExtensionCandidate** ec_results; - static const int kResultListSize = 10000; - pthread_mutex_t result_write_lock; - int next_processed_id; - pthread_mutex_t read_retrieve_lock; - - PWThreadData(options_t* opt, volume_t* ref, volume_t* rd, ref_index* idx, std::ostream* o); - ~PWThreadData(); -}; - -struct SeedingBK -{ - int* index_list; - short* index_score; - Back_List* database; - int* kmer_ids; - - SeedingBK(const int ref_size); - ~SeedingBK(); -}; +#include "pw_options.h" // options_t +#include // string -void -process_one_volume(options_t* options, const int svid, const int evid, volume_names_t* vn, std::ostream* out); +void process_one_volume(const options_t* options, int svid, int evid, const std::string &volume_results_name, volume_names_t* vn); #endif // PW_IMPL_H diff --git a/src/mecat2pw/pw_options.cpp b/src/mecat2pw/pw_options.cpp index 5a41947..1c5ad4a 100644 --- a/src/mecat2pw/pw_options.cpp +++ b/src/mecat2pw/pw_options.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include static const int kDefaultNumThreads = 1; static const int kDefaultNumCandidates = 100; @@ -27,6 +29,59 @@ print_options(options_t* options) LOG(stderr, "tech\t%d", options->tech); } +// given options, recreate arguments from the command line +std::string +make_options(options_t* options) +{ + std::stringstream cmd; + if (options->task == TASK_ALN || options->task == TASK_SEED) { + cmd << " -j " << options->task; + } + if (options->reads != NULL) { + cmd << " -d " << options->reads; + } + if (options->output != NULL) { + cmd << " -o " << options->output; + } + if (options->wrk_dir != NULL) { + cmd << " -w " << options->wrk_dir; + } + if (options->grid_options != NULL) { + cmd << " -G " << options->grid_options; + } + if (options->grid_options_split != NULL) { + cmd << " -S " << options->grid_options_split; + } + if (options->num_threads > 0) { + cmd << " -t " << options->num_threads; + } + if (options->num_candidates > 0) { + cmd << " -n " << options->num_candidates; + } + if (options->min_align_size > 0) { + cmd << " -a " << options->min_align_size; + } + if (options->min_kmer_match > 0) { + cmd << " -k " << options->min_kmer_match; + } + if (options->output_gapped_start_point > 0) { + cmd << " -g " << options->output_gapped_start_point; + } + if (options->tech == TECH_PACBIO || options->tech == TECH_NANOPORE) { + cmd << " -x " << (options->tech == TECH_PACBIO ? 0 : 1); + } + if (options->num_vols != -1) { + cmd << " -N " << options->num_vols; + } + if (options->job_index != -1) { + cmd << " -i " << options->job_index; + } + if (options->reads_to_correct) { + cmd << " -R " << options->reads_to_correct; + } + return cmd.str(); +} + void init_options(options_t* options, int tech) { @@ -35,10 +90,15 @@ init_options(options_t* options, int tech) options->reads = NULL; options->output = NULL; options->wrk_dir = NULL; + options->grid_options = NULL; + options->grid_options_split = NULL; options->num_threads = 1; options->num_candidates = 100; options->output_gapped_start_point = 0; options->tech = tech; + options->job_index = -1; + options->num_vols = -1; + options->reads_to_correct = 0; if (tech == TECH_PACBIO) { options->min_align_size = kDefaultAlignSizePacbio; @@ -68,6 +128,11 @@ void print_usage(const char* prog) fprintf(stderr, "Default: %d if x = %d, %d if x = %d\n", kDefaultKmerMatchPacbio, TECH_PACBIO, kDefaultKmerMatchNanopore, TECH_NANOPORE); fprintf(stderr, "-g <0/1>\twhether print gapped extension start point, 0 = no, 1 = yes\n\t\tDefault: 0\n"); fprintf(stderr, "-x <0/x>\tsequencing technology: 0 = pacbio, 1 = nanopore\n\t\tDefault: 0\n"); + fprintf(stderr, "-R \tnumber of reads to error correct [all]\n"); + fprintf(stderr, "-G \tscheduler command/options\n"); + fprintf(stderr, "-S \tscheduler command/options (for split)\n"); + fprintf(stderr, "(note that grid command/options have to start with \"qsub\" and end with \"-N\")\n"); + fprintf(stderr, "(note that slurm command/options have to start with \"sbatch\" and end with \"-J\")\n"); } int @@ -82,14 +147,19 @@ parse_arguments(int argc, char* argv[], options_t* options) const char* reads = NULL; const char* output = NULL; const char* wrk_dir = NULL; + const char* grid_options = NULL; + const char* grid_options_split = NULL; int num_threads = -1; int num_candidates = -1; int min_align_size = -1; int min_kmer_match = -1; int output_gapped_start_point = -1; int tech = TECH_PACBIO; + int job_index = -1; + int num_vols = -1; + int reads_to_correct = 0; - while((opt_char = getopt(argc, argv, "j:d:o:w:t:n:g:x:a:k:")) != -1) + while((opt_char = getopt(argc, argv, "j:d:o:w:t:n:g:x:a:k:G:i:N:R:S:")) != -1) { switch(opt_char) { @@ -105,9 +175,24 @@ parse_arguments(int argc, char* argv[], options_t* options) case 'w': wrk_dir = optarg; break; + case 'G': + grid_options = optarg; + break; + case 'S': + grid_options_split = optarg; + break; case 't': num_threads = atoi(optarg); break; + case 'i': + job_index = atoi(optarg); + break; + case 'N': + num_vols = atoi(optarg); + break; + case 'R': + reads_to_correct = atoi(optarg); + break; case 'n': num_candidates = atoi(optarg); break; @@ -155,11 +240,16 @@ parse_arguments(int argc, char* argv[], options_t* options) options->reads = reads; options->output = output; options->wrk_dir = wrk_dir; + if (grid_options != NULL) options->grid_options = grid_options; + if (grid_options_split != NULL) options->grid_options_split = grid_options_split; if (num_threads != -1) options->num_threads = num_threads; if (num_candidates != -1) options->num_candidates = num_candidates; if (min_align_size != -1) options->min_align_size = min_align_size; if (min_kmer_match != -1) options->min_kmer_match = min_kmer_match; if (output_gapped_start_point != -1) options->output_gapped_start_point = output_gapped_start_point; + if (job_index != -1) options->job_index = job_index; + if (num_vols != -1) options->num_vols = num_vols; + if (reads_to_correct) options->reads_to_correct = reads_to_correct; if (options->task != TASK_SEED && options->task != TASK_ALN) { @@ -192,6 +282,11 @@ parse_arguments(int argc, char* argv[], options_t* options) LOG(stderr, "number of candidates must be > 0."); ret = 1; } + else if (options->reads_to_correct < 0) + { + LOG(stderr, "reads to correct must be > 0."); + ret = 1; + } if (ret) return ret; diff --git a/src/mecat2pw/pw_options.h b/src/mecat2pw/pw_options.h index e15cd57..5c5fa8c 100644 --- a/src/mecat2pw/pw_options.h +++ b/src/mecat2pw/pw_options.h @@ -1,6 +1,8 @@ #ifndef MP_OPTIONS_H #define MP_OPTIONS_H +#include + #include "../common/defs.h" #define TASK_SEED 0 @@ -12,14 +14,22 @@ typedef struct const char* reads; const char* output; const char* wrk_dir; + const char* grid_options; + const char* grid_options_split; int num_threads; int num_candidates; int min_align_size; int min_kmer_match; int output_gapped_start_point; int tech; + int num_vols; + int job_index; + int reads_to_correct; // reads must lead fasta file } options_t; +std::string +make_options(options_t* options); + void print_options(options_t* options); From 52683346682ae06f6a3fa4c62a34cfd4ebc1d97f Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 22 Mar 2019 13:28:07 -0500 Subject: [PATCH 02/56] Mild reformat and warning removal Some minor warnings (signed/unsigned comparisons and such) were fixed, and packed_db has been reformatted in preparation to allow larger read sets. --- src/common/defs.h | 5 +- src/common/packed_db.cpp | 263 +++++++---------- src/common/packed_db.h | 213 +++++++------- src/common/xdrop_gapalign.cpp | 3 +- src/mecat2cns/mecat_correction.cpp | 382 ++++++++++++++----------- src/mecat2cns/options.cpp | 13 +- src/mecat2cns/options.h | 1 + src/mecat2cns/reads_correction_aux.h | 2 +- src/mecat2cns/reads_correction_can.cpp | 4 +- src/mecat2pw/pw_impl.cpp | 2 +- src/mecat2ref/mecat2ref.cpp | 3 +- src/mecat2ref/mecat2ref_aux.cpp | 2 +- src/mecat2ref/mecat2ref_impl_large.cpp | 5 +- 13 files changed, 450 insertions(+), 448 deletions(-) diff --git a/src/common/defs.h b/src/common/defs.h index ffbd630..b11acf8 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -89,14 +89,14 @@ do { \ do { \ size_t __sm__sz__ = sizeof(type) * (count); \ (arr) = (type *)malloc(__sm__sz__); \ - if (!(arr)) ERROR("malloc fail"); \ + if (!(arr)) ERROR("malloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sm__sz__); \ } while(0) #define safe_calloc(arr, type, count) \ do { \ size_t __sc__sz__ = sizeof(type) * (count); \ (arr) = (type *)calloc(1, __sc__sz__); \ - if (!(arr)) ERROR("calloc fail"); \ + if (!(arr)) ERROR("calloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sc__sz__); \ } while(0) #define safe_realloc(arr, type, count) \ @@ -105,6 +105,7 @@ do { \ arr = (type *)realloc(arr, __sr__size__); \ if (!arr) \ { \ + ERROR("failed to realloc memory: %lu * %lu = %lu", sizeof(type), size_t(count), __sr__size__); \ ERROR("failed to realloc memory."); \ exit(1); \ } \ diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp index 7441d92..ab8f46c 100644 --- a/src/common/packed_db.cpp +++ b/src/common/packed_db.cpp @@ -6,221 +6,178 @@ #include "defs.h" #include "fasta_reader.h" -using namespace std; - -void -PackedDB::dump_pac(u1_t* p, const idx_t size, const char* path) -{ - ofstream out; - open_fstream(out, path, std::ios::out | std::ios::binary); - streambuf* sb = out.rdbuf(); - sb_write(sb, p, (size + 3)/4); - sb_write(sb, &size, sizeof(idx_t)); - close_fstream(out); +void PackedDB::dump_pac(const u1_t* const p, const idx_t size, const char* const path) { + std::ofstream out; + open_fstream(out, path, std::ios::out | std::ios::binary); + std::streambuf* sb(out.rdbuf()); + sb_write(sb, p, (size + 3) / 4); + sb_write(sb, &size, sizeof(idx_t)); + close_fstream(out); } -u1_t* -PackedDB::load_pac(const char* path, idx_t& size) -{ - ifstream in; - open_fstream(in, path, std::ios::in | std::ios::binary); - streambuf* sb = in.rdbuf(); - in.seekg(-sizeof(idx_t), ios::end); - sb_read(sb, &size, sizeof(idx_t)); - in.seekg(0, ios::beg); +u1_t* PackedDB::load_pac(const char* path, idx_t& size) { + std::ifstream in; + open_fstream(in, path, std::ios::in | std::ios::binary); + std::streambuf* sb(in.rdbuf()); + in.seekg(-sizeof(idx_t), std::ios::end); + sb_read(sb, &size, sizeof(idx_t)); + in.seekg(0, std::ios::beg); u1_t* p; - safe_calloc(p, u1_t, (size+3)/4); - sb_read(sb, p, (size + 3)/4); - close_fstream(in); + safe_calloc(p, u1_t, (size + 3) / 4); + sb_read(sb, p, (size + 3) / 4); + close_fstream(in); return p; } -void -PackedDB::dump_idx(PODArray& idx_list, const char* path) -{ - ofstream out; - open_fstream(out, path, ios::out); - idx_t i = 0, n = idx_list.size(); - for (i = 0; i < n; ++i) - out << idx_list[i].id << "\t" << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; - close_fstream(out); +void PackedDB::dump_idx(const PODArray& idx_list, const char* const path) { + std::ofstream out; + open_fstream(out, path, std::ios::out); + idx_t i(0), n(idx_list.size()); + for (i = 0; i < n; ++i) { + out << idx_list[i].id << "\t" << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; + } + close_fstream(out); } -void -PackedDB::load_idx(const char* path, PODArray& idx_list) -{ - ifstream in; - open_fstream(in, path, ios::in); - SeqIndex si; - idx_list.clear(); - while(in >> si.id >> si.offset >> si.size) idx_list.push_back(si); - close_fstream(in); +void PackedDB::load_idx(const char* const path, PODArray& idx_list) { + idx_list.clear(); + std::ifstream in; + open_fstream(in, path, std::ios::in); + SeqIndex si; + while (in >> si.id >> si.offset >> si.size) { + idx_list.push_back(si); + } + close_fstream(in); } -void -PackedDB::dump_packed_db(const char* path) -{ - string n; - generate_pac_name(path, n); - dump_pac(pac, db_size, n.data()); - generate_idx_name(path, n); - dump_idx(seq_idx, n.data()); +void PackedDB::dump_packed_db(const char* const path) const { + std::string n; + generate_pac_name(path, n); + dump_pac(pac, db_size, n.c_str()); + generate_idx_name(path, n); + dump_idx(seq_idx, n.c_str()); } -void -PackedDB::load_packed_db(const char* path) -{ - string n; - generate_pac_name(path, n); - pac = load_pac(n.data(), db_size); +void PackedDB::load_packed_db(const char* const path) { + std::string n; + generate_pac_name(path, n); + pac = load_pac(n.c_str(), db_size); max_db_size = db_size; - generate_idx_name(path, n); - load_idx(n.data(), seq_idx); + generate_idx_name(path, n); + load_idx(n.c_str(), seq_idx); } -void -PackedDB::pack_fasta_db(const char* path, const char* output_prefix, const idx_t min_size) -{ +void PackedDB::pack_fasta_db(const char* const path, const char* const output_prefix, const idx_t min_size) { u1_t* buffer; safe_malloc(buffer, u1_t, MAX_SEQ_SIZE); - Sequence read; + const u1_t* const et(get_dna_encode_table()); FastaReader fr(path); - string n; + std::string n; generate_pac_name(output_prefix, n); - ofstream pout; - open_fstream(pout, n.data(), ios::out | ios::binary); - streambuf* psb = pout.rdbuf(); + std::ofstream pout; + open_fstream(pout, n.c_str(), std::ios::out | std::ios::binary); + std::streambuf* psb(pout.rdbuf()); generate_idx_name(output_prefix, n); - ofstream iout; - open_fstream(iout, n.data(), ios::out); - - const u1_t* et = get_dna_encode_table(); - idx_t id = 0, tsize = 0, rsize = 0; - SeqIndex si; - while(1) - { - rsize = fr.read_one_seq(read); - if (rsize == -1) break; - if (rsize < min_size) continue; - Sequence::str_t& s = read.sequence(); + std::ofstream iout; + open_fstream(iout, n.c_str(), std::ios::out); + Sequence read; + idx_t id(-1), tsize(0); + for (;;) { + idx_t rsize(fr.read_one_seq(read)); + if (rsize == -1) { + break; + } else if (rsize < min_size) { + continue; + } + Sequence::str_t& s(read.sequence()); memset(buffer, 0, MAX_SEQ_SIZE); - for(idx_t i = 0; i < rsize; ++i) - { - u1_t c = s[i]; - c = et[c]; - if (c > 3) c = 0; - set_char(buffer, i, c); + for (idx_t i(0); i < rsize; ++i) { + const u1_t c(et[static_cast(s[i])]); + set_char(buffer, i, c < 3 ? c : 3); } - - si.offset = tsize; - si.size = rsize; - si.id = id++; - iout << si.id << "\t" << si.offset << "\t" << si.size << "\n"; - + iout << ++id << "\t" << tsize << "\t" << rsize << "\n"; rsize = (rsize + 3) / 4; sb_write(psb, buffer, rsize); - rsize *= 4; - tsize += rsize; + tsize += rsize * 4; } - sb_write(psb, &tsize, sizeof(idx_t)); close_fstream(pout); close_fstream(iout); safe_free(buffer); - - LOG(stdout, "pack %lld reads, totally %lld residues", (long long)id, (long long)tsize); + LOG(stdout, "pack %lld reads, totally %lld residues", (long long)(id + 1), (long long)tsize); } -void PackedDB::add_one_seq(const Sequence& seq) -{ +void PackedDB::add_one_seq(const Sequence& seq) { SeqIndex si; si.size = seq.size(); si.offset = db_size; seq_idx.push_back(si); - - if (db_size + si.size > max_db_size) - { - idx_t new_size = (max_db_size) ? max_db_size : 1024; - while (db_size + si.size > new_size) new_size *= 2; - u1_t* new_pac = NULL; - safe_calloc(new_pac, u1_t, (new_size + 3)/4); - memcpy(new_pac, pac, (db_size + 3)/4); + const idx_t needed_size(db_size + si.size); + if (max_db_size < needed_size) { + idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + for (; new_size < needed_size; new_size *= 2) { } + u1_t* new_pac(NULL); + safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + memcpy(new_pac, pac, (db_size + 3) / 4); safe_free(pac); pac = new_pac; max_db_size = new_size; } - const Sequence::str_t& org_seq = seq.sequence(); - const u1_t* table = get_dna_encode_table(); - for (idx_t i = 0; i < si.size; ++i) - { - u1_t c = org_seq[i]; - c = table[c]; - if (c > 3) c = rand() & 3; - set_char(db_size, c); - ++db_size; + const Sequence::str_t& org_seq(seq.sequence()); + const u1_t* const table(get_dna_encode_table()); + for (idx_t i(0); i < si.size; ++i, ++db_size) { + const u1_t c(table[static_cast(org_seq[i])]); + set_char(db_size, c < 3 ? c : 3); } } -void PackedDB::add_one_seq(const char* seq, const idx_t size) -{ +void PackedDB::add_one_seq(const char* const seq, const idx_t size) { SeqIndex si; si.size = size; si.offset = db_size; seq_idx.push_back(si); - - if (db_size + si.size > max_db_size) - { - idx_t new_size = (max_db_size) ? max_db_size : 1024; - while (db_size + si.size > new_size) new_size *= 2; - u1_t* new_pac = NULL; - safe_calloc(new_pac, u1_t, (new_size + 3)/4); - memcpy(new_pac, pac, (db_size + 3)/4); + const idx_t needed_size(db_size + si.size); + if (max_db_size < needed_size) { + idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + for (; new_size < needed_size; new_size *= 2) { } + u1_t* new_pac(NULL); + safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + memcpy(new_pac, pac, (db_size + 3) / 4); safe_free(pac); pac = new_pac; max_db_size = new_size; } - - const u1_t* table = get_dna_encode_table(); - for (idx_t i = 0; i < si.size; ++i) - { - u1_t c = seq[i]; - c = table[c]; - if (c > 3) c = rand() & 3; - set_char(db_size, c); - ++db_size; + const u1_t* const table(get_dna_encode_table()); + for (idx_t i(0); i < si.size; ++i, ++db_size) { + const u1_t c(table[static_cast(seq[i])]); + set_char(db_size, c < 3 ? c : 3); } } -void PackedDB::load_fasta_db(const char* dbname) -{ +void PackedDB::load_fasta_db(const char* const dbname) { DynamicTimer dtimer(__func__); FastaReader freader(dbname); Sequence seq; - while (1) - { - idx_t size = freader.read_one_seq(seq); - if (size == -1) break; + while (freader.read_one_seq(seq) != -1) { add_one_seq(seq); } } -idx_t PackedDB::offset_to_rid(const idx_t offset) const -{ - if (offset >= db_size) return -1; - idx_t left, mid, right; - left = 0, mid = 0, right = seq_idx.size(); - while (left < right) - { +idx_t PackedDB::offset_to_rid(const idx_t offset) const { + if (offset >= db_size) { + return -1; + } + idx_t left(0), mid(0), right(seq_idx.size()); + while (left < right) { mid = (left + right) >> 1; - if (offset >= seq_idx[mid].offset) - { - if (mid == seq_idx.size() - 1) break; - if (offset < seq_idx[mid + 1].offset) break; - left = mid + 1; - } - else - { + if (offset < seq_idx[mid].offset) { right = mid; + } else if (mid == seq_idx.size() - 1) { + break; + } else if (offset < seq_idx[mid + 1].offset) { + break; + } else { + left = mid + 1; } } return mid; diff --git a/src/common/packed_db.h b/src/common/packed_db.h index 6360557..293831b 100644 --- a/src/common/packed_db.h +++ b/src/common/packed_db.h @@ -4,29 +4,23 @@ #include "defs.h" #include "sequence.h" -class PackedDB -{ -public: - struct SeqIndex - { - idx_t id; - idx_t offset; - idx_t size; - }; - -public: - PackedDB() : pac(NULL), db_size(0), max_db_size(0) {} - ~PackedDB() { destroy(); } - void reserve(const idx_t& size) - { +class PackedDB { + public: + struct SeqIndex { + idx_t id, offset, size; + }; + public: + PackedDB() : pac(NULL), db_size(0), max_db_size(0) {} + ~PackedDB() { + destroy(); + } + void reserve(const idx_t size) { destroy(); - seq_idx.clear(); max_db_size = size; - idx_t bytes = (max_db_size / 4); + idx_t bytes = max_db_size / 4; safe_calloc(pac, u1_t, bytes); } - - void GetSequence(const idx_t id, const bool fwd, char* const seq, const idx_t size) { + void GetSequence(const idx_t id, const bool fwd, char* const seq, const idx_t size) const { r_assert(size == seq_idx[id].size); if (fwd) { const idx_t offset(seq_idx[id].offset); @@ -40,107 +34,104 @@ class PackedDB } } } - - void get_sequence(const idx_t from, const idx_t to, const bool forward, char* seq) const - { - idx_t idx = 0; - if (forward) - for(idx_t i = from; i < to; ++i) - { - u1_t c = get_char(pac, i); - seq[idx++] = c; - } - else - for(idx_t i = to - 1; i >= from; --i) - { - u1_t c = get_char(pac, i); - c = 3 - c; - seq[idx++] = c; - } - } - - void get_sequence(const idx_t rid, const bool forward, char* seq) const - { - const idx_t s = seq_idx[rid].offset; - const idx_t e = s + seq_idx[rid].size; - get_sequence(s, e, forward, seq); - } - - void get_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* seq) const - { - const idx_t s = seq_idx[rid].offset + from; - const idx_t e = seq_idx[rid].offset + to; - get_sequence(s, e, forward, seq); - } - - void get_decode_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* seq) const - { - get_sequence(rid, from, to, forward, seq); - for(idx_t i = 0; i < to - from; ++i) - { - u1_t c = seq[i]; - r_assert(c >= 0 && c < 4); - c = "ACGT"[c]; - seq[i] = c; + void get_sequence(const idx_t from, const idx_t to, const bool forward, char* const seq) const { + if (forward) { + idx_t idx(0); + for (idx_t i(from); i < to; ++i, ++idx) { + seq[idx] = get_char(pac, i); + } + } else { + idx_t idx(to - from - 1); + for (idx_t i(from); i < to; ++i, --idx) { + seq[idx] = 3 - get_char(pac, i); + } } } - - static void set_char(u1_t* p, const idx_t idx, const u1_t c) - { - p[idx >> 2] |= c << ((~idx&3)<<1); - } - - static u1_t get_char(const u1_t* p, const idx_t idx) - { - u1_t c = p[idx >> 2] >> ((~idx&3)<<1)&3; - return c; - } - - void set_char(const idx_t idx, const u1_t c) - { + void get_sequence(const idx_t rid, const bool forward, char* const seq) const { + const SeqIndex& a(seq_idx[rid]); + get_sequence(a.offset, a.offset + a.size, forward, seq); + } + void get_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { + const idx_t offset(seq_idx[rid].offset); + get_sequence(offset + from, offset + to, forward, seq); + } + static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { + s.reserve(s.size() + end_i - i); + for (; i < end_i; ++i) { + s += "ACGT"[static_cast(seq[i])]; + } + } + static void decode_sequence(char* const seq, const idx_t seq_size) { + for (idx_t i(0); i < seq_size; ++i) { + const u1_t c(seq[i]); + r_assert(c < 4); // c is unsigned, so always >= 0 + seq[i] = "ACGT"[c]; + } + } + void get_decode_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { + get_sequence(rid, from, to, forward, seq); + decode_sequence(seq, to - from); + } + static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { + p[idx >> 2] |= c << ((~idx & 3) << 1); + } + static u1_t get_char(const u1_t* const p, const idx_t idx) { + return p[idx >> 2] >> ((~idx & 3) << 1) & 3; + } + void set_char(const idx_t idx, const u1_t c) { set_char(pac, idx, c); } - u1_t get_char(const idx_t idx) const - { + u1_t get_char(const idx_t idx) const { return get_char(pac, idx); } - - idx_t size() const { return db_size; } - idx_t num_seqs() const { return seq_idx.size(); } - idx_t seq_offset(const idx_t rid) const { return seq_idx[rid].offset; } - idx_t seq_size(const idx_t rid) const { return seq_idx[rid].size; } - idx_t offset_to_rid(const idx_t offset) const; - void add_one_seq(const Sequence& seq); + idx_t size() const { + return db_size; + } + idx_t num_seqs() const { + return seq_idx.size(); + } + idx_t seq_offset(const idx_t rid) const { + return seq_idx[rid].offset; + } + idx_t seq_size(const idx_t rid) const { + return seq_idx[rid].size; + } + void destroy() { + if (pac) { + safe_free(pac); + } + seq_idx.clear(); + db_size = max_db_size = 0; + } + void clear() { + seq_idx.clear(); + db_size = 0; + memset(pac, 0, (max_db_size + 3) / 4); + } + static void generate_pac_name(const char* const prefix, std::string& ret) { + ret = prefix; + ret += ".pac"; + } + static void generate_idx_name(const char* const prefix, std::string& ret) { + ret = prefix; + ret += ".idx"; + } + idx_t offset_to_rid(const idx_t offset) const; + void add_one_seq(const Sequence& seq); void add_one_seq(const char* seq, const idx_t size); - void destroy() { if (pac) safe_free(pac); db_size = max_db_size = 0; } - void clear() { seq_idx.clear(); db_size = 0; memset(pac, 0, (max_db_size + 3)/4); } - - static void generate_pac_name(const char* prefix, std::string& ret) - { - ret = prefix; - ret += ".pac"; - } - static void generate_idx_name(const char* prefix, std::string& ret) - { - ret = prefix; - ret += ".idx"; - } - static void dump_pac(u1_t* p, const idx_t size, const char* path); - static u1_t* load_pac(const char* path, idx_t& size); - static void dump_idx(PODArray& idx_list, const char* path); - static void load_idx(const char* path, PODArray& idx_list); - - void dump_packed_db(const char* path); - void load_packed_db(const char* path); - + static void dump_pac(const u1_t* p, idx_t size, const char* path); + static u1_t* load_pac(const char* path, idx_t& size); + static void dump_idx(const PODArray& idx_list, const char* path); + static void load_idx(const char* path, PODArray& idx_list); + void dump_packed_db(const char* path) const; + void load_packed_db(const char* path); static void pack_fasta_db(const char* fasta, const char* output_prefix, const idx_t min_size); void load_fasta_db(const char* fasta); - -private: - u1_t* pac; - idx_t db_size; - idx_t max_db_size; - PODArray seq_idx; + private: + u1_t* pac; + idx_t db_size; + idx_t max_db_size; + PODArray seq_idx; }; #endif // PACKED_DB_H diff --git a/src/common/xdrop_gapalign.cpp b/src/common/xdrop_gapalign.cpp index 25ec2ec..7a98ed8 100644 --- a/src/common/xdrop_gapalign.cpp +++ b/src/common/xdrop_gapalign.cpp @@ -302,7 +302,8 @@ align_ex(const char* query, qblk, tblk); - int score = xdrop_align(Q, + //int score = xdrop_align(Q, + xdrop_align(Q, qblk, T, tblk, diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 411629e..d977bcc 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -11,15 +11,21 @@ namespace ns_meap_cns { #define FINS 4 #define UNDS 8 -inline uint1 -identify_one_consensus_item(CnsTableItem& cns_item, const int min_cov) -{ - uint1 ident = 0; - int cov = cns_item.mat_cnt + cns_item.ins_cnt; - if (cns_item.mat_cnt >= cov * 0.8) ident |= FMAT; - if (cns_item.ins_cnt >= cov * 0.8) ident |= FINS; - if (!ident) ident |= UNDS; - if (cns_item.del_cnt >= cov * 0.4) ident |= FDEL; +// returns type of coverage present + +inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item, const int min_cov) { + const int cov(ceil((cns_item.mat_cnt + cns_item.ins_cnt) * 0.8)); + uint1 ident; + if (cns_item.mat_cnt >= cov) { // coverage is 80% or more matches + ident = FMAT; + } else if (cns_item.ins_cnt >= cov) { // coverage is 80% or more inserts + ident = FINS; + } else { // neither of the above + ident = UNDS; + } + if (2 * cns_item.del_cnt >= cov) { // deletes are 40% or more than the coverage + ident |= FDEL; + } return ident; } @@ -33,43 +39,36 @@ struct CompareOverlapByOverlapSize } }; -void -meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* cns_table, const char* org_seq) -{ +void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* const cns_table, const char* const org_seq) { r_assert(qaln.size() == saln.size()); - const idx_t aln_size = qaln.size(); - idx_t i = 0; - const char kGap = '-'; - while (i < aln_size) - { - const char q = qaln[i]; - const char s = saln[i]; - if (q == kGap && s == kGap) { ++i; continue; } - - if (q == s) { ++cns_table[start_soff].mat_cnt; cns_table[start_soff].base = s; ++start_soff; ++i; } - else if (q == kGap) { ++cns_table[start_soff].ins_cnt; ++start_soff; ++i; } - else - { - r_assert(s == kGap); - idx_t j = i + 1; - while (j < aln_size && saln[j] == kGap) ++j; + const idx_t aln_size(qaln.size()); + for (idx_t i(0); i < aln_size; ) { + const char q(qaln[i]); + const char s(saln[i]); + if (q == '-' && s == '-') { // skip + ++i; + } else if (q == s) { // match + ++cns_table[start_soff].mat_cnt; + cns_table[start_soff].base = s; + ++start_soff; + ++i; + } else if (q == '-') { // insert + ++cns_table[start_soff].ins_cnt; + ++start_soff; + ++i; + } else { // delete + r_assert(s == '-'); + for (++i; i < aln_size && saln[i] == '-'; ++i) { } ++cns_table[start_soff - 1].del_cnt; - i = j; } } } -void -meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, - const int min_cov, std::string& aux_qstr, - std::string& aux_tstr, std::string& cns) -{ +void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int min_cov, std::string& aux_qstr, std::string& aux_tstr, std::string& cns) { AlnGraphBoost ag(se - sb + 1); int sb_out; - for (CnsAln* iter = cns_vec.begin(); iter != cns_vec.end(); ++iter) - { - if ((*iter).retrieve_aln_subseqs(sb, se, aux_qstr, aux_tstr, sb_out)) - { + for (CnsAln* a(cns_vec.begin()); a != cns_vec.end(); ++a) { + if (a->retrieve_aln_subseqs(sb, se, aux_qstr, aux_tstr, sb_out)) { ag.addAln(aux_qstr, aux_tstr, sb_out - sb + 1); } } @@ -77,87 +76,102 @@ meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, ag.consensus(min_cov * 0.4, cns); } -void -meap_consensus_one_segment(CnsTableItem* cns_list, const int cns_list_size, - uint1* cns_id_vec, - int start_soff, CnsAlns& cns_vec, - std::string& aux_qstr, std::string& aux_tstr, - std::string& target, const int min_cov) -{ - for (int i = 0; i < cns_list_size; ++i) cns_id_vec[i] = identify_one_consensus_item(cns_list[i], min_cov); - int i = 0, j; +void meap_consensus_one_segment(const CnsTableItem* const cns_list, const int cns_list_size, uint1* const cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { + // get types of coverage + for (int i(0); i < cns_list_size; ++i) { + cns_id_vec[i] = identify_one_consensus_item(cns_list[i], min_cov); + } std::string cns; target.clear(); - while (i < cns_list_size && !(cns_id_vec[i] & FMAT)) ++i; - while (i < cns_list_size) - { + const uint1 unds_or_fdel(UNDS | FDEL); // questionable coverage types + int i(0); + // advance to matching coverage + for (; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } + while (i < cns_list_size) { target.push_back(cns_list[i].base); - j = i + 1; - while (j < cns_list_size && !(cns_id_vec[j] & FMAT)) ++j; - - bool need_refinement = false; - for (int k = i; k < j; ++k) - if ((cns_id_vec[k] & UNDS) || (cns_id_vec[k] & FDEL)) { need_refinement = true; break; } - if (need_refinement) - { - meap_cns_one_indel(i + start_soff, j + start_soff, cns_vec, cns_list[i].mat_cnt + cns_list[i].ins_cnt, aux_qstr, aux_tstr, cns); - if (cns.size() > 2) target.append(cns.data() + 1, cns.size() - 2); + const int start(i); + // advance to next matching coverage + for (++i; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } + int need_refinement(0); + // check to see if anything in-between has questionable coverage + for (int k(start); k < i; ++k) { + if (cns_id_vec[k] & unds_or_fdel) { + need_refinement = 1; + break; + } + } + if (need_refinement) { + meap_cns_one_indel(start + start_soff, i + start_soff, cns_vec, cns_list[start].mat_cnt + cns_list[start].ins_cnt, aux_qstr, aux_tstr, cns); + // trim first and last as they have good coverage + if (cns.size() > 2) { + target.append(cns.data() + 1, cns.size() - 2); + } } - i = j; } } -struct CmpMappingRangeBySoff -{ - bool operator()(const MappingRange& a, const MappingRange& b) - { - return (a.start == b.start) ? (a.end > b.end) : (a.start < b.start); +struct CmpMappingRangeBySoff { + bool operator()(const MappingRange& a, const MappingRange& b) { + if (a.start != b.start) { + return a.start < b.start; + } else { + return b.end < a.end; + } } }; -void -get_effective_ranges(std::vector& mranges, std::vector& eranges, const int read_size, const int min_size) -{ +void get_effective_ranges(std::vector& mranges, std::vector& eranges, const int read_size, const int min_size) { eranges.clear(); - if (mranges.size() == 0) return; - std::vector::iterator iter; - for (iter = mranges.begin(); iter != mranges.end(); ++iter) - if (iter->start <= 500 && read_size - iter->end <= 500) - { + if (mranges.size() == 0) { + return; + } + // see if any ranges are effectively the whole read + std::vector::const_iterator a(mranges.begin()); + const std::vector::const_iterator end_a(mranges.end()); + for (; a != end_a; ++a) { + if (a->start <= 500 && read_size - a->end <= 500) { eranges.push_back(MappingRange(0, read_size)); return; } - + } std::sort(mranges.begin(), mranges.end(), CmpMappingRangeBySoff()); - const int nr = mranges.size(); - int i = 0, j; - int left = mranges[i].start, right; - while (i < nr) - { - j = i + 1; - while (j < nr && mranges[j].end <= mranges[i].end) ++j; - if (j == nr) - { - right = mranges[i].end; - if (right - left >= min_size * 0.95) eranges.push_back(MappingRange(left, right)); + const int nr(mranges.size()); + // -1 so we can use > instead of >= + const int min_size_95(ceil(min_size * 0.95) - 1); + int i(0), left(mranges[0].start); + for (;;) { + const int right(mranges[i].end); + // include ranges we completely overlap + for (++i; i < nr && mranges[i].end <= right; ++i) { } + if (i == nr) { + if (right - left > min_size_95) { + eranges.push_back(MappingRange(left, right)); + } break; } - if (mranges[i].end - mranges[j].start < 1000) - { - right = std::min(mranges[i].end, mranges[j].start); - if (right - left >= min_size * 0.95) eranges.push_back(MappingRange(left, right)); - left = std::max(mranges[i].end, mranges[j].start); + // if the next non-contained range overlaps by 1k or more, + // treat as part of this range and keep extending + if (right - mranges[i].start < 1000) { + // truncate current range at overlap start + const int eff_right(std::min(right, mranges[i].start)); + if (eff_right - left > min_size_95) { + eranges.push_back(MappingRange(left, eff_right)); + } + // put some space between the effective ranges + left = std::max(right, mranges[i].start); } - i = j; } } +// breaks output sequence into chunks of no more than MaxSeqSize (if needed); +// split chunks will have an OvlpSize overlap + void output_cns_result(std::vector& cns_results, CnsResult& cr, const idx_t beg, const idx_t end, - std::string& cns_seq) + const std::string& cns_seq) { const size_t MaxSeqSize = 60000; const size_t OvlpSize = 10000; @@ -199,103 +213,128 @@ check_ovlp_mapping_range(const int qb, const int qe, const int qs, return oq >= qqs || os >= qss; } -void -consensus_worker(CnsTableItem* cns_table, - uint1* id_list, - CnsAlns& cns_vec, - std::string& aux_qstr, - std::string& aux_tstr, - std::vector& eranges, - const int min_cov, - const int min_size, - const int read_id, - std::vector& cns_results) -{ - idx_t beg = 0, end; +// look for areas of high coverage of about min_size or more, +// improve them and stick on the results pile + +void consensus_worker(const CnsTableItem* const cns_table, uint1* const id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { CnsResult cns_result; - std::string cns_seq; cns_result.id = read_id; - std::vector::iterator miter; - for (miter = eranges.begin(); miter != eranges.end(); ++miter) - { - int L = miter->start, R = miter->end; - beg = L; - while (beg < R) - { - while (beg < R && cns_table[beg].mat_cnt + cns_table[beg].ins_cnt < min_cov) ++beg; - end = beg + 1; - while (end < R && cns_table[end].mat_cnt + cns_table[end].ins_cnt >= min_cov) ++end; - if (end - beg >= 0.95 * min_size) - { - meap_consensus_one_segment(cns_table + beg, end - beg, id_list, - beg, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); - - if (cns_seq.size() >= min_size) output_cns_result(cns_results, cns_result, beg, end, cns_seq); + const idx_t min_size_95(ceil(0.95 * min_size)); + std::string cns_seq; + std::vector::const_iterator a(eranges.begin()); + const std::vector::const_iterator end_a(eranges.end()); + for (; a != end_a; ++a) { + const int end_i(a->end); + for (idx_t i(a->start); i < end_i;) { + // find start of next high coverage area + for (; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } + const idx_t start(i); + // find end of high coverage area + for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } + if (i - start >= min_size_95) { + meap_consensus_one_segment(cns_table + start, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + if (cns_seq.size() >= static_cast(min_size)) { + output_cns_result(cns_results, cns_result, start, i, cns_seq); + } } - - beg = end; } } } -void -consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) -{ - PackedDB& reads = ctd.reads; - ExtensionCandidate* overlaps = pctd.candidates; - DiffRunningData* drd_s = pctd.drd_s; - DiffRunningData* drd = NULL; - M5Record& m5 = pctd.m5; - CnsAlns& cns_vec = pctd.cns_alns; - std::vector& cns_results = pctd.cns_results; - const idx_t read_size = overlaps[sid].ssize; - std::vector& qstr = pctd.query; - std::vector& tstr = pctd.target; - tstr.resize(read_size); - reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = pctd.qaln; - std::string& ntstr = pctd.saln; - const int min_align_size = ctd.rco.min_align_size; - const int max_added = 60; +// same as consensus_worker, but produces entire read as one entry; +// uncorrected sections are just copied as is; - idx_t L, R; - if (eid - sid <= max_added) - { - L = sid; - R = eid; +void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { + CnsResult cns_result; + cns_result.id = read_id; + const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); + std::string cns_seq; + std::vector::const_iterator a(eranges.begin()); + const std::vector::const_iterator end_a(eranges.end()); + std::vector::const_iterator last_a(eranges.end()); + for (; a != end_a; last_a = a++) { + if (last_a != end_a) { // add in-between range to cns_result + PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, a->start); + } else if (a->start > 0) { // add beginning of read + PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), 0, a->start); + } + const int begin_i(a->start - 1); + const int end_i(a->end); + for (idx_t i(begin_i); i < end_i;) { + // find start of next high coverage area + const idx_t last_end(i != begin_i ? i : a->start); + for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } + // add low coverage area as-is + PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_end, i); + if (i == end_i) { + break; + } + const idx_t start(i); + // find end of high coverage area + for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } + if (i - start > min_size_95) { + meap_consensus_one_segment(cns_table + start, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + if (cns_seq.size() >= static_cast(min_size)) { + // add corrected sequence + cns_result.seq += cns_seq; + continue; + } + } + // add uncorrected sequence + PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), start, i); + } } - else - { - L = sid; - R = L + max_added; - std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); + // add end of read + if (last_a != end_a) { + PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, tstr.size()); } + cns_result.range[0] = 0; + cns_result.range[1] = cns_result.seq.size(); + cns_results.push_back(cns_result); +} - CnsTableItem* cns_table = pctd.cns_table; +void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { + PackedDB& reads(ctd.reads); + ExtensionCandidate* overlaps(pctd.candidates); + DiffRunningData* const drd_s(pctd.drd_s); + DiffRunningData* drd(NULL); + M5Record& m5(pctd.m5); + CnsAlns& cns_vec(pctd.cns_alns); + std::vector& cns_results(pctd.cns_results); + const idx_t read_size(overlaps[sid].ssize); + std::vector& qstr(pctd.query); + std::vector& tstr(pctd.target); + tstr.resize(read_size); + reads.GetSequence(read_id, true, tstr.data(), read_size); + std::string& nqstr(pctd.qaln); + std::string& ntstr(pctd.saln); + const int min_align_size(ctd.rco.min_align_size); + const int max_added(60); + CnsTableItem* cns_table(pctd.cns_table); std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); cns_vec.clear(); - for (idx_t i = L; i < R; ++i) - { - Overlap& ovlp = overlaps[i]; + const idx_t L(sid); + const idx_t R(eid - sid <= max_added ? eid : L + max_added); + if (eid - sid > max_added) { // only use largest max_added overlaps + std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); + } + for (idx_t i(L); i < R; ++i) { + Overlap& ovlp(overlaps[i]); qstr.resize(ovlp.qsize); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); - idx_t qext = ovlp.qext; - idx_t sext = ovlp.sext; - if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; + const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); + const idx_t sext(ovlp.sext); drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size); - if (r) - { + const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size)); + if (r) { normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } } - std::vector mranges, eranges; cns_vec.get_mapping_ranges(mranges); get_effective_ranges(mranges, eranges, read_size, ctd.rco.min_size); - consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } @@ -409,10 +448,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD qstr.resize(ec.qsize); reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); const idx_t sext(ec.sext); - idx_t qext(ec.qext); - if (ec.qdir == REV) { - qext = ec.qsize - 1 - qext; - } + const idx_t qext(ec.qdir == FWD ? ec.qext : ec.qsize - 1 - ec.qext); const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) { if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { @@ -427,7 +463,11 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD std::vector mranges, eranges; cns_vec.get_mapping_ranges(mranges); get_effective_ranges(mranges, eranges, read_size, ctd.rco.min_size); - consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); + if (ctd.rco.full_reads) { + consensus_worker_one_read(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, tstr, cns_results); + } else { + consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); + } } void diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index caf0cd3..6699109 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -29,6 +29,7 @@ static int tech_nanopore = TECH_NANOPORE; static int default_tech = TECH_PACBIO; static int num_partition_files = 0; +static int full_reads = 0; static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -45,6 +46,7 @@ static const char grid_options_split_n = 'S'; static const char job_index_n = 'I'; static const char reads_to_correct_n = 'R'; static const char grid_start_delay_n = 'D'; +static const char full_reads_n = 'F'; void print_pacbio_default_options() @@ -113,6 +115,9 @@ make_options(const ConsensusOptions& options) if (options.grid_start_delay) { cmd << " -" << grid_start_delay_n << " " << options.grid_start_delay; } + if (options.full_reads) { + cmd << " -" << full_reads_n; + } cmd << " " << options.m4; cmd << " " << options.reads; cmd << " " << options.corrected_reads; @@ -138,6 +143,7 @@ void print_usage(const char* prog) { << "-" << grid_options_split_n << " \toptions for split grid submission\n" << "-" << reads_to_correct_n << " \tnumber of reads to correct [all]\n" << "-" << grid_start_delay_n << " \tseconds to delay between starting grid jobs\n" + << "-" << full_reads_n << "\t\toutput full reads, not just the corrected parts\n" << "-" << usage_n << "\t\tprint usage info.\n" << "\n" << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; @@ -158,6 +164,7 @@ ConsensusOptions init_consensus_options(const int tech) { t.job_index = -1; t.reads_to_correct = 0; t.grid_start_delay = 0; + t.full_reads = full_reads; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; t.num_threads = num_threads_pacbio; @@ -216,7 +223,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { int opt_char; char err_char; opterr = 0; - while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:")) != -1) { + while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:F")) != -1) { switch (opt_char) { case input_type_n: if (optarg[0] == '0') { @@ -269,6 +276,9 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { case num_partition_files_n: t.num_partition_files = atoi(optarg); break; + case full_reads_n: + t.full_reads = 1; + break; case '?': err_char = (char)optopt; fprintf(stderr, "unrecognised option '%c'\n", err_char); @@ -327,6 +337,7 @@ print_options(ConsensusOptions& t) if (t.corrected_reads) std::cout << "m4\t" << t.corrected_reads << "\n"; if (t.grid_options) std::cout << "grid\t" << t.grid_options << "\n"; if (t.grid_options_split) std::cout << "grid_split\t" << t.grid_options_split << "\n"; + if (t.full_reads) std::cout << "full reads\n"; std::cout << "number of threads:\t" << t.num_threads << "\n"; std::cout << "batch size:\t" << t.batch_size << "\n"; std::cout << "mapping ratio:\t" << t.min_mapping_ratio << "\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index 6ed35ae..0a7bd7f 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -27,6 +27,7 @@ struct ConsensusOptions int job_index; int reads_to_correct; int grid_start_delay; + int full_reads; }; void diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 1e9e396..dd5ac01 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -209,7 +209,7 @@ class ConsensusThreadData { } } pdata.next_candidate = i; - if (++num_threads_written_ <= rco.num_threads) { + if (++num_threads_written_ >= rco.num_threads) { checkpoint(); num_threads_written_ = 0; } diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index f0df158..03c1be1 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -14,14 +14,14 @@ static void* reads_correction_func_can(void* const arg) { const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); const ExtensionCandidate* const candidates(pdata.candidates); - const idx_t min_size(data.rco.min_size * 0.95); + const idx_t min_size(ceil(data.rco.min_size * 0.95)); const int tech_is_pacbio(data.rco.tech == TECH_PACBIO ? 1 : 0); idx_t i(pdata.next_candidate); while (i < pdata.num_candidates) { const idx_t start(i); const idx_t sid(candidates[start].sid); for (++i; i < pdata.num_candidates && candidates[i].sid == sid; ++i) { } - if (i < start + data.rco.min_cov || candidates[start].ssize < min_size) { + if (i - start < data.rco.min_cov || candidates[start].ssize < min_size) { continue; } if (tech_is_pacbio) { diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index f5d8ce0..6818e55 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -203,7 +203,7 @@ void insert_loc(Back_List *spr,int loc,int seedn,float len) int find_location(int *t_loc,int *t_seedn,int *t_score,int *loc,int k,int *rep_loc,float len,int read_len1) { - int i,j,maxval=0,maxi,rep=0,lasti = 0,tempi; + int i,j,maxval=0,maxi=0,rep=0,lasti = 0,tempi; for(i=0; i0&&t_loc[j]-t_loc[i]>0&&t_loc[j]-t_loc[i]0&&t_loc[j]-t_loc[i]>0&&t_loc[j]-t_loc[i] Date: Fri, 22 Mar 2019 13:45:08 -0500 Subject: [PATCH 03/56] removed id from seq index It was just the position of the entry in the array, which is kinda pointless. --- src/common/packed_db.cpp | 11 ++++++----- src/common/packed_db.h | 2 +- src/common/split_database.cpp | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp index ab8f46c..b477197 100644 --- a/src/common/packed_db.cpp +++ b/src/common/packed_db.cpp @@ -34,7 +34,7 @@ void PackedDB::dump_idx(const PODArray& idx_list, const char* const pa open_fstream(out, path, std::ios::out); idx_t i(0), n(idx_list.size()); for (i = 0; i < n; ++i) { - out << idx_list[i].id << "\t" << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; + out << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; } close_fstream(out); } @@ -44,7 +44,7 @@ void PackedDB::load_idx(const char* const path, PODArray& idx_list) { std::ifstream in; open_fstream(in, path, std::ios::in); SeqIndex si; - while (in >> si.id >> si.offset >> si.size) { + while (in >> si.offset >> si.size) { idx_list.push_back(si); } close_fstream(in); @@ -81,7 +81,7 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr std::ofstream iout; open_fstream(iout, n.c_str(), std::ios::out); Sequence read; - idx_t id(-1), tsize(0); + idx_t count(0), tsize(0); for (;;) { idx_t rsize(fr.read_one_seq(read)); if (rsize == -1) { @@ -95,16 +95,17 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr const u1_t c(et[static_cast(s[i])]); set_char(buffer, i, c < 3 ? c : 3); } - iout << ++id << "\t" << tsize << "\t" << rsize << "\n"; + iout << tsize << "\t" << rsize << "\n"; rsize = (rsize + 3) / 4; sb_write(psb, buffer, rsize); tsize += rsize * 4; + ++count; } sb_write(psb, &tsize, sizeof(idx_t)); close_fstream(pout); close_fstream(iout); safe_free(buffer); - LOG(stdout, "pack %lld reads, totally %lld residues", (long long)(id + 1), (long long)tsize); + LOG(stdout, "pack %lld reads, totally %lld residues", (long long)count, (long long)tsize); } void PackedDB::add_one_seq(const Sequence& seq) { diff --git a/src/common/packed_db.h b/src/common/packed_db.h index 293831b..43e3250 100644 --- a/src/common/packed_db.h +++ b/src/common/packed_db.h @@ -7,7 +7,7 @@ class PackedDB { public: struct SeqIndex { - idx_t id, offset, size; + idx_t offset, size; }; public: PackedDB() : pac(NULL), db_size(0), max_db_size(0) {} diff --git a/src/common/split_database.cpp b/src/common/split_database.cpp index d7ae843..f334997 100644 --- a/src/common/split_database.cpp +++ b/src/common/split_database.cpp @@ -414,7 +414,7 @@ split_dataset(const char* reads, const char* wrk_dir, int* num_vols) generate_idx_file_name(wrk_dir, idx_file_name); FILE* idx_file = fopen(idx_file_name, "w"); PackedDB::SeqIndex si; - while (in_idx_file >> si.id >> si.offset >> si.size) + while (in_idx_file >> si.offset >> si.size) { if (v->curr + si.size + 1 > MCS) { From d97eada1c82366c955fa404165b2b295eb536685 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 25 Mar 2019 00:13:41 -0500 Subject: [PATCH 04/56] remove rand() calls, reformatting The rand() calls for non-AGCT basepairs in packed_db got replaced by a deterministic function to allow identical output from reruns. Some more reformatting while planning the upcoming change allowing for large fasta files in mecat2cns. --- src/common/fasta_reader.cpp | 187 ++++++++++--------------- src/common/fasta_reader.h | 89 ++++++------ src/common/packed_db.cpp | 16 ++- src/common/packed_db.h | 13 +- src/mecat2cns/reads_correction_can.cpp | 9 +- 5 files changed, 136 insertions(+), 178 deletions(-) diff --git a/src/common/fasta_reader.cpp b/src/common/fasta_reader.cpp index 25a0bab..c0d1586 100644 --- a/src/common/fasta_reader.cpp +++ b/src/common/fasta_reader.cpp @@ -3,128 +3,87 @@ #include #include -idx_t FastaReader::read_one_seq(Sequence& seq) -{ - seq.clear(); - bool need_defline = true; - while (++m_Reader) - { - const OneDataLine& odl = m_Reader.get_line(); - if (odl.size() == 0) continue; - const int c = odl.front(); - if (c == '>' || c == '@') - { - if (need_defline) - { - x_parse_defline(odl, seq.header()); - need_defline = false; - continue; - } - else - { - m_Reader.unget_line(); - break; - } - } - else if (c == '+') - { - if (!++m_Reader) +idx_t FastaReader::read_one_seq(Sequence& seq) { + seq.clear(); + bool need_defline(1); + while (++m_Reader) { + const OneDataLine& line(m_Reader.get_line()); + if (line.size() == 0) { + continue; + } + const int c(line.front()); + if (c == '>' || c == '@') { + if (need_defline) { + x_parse_defline(line, seq.header()); + need_defline = 0; + continue; + } else { + m_Reader.unget_line(); + break; + } + } else if (c == '+') { + if (!++m_Reader) { ERROR("FastaReader: quality score line is missing at around line %lld", (long long)m_Reader.line_number()); - break; - } - else if (is_comment_line(odl)) - { - continue; - } - else if (need_defline) - { + } + break; + } else if (is_comment_line(line)) { + continue; + } else if (need_defline) { ERROR("FastaReader: Input doesn't start with a defline or comment around line %lld", (long long)m_Reader.line_number()); - } - - x_parse_data_line(odl, seq.sequence()); - } - - if (seq.size() == 0 && seq.header().size() > 0) - { + } + x_parse_data_line(line, seq.sequence()); + } + if (seq.size() == 0 && seq.header().size() > 0) { ERROR("FastaReader: Near line %lld, sequence data is missing.", (long long)m_Reader.line_number()); - } - - if (seq.header().size() == 0 && seq.sequence().size() == 0) return -1; - return seq.size(); + } + if (seq.header().size() == 0 && seq.sequence().size() == 0) { + return -1; + } + return seq.size(); } - -void FastaReader::x_parse_data_line(const OneDataLine& line, str_t& seq) -{ - x_check_data_line(line); - const idx_t len = line.size(); - idx_t curr_pos = seq.size(); - seq.resize(seq.size() + len); - idx_t pos = 0; - for (pos = 0; pos < len; ++pos) - { - const int c = line[pos]; - if (c == ';') break; - if (is_nucl(c) || c == '-') - { - seq[curr_pos++] = c; - } - else if (!isspace(c)) - { +void FastaReader::x_parse_data_line(const OneDataLine& line, str_t& seq) { + x_check_data_line(line); + const idx_t len(line.size()); + idx_t curr_pos(seq.size()); + seq.resize(seq.size() + len); + for (idx_t pos(0); pos < len; ++pos) { + const int c(line[pos]); + if (c == ';') { + break; + } else if (is_nucl(c) || c == '-') { + seq[curr_pos++] = c; + } else if (!isspace(c)) { ERROR("FastaReader: There are invalid residue(s) around position %d of line %lld.", (int)(pos + 1), (long long)m_Reader.line_number()); - } - } - - seq.resize(curr_pos); + } + } + seq.resize(curr_pos); } - -void FastaReader::x_parse_defline(const OneDataLine& line, str_t& header) -{ - header.clear(); - header.push_back(line.begin() + 1, line.size() - 1); - if (header.size() == 0) - { - const idx_t line_number = m_Reader.line_number(); - ERROR("A sequence is given an empty header around line %lld.", (long long)line_number); - } +void FastaReader::x_parse_defline(const OneDataLine& line, str_t& header) { + header.clear(); + header.push_back(line.begin() + 1, line.size() - 1); + if (header.size() == 0) { + ERROR("FastaReader: A sequence is given an empty header around line %lld.", (long long)m_Reader.line_number()); + } } -void FastaReader::x_check_data_line(const OneDataLine& line) -{ - idx_t good = 0, bad = 0, len = line.size(); - idx_t ambig_nucl = 0; - for (idx_t pos = 0; pos < len; ++pos) - { - const unsigned char c = line[pos]; - if (is_alpha(c) || c == '*') - { - ++good; - if (is_nucl(c) && is_ambig_nucl(c)) ++ambig_nucl; - } - else if (c == '-') - { - ++good; - } - else if (isspace(c) || (c >= '0' && c <= '9')) - { - - } - else if (c == ';') - { - break; - } - else - { - ++bad; - } - } - - if (bad >= good / 3 && (len > 3 || good == 0 || bad > good)) - { - ERROR("FastaReader: Near line %lld, there's a line that doesn't look like plausible data, but it's not marked as defline or commnet.", - (long long)m_Reader.line_number()); - } +void FastaReader::x_check_data_line(const OneDataLine& line) { + idx_t good(0), bad(0); + const idx_t len(line.size()); + for (idx_t pos(0); pos < len; ++pos) { + const unsigned char c(line[pos]); + if (is_alpha(c) || c == '*') { // potentially ambiguous bases + ++good; + } else if (c == '-') { + ++good; + } else if (c == ';') { + break; + } else if (!isspace(c) && (c < '0' || '9' < c)) { + ++bad; + } + } + if (3 * bad >= good && (len > 3 || good == 0 || bad > good)) { + ERROR("FastaReader: Near line %lld, there's a line that doesn't look like plausible data, but it's not marked as defline or comment.", (long long)m_Reader.line_number()); + } } - - diff --git a/src/common/fasta_reader.h b/src/common/fasta_reader.h index 8c6ac5d..8b43d12 100644 --- a/src/common/fasta_reader.h +++ b/src/common/fasta_reader.h @@ -4,53 +4,48 @@ #include "buffer_line_iterator.h" #include "sequence.h" -class FastaReader -{ -public: - typedef Sequence::str_t str_t; - typedef BufferLineReader::OneDataLine OneDataLine; - -public: - FastaReader(const char* fasta_file_name) : m_Reader(fasta_file_name) { encode_table = get_dna_encode_table(); } - idx_t read_one_seq(Sequence& seq); - std::streampos tellg() const { return m_Reader.tellg(); } - void seekg(std::streampos pos) { m_Reader.seekg(pos); } - -private: - void x_parse_defline(const OneDataLine& line, str_t& header); - void x_parse_data_line(const OneDataLine& line, str_t& seq); - void x_check_data_line(const OneDataLine& line); - bool is_header_line(const OneDataLine& line) - { return line.size() > 0 && (line.front() == '>' || line.front() == '@'); } - bool is_nucl(const unsigned char ch) - { - int r = encode_table[ch]; - return r < 16; - } - bool is_ambig_nucl(const unsigned char ch) - { - int r = encode_table[ch]; - return (r < 16) && (r > 3); - } - bool is_upper_case_letter(const char ch) - { - return ch >= 'A' && ch <= 'Z'; - } - bool is_lower_case_letter(const char ch) - { - return ch >= 'a' && ch <= 'z'; - } - bool is_alpha(const unsigned char c) - { - return is_upper_case_letter(c) || is_lower_case_letter(c); - } - bool is_comment_line(const OneDataLine& line) - { return line.front() == '#' || line.front() == '!'; } - - -private: - BufferLineReader m_Reader; - const u1_t* encode_table; +class FastaReader { + public: + typedef Sequence::str_t str_t; + typedef BufferLineReader::OneDataLine OneDataLine; + public: + FastaReader(const char* fasta_file_name) : m_Reader(fasta_file_name), encode_table(get_dna_encode_table()) { } + idx_t read_one_seq(Sequence& seq); + std::streampos tellg() const { + return m_Reader.tellg(); + } + void seekg(std::streampos pos) { + m_Reader.seekg(pos); + } + private: + void x_parse_defline(const OneDataLine& line, str_t& header); + void x_parse_data_line(const OneDataLine& line, str_t& seq); + void x_check_data_line(const OneDataLine& line); + static bool is_header_line(const OneDataLine& line) { + return line.size() > 0 && (line.front() == '>' || line.front() == '@'); + } + bool is_nucl(const unsigned char c) const { + return encode_table[c] < 16; + } + bool is_ambig_nucl(const unsigned char c) const { + const int r(encode_table[c]); + return 3 < r && r < 16; + } + static bool is_upper_case_letter(const char c) { + return 'A' <= c && c <= 'Z'; + } + static bool is_lower_case_letter(const char c) { + return 'a' <= c && c <= 'z'; + } + static bool is_alpha(const char c) { + return is_upper_case_letter(c) || is_lower_case_letter(c); + } + static bool is_comment_line(const OneDataLine& line) { + return line.front() == '#' || line.front() == '!'; + } + private: + BufferLineReader m_Reader; + const u1_t* const encode_table; }; #endif // FASTA_READER_H diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp index b477197..e83fa40 100644 --- a/src/common/packed_db.cpp +++ b/src/common/packed_db.cpp @@ -32,8 +32,8 @@ u1_t* PackedDB::load_pac(const char* path, idx_t& size) { void PackedDB::dump_idx(const PODArray& idx_list, const char* const path) { std::ofstream out; open_fstream(out, path, std::ios::out); - idx_t i(0), n(idx_list.size()); - for (i = 0; i < n; ++i) { + const idx_t n(idx_list.size()); + for (idx_t i(0); i < n; ++i) { out << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; } close_fstream(out); @@ -81,6 +81,7 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr std::ofstream iout; open_fstream(iout, n.c_str(), std::ios::out); Sequence read; + unsigned int rand_char(-1); idx_t count(0), tsize(0); for (;;) { idx_t rsize(fr.read_one_seq(read)); @@ -90,10 +91,11 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr continue; } Sequence::str_t& s(read.sequence()); - memset(buffer, 0, MAX_SEQ_SIZE); + // set_char uses | to set bits, so clear first + memset(buffer, 0, (rsize + 3) / 4); for (idx_t i(0); i < rsize; ++i) { const u1_t c(et[static_cast(s[i])]); - set_char(buffer, i, c < 3 ? c : 3); + set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } iout << tsize << "\t" << rsize << "\n"; rsize = (rsize + 3) / 4; @@ -126,9 +128,10 @@ void PackedDB::add_one_seq(const Sequence& seq) { } const Sequence::str_t& org_seq(seq.sequence()); const u1_t* const table(get_dna_encode_table()); + unsigned int rand_char(-1); for (idx_t i(0); i < si.size; ++i, ++db_size) { const u1_t c(table[static_cast(org_seq[i])]); - set_char(db_size, c < 3 ? c : 3); + set_char(db_size, c < 4 ? c : ++rand_char & 3); } } @@ -149,9 +152,10 @@ void PackedDB::add_one_seq(const char* const seq, const idx_t size) { max_db_size = new_size; } const u1_t* const table(get_dna_encode_table()); + unsigned int rand_char(-1); for (idx_t i(0); i < si.size; ++i, ++db_size) { const u1_t c(table[static_cast(seq[i])]); - set_char(db_size, c < 3 ? c : 3); + set_char(db_size, c < 4 ? c : ++rand_char & 3); } } diff --git a/src/common/packed_db.h b/src/common/packed_db.h index 43e3250..8859274 100644 --- a/src/common/packed_db.h +++ b/src/common/packed_db.h @@ -10,27 +10,26 @@ class PackedDB { idx_t offset, size; }; public: - PackedDB() : pac(NULL), db_size(0), max_db_size(0) {} + PackedDB() : pac(NULL), db_size(0), max_db_size(0) { } ~PackedDB() { destroy(); } void reserve(const idx_t size) { destroy(); max_db_size = size; - idx_t bytes = max_db_size / 4; - safe_calloc(pac, u1_t, bytes); + safe_calloc(pac, u1_t, (max_db_size + 3) / 4); } - void GetSequence(const idx_t id, const bool fwd, char* const seq, const idx_t size) const { + void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { r_assert(size == seq_idx[id].size); - if (fwd) { + if (forward) { const idx_t offset(seq_idx[id].offset); for (idx_t i(0); i < size; ++i) { - seq[i] = get_char(offset + i); + seq[i] = get_char(pac, offset + i); } } else { const idx_t offset(seq_idx[id].offset + size - 1); for (idx_t i(0); i < size; ++i) { - seq[i] = 3 - get_char(offset - i); + seq[i] = 3 - get_char(pac, offset - i); } } } diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 03c1be1..af04201 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -37,6 +37,7 @@ static void* reads_correction_func_can(void* const arg) { return NULL; } +// load and sort partition data, assign to threads, start threads static void consensus_one_partition_can(const char* const m4_file_name, const idx_t min_read_id, const idx_t max_read_id, ConsensusThreadData& data) { idx_t nec; ExtensionCandidate* const ec_list(load_partition_data(m4_file_name, nec)); @@ -52,6 +53,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, const id delete[] ec_list; } +// set up output file and thread data, handle restart if needed static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB& reads) { const PartitionFileInfo& p(partition_file_vec[rco.job_index]); std::ostringstream os; @@ -61,8 +63,7 @@ static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector Date: Mon, 25 Mar 2019 12:55:54 -0500 Subject: [PATCH 05/56] changed min/max read id to overlap counts --- src/common/packed_db.cpp | 6 +- src/mecat2cns/main.cpp | 4 +- src/mecat2cns/overlaps_partition.cpp | 148 ++++++++++--------------- src/mecat2cns/overlaps_partition.h | 25 ++--- src/mecat2cns/overlaps_store.h | 16 +-- src/mecat2cns/reads_correction_can.cpp | 20 ++-- src/mecat2cns/reads_correction_m4.cpp | 20 ++-- 7 files changed, 99 insertions(+), 140 deletions(-) diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp index e83fa40..deceba5 100644 --- a/src/common/packed_db.cpp +++ b/src/common/packed_db.cpp @@ -81,7 +81,7 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr std::ofstream iout; open_fstream(iout, n.c_str(), std::ios::out); Sequence read; - unsigned int rand_char(-1); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion idx_t count(0), tsize(0); for (;;) { idx_t rsize(fr.read_one_seq(read)); @@ -128,7 +128,7 @@ void PackedDB::add_one_seq(const Sequence& seq) { } const Sequence::str_t& org_seq(seq.sequence()); const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion for (idx_t i(0); i < si.size; ++i, ++db_size) { const u1_t c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); @@ -152,7 +152,7 @@ void PackedDB::add_one_seq(const char* const seq, const idx_t size) { max_db_size = new_size; } const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion for (idx_t i(0); i < si.size; ++i, ++db_size) { const u1_t c(table[static_cast(seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index b7f8cb7..66aa5ca 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include // string #include #include // ... unlink() #include // S_IRUSR, S_IXUSR @@ -124,7 +124,7 @@ int main(int argc, char** argv) { // get number of partitions std::string idx_file_name; generate_partition_index_file_name(rco.m4, idx_file_name); - std::vector partition_file_vec; + std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); std::list results; for (size_t i(0); i != partition_file_vec.size(); ++i) { diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index bb359c1..a2e960e 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -10,62 +10,45 @@ #define error_and_exit(msg) { std::cerr << msg << "\n"; abort(); } -inline bool -check_m4record_mapping_range(const M4Record& m4, const double min_cov_ratio) -{ - const idx_t qm = m4qend(m4) - m4qoff(m4); - const idx_t qs = m4qsize(m4) * min_cov_ratio; - const idx_t sm = m4send(m4) - m4soff(m4); - const idx_t ss = m4ssize(m4) * min_cov_ratio; - return qm >= qs || sm >= ss; +inline static bool query_is_contained(const M4Record& m4, const double min_cov_ratio) { + return m4qend(m4) - m4qoff(m4) >= m4qsize(m4) * min_cov_ratio; } -inline bool -query_is_contained(const M4Record& m4, const double min_cov_ratio) -{ - const idx_t qm = m4qend(m4) - m4qoff(m4); - const idx_t qs = m4qsize(m4) * min_cov_ratio; - return qm >= qs; +inline static bool subject_is_contained(const M4Record& m4, const double min_cov_ratio) { + return m4send(m4) - m4soff(m4) >= m4ssize(m4) * min_cov_ratio; } -inline bool -subject_is_contained(const M4Record& m4, const double min_cov_ratio) -{ - const idx_t sm = m4send(m4) - m4soff(m4); - const idx_t ss = m4ssize(m4) * min_cov_ratio; - return sm >= ss; +inline static bool check_m4record_mapping_range(const M4Record& m4, const double min_cov_ratio) { + return query_is_contained(m4, min_cov_ratio) || subject_is_contained(m4, min_cov_ratio); } -void -get_qualified_m4record_counts(const char* m4_file_name, const double min_cov_ratio, idx_t& num_qualified_records, idx_t& num_reads) -{ - std::ifstream in; - open_fstream(in, m4_file_name, std::ios::in); - num_qualified_records = 0; - num_reads = -1; - idx_t num_records = 0; - M4Record m4; +static void get_qualified_m4record_counts(const char* const m4_file_name, const double min_cov_ratio, idx_t& num_qualified_records, idx_t& num_reads) { + std::ifstream in; + open_fstream(in, m4_file_name, std::ios::in); + num_qualified_records = 0; + num_reads = -1; + idx_t num_records(0); + M4Record m4; m4qext(m4) = m4sext(m4) = INVALID_IDX; - while (in >> m4) - { - if (m4qext(m4) == INVALID_IDX || m4sext(m4) == INVALID_IDX) - { + while (in >> m4) { + if (m4qext(m4) == INVALID_IDX || m4sext(m4) == INVALID_IDX) { ERROR("no gapped start position is provided, please make sure that you have run \'meap_pairwise\' with option \'-g 1\'"); } - ++num_records; - if (check_m4record_mapping_range(m4, min_cov_ratio)) ++num_qualified_records; - num_reads = std::max(num_reads, m4qid(m4)); - num_reads = std::max(num_reads, m4sid(m4)); - } - close_fstream(in); - - LOG(stderr, "there are %d overlaps, %d are qualified.", (int)num_records, (int)num_qualified_records); - ++num_reads; + ++num_records; + if (check_m4record_mapping_range(m4, min_cov_ratio)) { + ++num_qualified_records; + } + num_reads = std::max(num_reads, m4qid(m4)); + num_reads = std::max(num_reads, m4sid(m4)); + } + close_fstream(in); + LOG(stderr, "there are %ld overlaps, %ld are qualified.", num_records, num_qualified_records); + ++num_reads; } void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const idx_t num_reads, std::set& repeat_reads) { - const int MaxContained = 100; - // used to imcrement to a max of MaxContained + const int MaxContained(100); + // used to increment to a max of MaxContained char cnt_table[MaxContained + 1]; for (int i = 0; i < MaxContained; ++i) { cnt_table[i] = i + 1; @@ -80,48 +63,43 @@ void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio if (query_is_contained(m4, min_cov_ratio)) { const idx_t qid = m4qid(m4); // increments count - cnts[qid] = cnt_table[int(cnts[qid])]; + cnts[qid] = cnt_table[static_cast(cnts[qid])]; } if (subject_is_contained(m4, min_cov_ratio)) { const idx_t sid = m4sid(m4); // increments count - cnts[sid] = cnt_table[int(cnts[sid])]; + cnts[sid] = cnt_table[static_cast(cnts[sid])]; } } close_fstream(in); - for(idx_t i = 0; i < num_reads; ++i) { - if (cnts[i] >= MaxContained) { + for (idx_t i(0); i < num_reads; ++i) { + if (cnts[i] == MaxContained) { std::cerr << "repeat read " << i << "\n"; repeat_reads.insert(i); } } - LOG(stderr, "number of repeat reads: %d", (int)repeat_reads.size()); + LOG(stderr, "number of repeat reads: %lu", repeat_reads.size()); } -void generate_partition_index_file_name(const char* m4_file_name, std::string& ret) { - ret = m4_file_name; - ret += ".partition_files"; +void generate_partition_index_file_name(const char* const m4_file_name, std::string& ret) { + ret = m4_file_name; + ret += ".partition_files"; } -void -generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret) -{ - ret = m4_file_name; - ret += ".part"; - std::ostringstream os; - os << part; - ret += os.str(); +void generate_partition_file_name(const char* const m4_file_name, const idx_t part, std::string& ret) { + ret = m4_file_name; + ret += ".part"; + std::ostringstream os; + os << part; + ret += os.str(); } -idx_t -get_num_reads(const char* candidates_file) -{ +static idx_t get_num_reads(const char* const candidates_file) { std::ifstream in; open_fstream(in, candidates_file, std::ios::in); ExtensionCandidate ec; - int max_id = -1; - while (in >> ec) - { + int max_id(-1); + while (in >> ec) { max_id = std::max(ec.qid, max_id); max_id = std::max(ec.sid, max_id); } @@ -192,13 +170,13 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m } // not set by >> ec.qoff = ec.soff = ec.qend = ec.send = 0; - if (ec.qid >= L && ec.qid < R) { + if (L <= ec.qid && ec.qid < R) { normalise_candidate(ec, nec, false); if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { prw.checkpoint(in.tellg()); } } - if (ec.sid >= L && ec.sid < R) { + if (L <= ec.sid && ec.sid < R) { normalise_candidate(ec, nec, true); if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { prw.checkpoint(in.tellg()); @@ -206,11 +184,11 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m } } for (int k(0); k < nf; ++k) { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) { + if (prw.counts[k] == 0) { continue; } - idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; - fprintf(stderr, "%s contains reads %ld --- %ld\n", prw.file_names[k].c_str(), long(prw.min_seq_ids[k]), long(prw.max_seq_ids[k])); + idx_file << prw.file_names[k] << "\n"; + fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); } prw.CloseFiles(); } @@ -271,9 +249,9 @@ partition_m4records(const char* m4_file_name, const double min_cov_ratio, const for (int k = 0; k < nf; ++k) { - if (prw.max_seq_ids[k] == std::numeric_limits::min()) continue; - idx_file << prw.file_names[k] << "\t" << prw.min_seq_ids[k] << "\t" << prw.max_seq_ids[k] << "\n"; - fprintf(stderr, "%s contains reads %d --- %d\n", prw.file_names[k].c_str(), (int)prw.min_seq_ids[k], (int)prw.max_seq_ids[k]); + if (prw.counts[k] == 0) continue; + idx_file << prw.file_names[k] << "\n"; + fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); } prw.CloseFiles(); @@ -281,17 +259,13 @@ partition_m4records(const char* m4_file_name, const double min_cov_ratio, const close_fstream(idx_file); } -void -load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec) -{ - file_info_vec.clear(); - std::ifstream in; - open_fstream(in, idx_file_name, std::ios::in); - PartitionFileInfo pfi; - while (in >> pfi.file_name) - { - in >> pfi.min_seq_id >> pfi.max_seq_id; - file_info_vec.push_back(pfi); - } - close_fstream(in); +void load_partition_files_info(const char* const idx_file_name, std::vector& file_info_vec) { + file_info_vec.clear(); + std::ifstream in; + open_fstream(in, idx_file_name, std::ios::in); + std::string line; + while (in >> line) { + file_info_vec.push_back(line); + } + close_fstream(in); } diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index c74c936..607bff3 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -1,30 +1,19 @@ #ifndef OVERLAPS_PARTITION_H #define OVERLAPS_PARTITION_H -#include +#include // vector<> +#include // string #include "../common/alignment.h" -void -generate_partition_index_file_name(const char* m4_file_name, std::string& ret); +void generate_partition_index_file_name(const char* m4_file_name, std::string& ret); -void -generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret); +void generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret); -void -partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t batch_size, int min_read_size, int num_files); +void partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t batch_size, int min_read_size, int num_files); -void -partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); +void partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); -struct PartitionFileInfo -{ - std::string file_name; - idx_t min_seq_id; - idx_t max_seq_id; -}; - -void -load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); +void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 30e9006..9ed5e6e 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -22,8 +22,7 @@ template class PartitionResultsWriter { PODArray* results; // can't use vector<>, causes memory corruption std::ofstream* files; // can't use vector<>, non-copyable std::vector file_names; - std::vector min_seq_ids; - std::vector max_seq_ids; + std::vector counts; public: // can't make kNumFiles static, as sysconf() is run-time only; // leave room for stdin, stdout, stderr, a few others @@ -63,8 +62,7 @@ template class PartitionResultsWriter { delete[] results; delete[] files; file_names.clear(); - min_seq_ids.clear(); - max_seq_ids.clear(); + counts.clear(); kStoreSize = num_open_files = 0; results = 0; files = 0; @@ -80,8 +78,7 @@ template class PartitionResultsWriter { unlink(std::string(done_file_ + ".ckpt").c_str()); } int WriteOneResult(const int i, const idx_t seq_id, const T& r) { - min_seq_ids[i] = std::min(min_seq_ids[i], seq_id); - max_seq_ids[i] = std::max(max_seq_ids[i], seq_id); + ++counts[i]; results[i].push_back(r); if (results[i].size() == kStoreSize) { write_buffer_to_disk(i); @@ -108,7 +105,7 @@ template class PartitionResultsWriter { allocate_data(prefix, fng, 1); for (int i(0); i < num_open_files; ++i) { off_t file_pos; - in >> file_pos >> min_seq_ids[i] >> max_seq_ids[i]; + in >> file_pos >> counts[i]; if (!in) { ERROR("Read error while restore checkpoint from %s (%d)", ckpt_file_.c_str(), i); } @@ -138,7 +135,7 @@ template class PartitionResultsWriter { results[i].clear(); } files[i].flush(); - out << off_t(files[i].tellp()) << " " << min_seq_ids[i] << " " << max_seq_ids[i] << "\n"; + out << off_t(files[i].tellp()) << " " << counts[i] << "\n"; if (!out) { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); return; @@ -170,8 +167,7 @@ template class PartitionResultsWriter { kStoreSize = (1 << 30) / sizeof(T) / num_open_files; results = new PODArray[num_open_files]; file_names.assign(num_open_files, ""); - min_seq_ids.assign(num_open_files, std::numeric_limits::max()); - max_seq_ids.assign(num_open_files, std::numeric_limits::min()); + counts.assign(num_open_files, 0); files = new std::ofstream[num_open_files]; for (int i(0); i < num_open_files; ++i) { fng(prefix.c_str(), i + batch_start_, file_names[i]); diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index af04201..ce595bd 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -1,7 +1,7 @@ #include "reads_correction_can.h" #include -#include +#include // string #include #include "MECAT_AlnGraphBoost.H" @@ -38,7 +38,7 @@ static void* reads_correction_func_can(void* const arg) { } // load and sort partition data, assign to threads, start threads -static void consensus_one_partition_can(const char* const m4_file_name, const idx_t min_read_id, const idx_t max_read_id, ConsensusThreadData& data) { +static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { idx_t nec; ExtensionCandidate* const ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); @@ -54,8 +54,8 @@ static void consensus_one_partition_can(const char* const m4_file_name, const id } // set up output file and thread data, handle restart if needed -static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB& reads) { - const PartitionFileInfo& p(partition_file_vec[rco.job_index]); +static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB& reads) { + const std::string& p(partition_file_vec[rco.job_index]); std::ostringstream os; os << rco.corrected_reads << "." << rco.job_index; const std::string results_file(os.str()); @@ -72,9 +72,9 @@ static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector partition_file_vec; + std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; reads.load_fasta_db(rco.reads); @@ -109,10 +109,10 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { char process_info[1024]; const int job_end(partition_file_vec.size()); for (; rco.job_index < job_end; ++rco.job_index) { - const PartitionFileInfo& p(partition_file_vec[rco.job_index]); - sprintf(process_info, "processing %s", p.file_name.c_str()); + const std::string& p(partition_file_vec[rco.job_index]); + sprintf(process_info, "processing %s", p.c_str()); DynamicTimer dtimer(process_info); - consensus_one_partition_can(p.file_name.c_str(), p.min_seq_id, p.max_seq_id, data); + consensus_one_partition_can(p.c_str(), data); } close_fstream(out); assert(rename(tmp_file.c_str(), rco.corrected_reads) == 0); diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index 2dc4a03..369f10e 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -1,7 +1,7 @@ #include "reads_correction_m4.h" #include -#include +#include // string #include #include "mecat_correction.h" @@ -50,7 +50,7 @@ void* reads_correction_func_m4(void* arg) { return NULL; } -void consensus_one_partition_m4(const char* m4_file_name, const idx_t min_read_id, const idx_t max_read_id, ReadsCorrectionOptions& rco, PackedDB& reads, std::ostream& out) { +void consensus_one_partition_m4(const char* m4_file_name, ReadsCorrectionOptions& rco, PackedDB& reads, std::ostream& out) { idx_t nec; ExtensionCandidate* ec_list = load_partition_data(m4_file_name, nec); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); @@ -74,18 +74,18 @@ void consensus_one_partition_m4(const char* m4_file_name, const idx_t min_read_i delete[] ec_list; } -static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB &reads) { +static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector &partition_file_vec, PackedDB &reads) { std::ostringstream os; os << rco.corrected_reads << "." << rco.job_index; std::string results_file = os.str(); std::string working_file = results_file + ".working"; std::ofstream out; open_fstream(out, working_file.c_str(), std::ios::out); - const PartitionFileInfo &p = partition_file_vec[rco.job_index]; + const std::string &p = partition_file_vec[rco.job_index]; char process_info[1024]; - sprintf(process_info, "processing %s", p.file_name.c_str()); + sprintf(process_info, "processing %s", p.c_str()); DynamicTimer dtimer(process_info); - consensus_one_partition_m4(p.file_name.c_str(), p.min_seq_id, p.max_seq_id, rco, reads, out); + consensus_one_partition_m4(p.c_str(), rco, reads, out); assert(rename(working_file.c_str(), results_file.c_str()) == 0); return 0; } @@ -93,7 +93,7 @@ static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector partition_file_vec; + std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; reads.load_fasta_db(rco.reads); @@ -103,11 +103,11 @@ int reads_correction_m4(ReadsCorrectionOptions& rco) { std::ofstream out; open_fstream(out, rco.corrected_reads, std::ios::out); char process_info[1024]; - for (std::vector::iterator iter = partition_file_vec.begin(); iter != partition_file_vec.end(); ++iter) + for (std::vector::iterator iter = partition_file_vec.begin(); iter != partition_file_vec.end(); ++iter) { - sprintf(process_info, "processing %s", iter->file_name.c_str()); + sprintf(process_info, "processing %s", iter->c_str()); DynamicTimer dtimer(process_info); - consensus_one_partition_m4(iter->file_name.c_str(), iter->min_seq_id, iter->max_seq_id, rco, reads, out); + consensus_one_partition_m4(iter->c_str(), rco, reads, out); } return 0; From 48db467b3e3e768b92a708eea4e6672d6f62963f Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 25 Mar 2019 13:53:28 -0500 Subject: [PATCH 06/56] minor cleanup --- src/common/fasta_reader.h | 2 +- src/mecat2cns/overlaps_partition.cpp | 139 ++++++++++++--------------- 2 files changed, 65 insertions(+), 76 deletions(-) diff --git a/src/common/fasta_reader.h b/src/common/fasta_reader.h index 8b43d12..736853b 100644 --- a/src/common/fasta_reader.h +++ b/src/common/fasta_reader.h @@ -9,7 +9,7 @@ class FastaReader { typedef Sequence::str_t str_t; typedef BufferLineReader::OneDataLine OneDataLine; public: - FastaReader(const char* fasta_file_name) : m_Reader(fasta_file_name), encode_table(get_dna_encode_table()) { } + explicit FastaReader(const char* const fasta_file_name) : m_Reader(fasta_file_name), encode_table(get_dna_encode_table()) { } idx_t read_one_seq(Sequence& seq); std::streampos tellg() const { return m_Reader.tellg(); diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index a2e960e..620f238 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -22,12 +22,10 @@ inline static bool check_m4record_mapping_range(const M4Record& m4, const double return query_is_contained(m4, min_cov_ratio) || subject_is_contained(m4, min_cov_ratio); } -static void get_qualified_m4record_counts(const char* const m4_file_name, const double min_cov_ratio, idx_t& num_qualified_records, idx_t& num_reads) { +static idx_t get_qualified_m4record_counts(const char* const m4_file_name, const double min_cov_ratio) { std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); - num_qualified_records = 0; - num_reads = -1; - idx_t num_records(0); + idx_t num_reads(-1), num_records(0), num_qualified_records(0); M4Record m4; m4qext(m4) = m4sext(m4) = INVALID_IDX; while (in >> m4) { @@ -43,18 +41,20 @@ static void get_qualified_m4record_counts(const char* const m4_file_name, const } close_fstream(in); LOG(stderr, "there are %ld overlaps, %ld are qualified.", num_records, num_qualified_records); - ++num_reads; + return num_reads + 1; } -void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const idx_t num_reads, std::set& repeat_reads) { - const int MaxContained(100); - // used to increment to a max of MaxContained - char cnt_table[MaxContained + 1]; - for (int i = 0; i < MaxContained; ++i) { - cnt_table[i] = i + 1; +// not in use at the moment +#if 0 +static void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const idx_t num_reads, std::set& repeat_reads) { + const int max_contained(100); + // used to increment to a max of max_contained + char count_table[max_contained + 1]; + for (int i(0); i < max_contained; ++i) { + count_table[i] = i + 1; } - cnt_table[MaxContained] = MaxContained; - std::vector cnts(num_reads, 0); + count_table[max_contained] = max_contained; + std::vector counts(num_reads, 0); std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); M4Record m4; @@ -62,24 +62,25 @@ void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio while (in >> m4) { if (query_is_contained(m4, min_cov_ratio)) { const idx_t qid = m4qid(m4); - // increments count - cnts[qid] = cnt_table[static_cast(cnts[qid])]; + // increments count up to max_contained + counts[qid] = count_table[static_cast(counts[qid])]; } if (subject_is_contained(m4, min_cov_ratio)) { const idx_t sid = m4sid(m4); - // increments count - cnts[sid] = cnt_table[static_cast(cnts[sid])]; + // increments count up to max_contained + counts[sid] = count_table[static_cast(counts[sid])]; } } close_fstream(in); for (idx_t i(0); i < num_reads; ++i) { - if (cnts[i] == MaxContained) { + if (counts[i] == max_contained) { std::cerr << "repeat read " << i << "\n"; repeat_reads.insert(i); } } LOG(stderr, "number of repeat reads: %lu", repeat_reads.size()); } +#endif void generate_partition_index_file_name(const char* const m4_file_name, std::string& ret) { ret = m4_file_name; @@ -184,11 +185,10 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m } } for (int k(0); k < nf; ++k) { - if (prw.counts[k] == 0) { - continue; - } - idx_file << prw.file_names[k] << "\n"; fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); + if (prw.counts[k] != 0) { + idx_file << prw.file_names[k] << "\n"; + } } prw.CloseFiles(); } @@ -196,67 +196,56 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m prw.finalize(); } -void -partition_m4records(const char* m4_file_name, const double min_cov_ratio, const idx_t batch_size, const int min_read_size, const int num_files) -{ +void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const idx_t batch_size, const int min_read_size, const int num_files) { DynamicTimer dtimer(__func__); - - idx_t num_reads, num_qualified_records; - get_qualified_m4record_counts(m4_file_name, min_cov_ratio, num_qualified_records, num_reads); + idx_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); std::set repeat_reads; //get_repeat_reads(m4_file_name, min_cov_ratio, num_reads, repeat_reads); - const idx_t num_batches = (num_reads + batch_size - 1) / batch_size; - std::string idx_file_name; - generate_partition_index_file_name(m4_file_name, idx_file_name); - std::ofstream idx_file; - open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - - M4Record m4, nm4; + const idx_t num_batches((num_reads + batch_size - 1) / batch_size); + std::string idx_file_name; + generate_partition_index_file_name(m4_file_name, idx_file_name); + std::ofstream idx_file; + open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); + M4Record m4, nm4; ExtensionCandidate ec; - PartitionResultsWriter prw(num_files); - for (idx_t i = 0; i < num_batches; i += prw.kNumFiles) - { - const idx_t sfid = i; - const idx_t efid = std::min(sfid + prw.kNumFiles, num_batches); - const int nf = efid - sfid; - const idx_t L = batch_size * sfid; - const idx_t R = batch_size * efid; - std::ifstream in; - open_fstream(in, m4_file_name, std::ios::in); + PartitionResultsWriter prw(num_files); + for (idx_t i(0); i < num_batches; i += prw.kNumFiles) { + const idx_t sfid(i); + const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + const int nf(efid - sfid); + const idx_t L(batch_size * sfid); + const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + std::ifstream in; + open_fstream(in, m4_file_name, std::ios::in); prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name, "partition.done"); - - while (in >> m4) - { - if (m4qsize(m4) < min_read_size || m4ssize(m4) < min_read_size) continue; - if (!check_m4record_mapping_range(m4, min_cov_ratio)) continue; - if (repeat_reads.find(m4qid(m4)) != repeat_reads.end() - || - repeat_reads.find(m4sid(m4)) != repeat_reads.end()) continue; - - if (m4qid(m4) >= L && m4qid(m4) < R) - { - normalize_m4record(m4, false, nm4); + while (in >> m4) { + if (m4qsize(m4) < min_read_size || m4ssize(m4) < min_read_size) { + continue; + } else if (!check_m4record_mapping_range(m4, min_cov_ratio)) { + continue; + } else if (repeat_reads.find(m4qid(m4)) != repeat_reads.end() || repeat_reads.find(m4sid(m4)) != repeat_reads.end()) { + continue; + } + if (m4qid(m4) >= L && m4qid(m4) < R) { + normalize_m4record(m4, false, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4qid(m4) - L) / batch_size, m4qid(m4), ec); - } - if (m4sid(m4) >= L && m4sid(m4) < R) - { - normalize_m4record(m4, true, nm4); + prw.WriteOneResult((m4qid(m4) - L) / batch_size, m4qid(m4), ec); + } + if (m4sid(m4) >= L && m4sid(m4) < R) { + normalize_m4record(m4, true, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4sid(m4) - L) / batch_size, m4sid(m4), ec); - } - } - - for (int k = 0; k < nf; ++k) - { - if (prw.counts[k] == 0) continue; - idx_file << prw.file_names[k] << "\n"; + prw.WriteOneResult((m4sid(m4) - L) / batch_size, m4sid(m4), ec); + } + } + for (int k(0); k < nf; ++k) { fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); - } - - prw.CloseFiles(); - } - close_fstream(idx_file); + if (prw.counts[k] != 0) { + idx_file << prw.file_names[k] << "\n"; + } + } + prw.CloseFiles(); + } + close_fstream(idx_file); } void load_partition_files_info(const char* const idx_file_name, std::vector& file_info_vec) { From c0c3cd93fe4c1719c828a31d2ffb00c590892359 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 26 Mar 2019 13:35:04 -0500 Subject: [PATCH 07/56] cleaned up packdb commented out methods that weren't used anywhere --- src/common/packed_db.cpp | 113 +++++++++++++++-------------- src/common/packed_db.h | 130 ++++++++++++++++++---------------- src/common/split_database.cpp | 31 ++++---- 3 files changed, 143 insertions(+), 131 deletions(-) diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp index deceba5..6ab105f 100644 --- a/src/common/packed_db.cpp +++ b/src/common/packed_db.cpp @@ -2,10 +2,49 @@ #include #include +#include // bzero() #include "defs.h" #include "fasta_reader.h" +void PackedDB::add_one_seq(const Sequence& seq) { + SeqIndex si; + si.file_offset = -1; + si.memory_offset = db_size; + si.size = seq.size(); + seq_idx.push_back(si); + const idx_t size((si.size + 3) / 4); + const idx_t needed_size(db_size + size); + if (max_db_size < needed_size) { + idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + for (; new_size < needed_size; new_size *= 2) { } + u1_t* new_pac(NULL); + safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + memcpy(new_pac, pac, (db_size + 3) / 4); + safe_free(pac); + pac = new_pac; + max_db_size = new_size; + } + const Sequence::str_t& org_seq(seq.sequence()); + const u1_t* const table(get_dna_encode_table()); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + for (idx_t i(0); i < si.size; ++i, ++db_size) { + const u1_t c(table[static_cast(org_seq[i])]); + set_char(pac, db_size, c < 4 ? c : ++rand_char & 3); + } +} + +void PackedDB::load_fasta_db(const char* const dbname) { + DynamicTimer dtimer(__func__); + FastaReader freader(dbname); + Sequence seq; + while (freader.read_one_seq(seq) != -1) { + add_one_seq(seq); + } +} + +// these routines aren't used anywhere +/* void PackedDB::dump_pac(const u1_t* const p, const idx_t size, const char* const path) { std::ofstream out; open_fstream(out, path, std::ios::out | std::ios::binary); @@ -34,7 +73,7 @@ void PackedDB::dump_idx(const PODArray& idx_list, const char* const pa open_fstream(out, path, std::ios::out); const idx_t n(idx_list.size()); for (idx_t i(0); i < n; ++i) { - out << idx_list[i].offset << "\t" << idx_list[i].size << "\n"; + out << idx_list[i].memory_offset << "\t" << idx_list[i].size << "\n"; } close_fstream(out); } @@ -44,7 +83,8 @@ void PackedDB::load_idx(const char* const path, PODArray& idx_list) { std::ifstream in; open_fstream(in, path, std::ios::in); SeqIndex si; - while (in >> si.offset >> si.size) { + si.file_offset = -1; + while (in >> si.memory_offset >> si.size) { idx_list.push_back(si); } close_fstream(in); @@ -68,21 +108,21 @@ void PackedDB::load_packed_db(const char* const path) { } void PackedDB::pack_fasta_db(const char* const path, const char* const output_prefix, const idx_t min_size) { - u1_t* buffer; - safe_malloc(buffer, u1_t, MAX_SEQ_SIZE); + u1_t buffer[MAX_SEQ_SIZE]; const u1_t* const et(get_dna_encode_table()); FastaReader fr(path); - std::string n; - generate_pac_name(output_prefix, n); + std::string filename; + generate_pac_name(output_prefix, filename); std::ofstream pout; - open_fstream(pout, n.c_str(), std::ios::out | std::ios::binary); + open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); std::streambuf* psb(pout.rdbuf()); - generate_idx_name(output_prefix, n); + generate_idx_name(output_prefix, filename); std::ofstream iout; - open_fstream(iout, n.c_str(), std::ios::out); + open_fstream(iout, filename.c_str(), std::ios::out); Sequence read; unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion idx_t count(0), tsize(0); + off_t file_offset(0); for (;;) { idx_t rsize(fr.read_one_seq(read)); if (rsize == -1) { @@ -92,53 +132,29 @@ void PackedDB::pack_fasta_db(const char* const path, const char* const output_pr } Sequence::str_t& s(read.sequence()); // set_char uses | to set bits, so clear first - memset(buffer, 0, (rsize + 3) / 4); + bzero(buffer, (rsize + 3) / 4); for (idx_t i(0); i < rsize; ++i) { const u1_t c(et[static_cast(s[i])]); set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } - iout << tsize << "\t" << rsize << "\n"; + iout << file_offset << "\t" << rsize << "\n"; + tsize += rsize; rsize = (rsize + 3) / 4; sb_write(psb, buffer, rsize); - tsize += rsize * 4; + file_offset += rsize * 4; ++count; } - sb_write(psb, &tsize, sizeof(idx_t)); + sb_write(psb, &file_offset, sizeof(idx_t)); close_fstream(pout); close_fstream(iout); - safe_free(buffer); - LOG(stdout, "pack %lld reads, totally %lld residues", (long long)count, (long long)tsize); -} - -void PackedDB::add_one_seq(const Sequence& seq) { - SeqIndex si; - si.size = seq.size(); - si.offset = db_size; - seq_idx.push_back(si); - const idx_t needed_size(db_size + si.size); - if (max_db_size < needed_size) { - idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); - for (; new_size < needed_size; new_size *= 2) { } - u1_t* new_pac(NULL); - safe_calloc(new_pac, u1_t, (new_size + 3) / 4); - memcpy(new_pac, pac, (db_size + 3) / 4); - safe_free(pac); - pac = new_pac; - max_db_size = new_size; - } - const Sequence::str_t& org_seq(seq.sequence()); - const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < si.size; ++i, ++db_size) { - const u1_t c(table[static_cast(org_seq[i])]); - set_char(db_size, c < 4 ? c : ++rand_char & 3); - } + LOG(stdout, "pack %ld reads, totally %ld residues", count, tsize); } void PackedDB::add_one_seq(const char* const seq, const idx_t size) { SeqIndex si; + si.file_offset = -1; + si.memory_offset = db_size; si.size = size; - si.offset = db_size; seq_idx.push_back(si); const idx_t needed_size(db_size + si.size); if (max_db_size < needed_size) { @@ -159,15 +175,7 @@ void PackedDB::add_one_seq(const char* const seq, const idx_t size) { } } -void PackedDB::load_fasta_db(const char* const dbname) { - DynamicTimer dtimer(__func__); - FastaReader freader(dbname); - Sequence seq; - while (freader.read_one_seq(seq) != -1) { - add_one_seq(seq); - } -} - +// convert offset into read id idx_t PackedDB::offset_to_rid(const idx_t offset) const { if (offset >= db_size) { return -1; @@ -175,11 +183,11 @@ idx_t PackedDB::offset_to_rid(const idx_t offset) const { idx_t left(0), mid(0), right(seq_idx.size()); while (left < right) { mid = (left + right) >> 1; - if (offset < seq_idx[mid].offset) { + if (offset < seq_idx[mid].memory_offset) { right = mid; } else if (mid == seq_idx.size() - 1) { break; - } else if (offset < seq_idx[mid + 1].offset) { + } else if (offset < seq_idx[mid + 1].memory_offset) { break; } else { left = mid + 1; @@ -187,3 +195,4 @@ idx_t PackedDB::offset_to_rid(const idx_t offset) const { } return mid; } +*/ diff --git a/src/common/packed_db.h b/src/common/packed_db.h index 8859274..1b4e9e1 100644 --- a/src/common/packed_db.h +++ b/src/common/packed_db.h @@ -5,34 +5,87 @@ #include "sequence.h" class PackedDB { - public: + private: struct SeqIndex { - idx_t offset, size; + off_t file_offset; + idx_t memory_offset, size; }; public: PackedDB() : pac(NULL), db_size(0), max_db_size(0) { } ~PackedDB() { - destroy(); - } - void reserve(const idx_t size) { - destroy(); - max_db_size = size; - safe_calloc(pac, u1_t, (max_db_size + 3) / 4); + if (pac) { + safe_free(pac); + } } + void load_fasta_db(const char* fasta); void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { r_assert(size == seq_idx[id].size); if (forward) { - const idx_t offset(seq_idx[id].offset); + const idx_t offset(seq_idx[id].memory_offset); for (idx_t i(0); i < size; ++i) { seq[i] = get_char(pac, offset + i); } } else { - const idx_t offset(seq_idx[id].offset + size - 1); + const idx_t offset(seq_idx[id].memory_offset + size - 1); for (idx_t i(0); i < size; ++i) { seq[i] = 3 - get_char(pac, offset - i); } } } + static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { + s.reserve(s.size() + end_i - i); + for (; i < end_i; ++i) { + s += "ACGT"[static_cast(seq[i])]; + } + } + static void decode_sequence(char* const seq, const idx_t seq_size) { + for (idx_t i(0); i < seq_size; ++i) { + const u1_t c(seq[i]); + r_assert(c < 4); // c is unsigned, so always >= 0 + seq[i] = "ACGT"[c]; + } + } + static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { + // use ~x instead of 3 - x for speed + p[idx >> 2] |= c << ((~idx & 3) << 1); + } + static u1_t get_char(const u1_t* const p, const idx_t idx) { + return p[idx >> 2] >> ((~idx & 3) << 1) & 3; + } + static void generate_pac_name(const char* const prefix, std::string& ret) { + ret = prefix; + ret += ".pac"; + } + static void generate_idx_name(const char* const prefix, std::string& ret) { + ret = prefix; + ret += ".idx"; + } + private: + void add_one_seq(const Sequence& seq); + private: + u1_t* pac; + idx_t db_size; + idx_t max_db_size; + PODArray seq_idx; + +/* // these routines aren't used anywhere + void destroy() { + if (pac) { + safe_free(pac); + } + seq_idx.clear(); + max_db_size = db_size = 0; + } + void clear() { + seq_idx.clear(); + db_size = 0; + memset(pac, 0, (max_db_size + 3) / 4); + } + void reserve(const idx_t size) { + destroy(); + max_db_size = size; + safe_calloc(pac, u1_t, (max_db_size + 3) / 4); + } void get_sequence(const idx_t from, const idx_t to, const bool forward, char* const seq) const { if (forward) { idx_t idx(0); @@ -48,35 +101,16 @@ class PackedDB { } void get_sequence(const idx_t rid, const bool forward, char* const seq) const { const SeqIndex& a(seq_idx[rid]); - get_sequence(a.offset, a.offset + a.size, forward, seq); + get_sequence(a.memory_offset, a.memory_offset + a.size, forward, seq); } void get_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { - const idx_t offset(seq_idx[rid].offset); + const idx_t offset(seq_idx[rid].memory_offset); get_sequence(offset + from, offset + to, forward, seq); } - static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { - s.reserve(s.size() + end_i - i); - for (; i < end_i; ++i) { - s += "ACGT"[static_cast(seq[i])]; - } - } - static void decode_sequence(char* const seq, const idx_t seq_size) { - for (idx_t i(0); i < seq_size; ++i) { - const u1_t c(seq[i]); - r_assert(c < 4); // c is unsigned, so always >= 0 - seq[i] = "ACGT"[c]; - } - } void get_decode_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { get_sequence(rid, from, to, forward, seq); decode_sequence(seq, to - from); } - static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { - p[idx >> 2] |= c << ((~idx & 3) << 1); - } - static u1_t get_char(const u1_t* const p, const idx_t idx) { - return p[idx >> 2] >> ((~idx & 3) << 1) & 3; - } void set_char(const idx_t idx, const u1_t c) { set_char(pac, idx, c); } @@ -90,34 +124,13 @@ class PackedDB { return seq_idx.size(); } idx_t seq_offset(const idx_t rid) const { - return seq_idx[rid].offset; + return seq_idx[rid].memory_offset; } idx_t seq_size(const idx_t rid) const { return seq_idx[rid].size; } - void destroy() { - if (pac) { - safe_free(pac); - } - seq_idx.clear(); - db_size = max_db_size = 0; - } - void clear() { - seq_idx.clear(); - db_size = 0; - memset(pac, 0, (max_db_size + 3) / 4); - } - static void generate_pac_name(const char* const prefix, std::string& ret) { - ret = prefix; - ret += ".pac"; - } - static void generate_idx_name(const char* const prefix, std::string& ret) { - ret = prefix; - ret += ".idx"; - } - idx_t offset_to_rid(const idx_t offset) const; - void add_one_seq(const Sequence& seq); void add_one_seq(const char* seq, const idx_t size); + idx_t offset_to_rid(const idx_t offset) const; static void dump_pac(const u1_t* p, idx_t size, const char* path); static u1_t* load_pac(const char* path, idx_t& size); static void dump_idx(const PODArray& idx_list, const char* path); @@ -125,12 +138,9 @@ class PackedDB { void dump_packed_db(const char* path) const; void load_packed_db(const char* path); static void pack_fasta_db(const char* fasta, const char* output_prefix, const idx_t min_size); - void load_fasta_db(const char* fasta); - private: - u1_t* pac; - idx_t db_size; - idx_t max_db_size; - PODArray seq_idx; + +*/ + }; #endif // PACKED_DB_H diff --git a/src/common/split_database.cpp b/src/common/split_database.cpp index f334997..30247c7 100644 --- a/src/common/split_database.cpp +++ b/src/common/split_database.cpp @@ -238,21 +238,14 @@ generate_idx_file_name(const char* wrk_dir, char* idx_file_name) strcat(idx_file_name, "fileindex.txt"); } -void -extract_one_seq(ifstream& pac_file, PackedDB::SeqIndex& si, u1_t* buffer, char* seq) -{ - idx_t offset = si.offset / 4; - idx_t bytes = (si.size + 3) / 4; - pac_file.seekg(offset, ios::beg); - memset(buffer, 0, MAX_SEQ_SIZE); +void extract_one_seq(ifstream& pac_file, const idx_t offset, const idx_t size, u1_t* const buffer, char* const seq) { + const idx_t bytes((size + 3) / 4); + pac_file.seekg(offset / 4, ios::beg); pac_file.read((char*)buffer, bytes); - const char* dt = get_dna_decode_table(); - idx_t i = 0; - for(i = 0; i < si.size; ++i) - { - u1_t c = PackedDB::get_char(buffer, i); - c = dt[c]; - seq[i] = c; + const char* const dt(get_dna_decode_table()); + idx_t i(0); + for (; i < size; ++i) { + seq[i] = dt[PackedDB::get_char(buffer, i)]; } seq[i] = '\0'; } @@ -413,10 +406,10 @@ split_dataset(const char* reads, const char* wrk_dir, int* num_vols) char idx_file_name[PATH_MAX], vol_file_name[PATH_MAX]; generate_idx_file_name(wrk_dir, idx_file_name); FILE* idx_file = fopen(idx_file_name, "w"); - PackedDB::SeqIndex si; - while (in_idx_file >> si.offset >> si.size) + idx_t offset, size; + while (in_idx_file >> offset >> size) { - if (v->curr + si.size + 1 > MCS) + if (v->curr + size + 1 > MCS) { v->start_read_id = rid; rid += v->num_reads; @@ -425,8 +418,8 @@ split_dataset(const char* reads, const char* wrk_dir, int* num_vols) dump_volume(vol_file_name, v); clear_volume_t(v); } - extract_one_seq(pac_file, si, buffer, seq); - add_one_seq(v, seq, si.size); + extract_one_seq(pac_file, offset, size, buffer, seq); + add_one_seq(v, seq, size); ++v->curr; } From 9eca72d62d3d635f4d4f89aa038d0ca63c0e8075 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 1 Apr 2019 00:15:12 -0500 Subject: [PATCH 08/56] added read/candidate ordering for improved throughput in mecat2cns; also moved packed_db into mecat2cns, since that's the only place it's used (the bits that were kinda used in common (lookup_table and split_database) shouldn't have been using it, as they were treating it as subroutines with a fixed interface, not a class --- src/common/defs.h | 2 +- src/common/lookup_table.cpp | 6 +- src/common/packed_db.cpp | 198 -------------- src/common/split_database.cpp | 14 +- src/common/split_database.h | 7 +- src/filter_reads/filter_reads.cpp | 2 +- src/main.mk | 1 - src/mecat2cns/argument.h | 6 +- src/mecat2cns/dw.cpp | 1 + src/mecat2cns/dw.h | 1 - src/mecat2cns/main.cpp | 2 + src/mecat2cns/mecat2cns.mk | 1 + src/mecat2cns/mecat_correction.cpp | 17 +- src/mecat2cns/options.cpp | 19 +- src/mecat2cns/options.h | 1 + src/mecat2cns/packed_db.cpp | 365 +++++++++++++++++++++++++ src/{common => mecat2cns}/packed_db.h | 81 +++--- src/mecat2cns/reads_correction_aux.cpp | 10 +- src/mecat2cns/reads_correction_aux.h | 15 +- src/mecat2cns/reads_correction_can.cpp | 93 ++++++- src/mecat2pw/pw_impl.cpp | 1 - src/mecat2pw/pw_impl.h | 1 - 22 files changed, 564 insertions(+), 280 deletions(-) delete mode 100644 src/common/packed_db.cpp create mode 100644 src/mecat2cns/packed_db.cpp rename src/{common => mecat2cns}/packed_db.h (73%) diff --git a/src/common/defs.h b/src/common/defs.h index b11acf8..4235cc4 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -19,7 +19,7 @@ typedef uint64_t u8_t; typedef i8_t idx_t; typedef u1_t uint1; -typedef idx_t index_t; +//typedef idx_t index_t; // solaris conflict #define INVALID_IDX (-1) diff --git a/src/common/lookup_table.cpp b/src/common/lookup_table.cpp index e4ee913..253efcb 100644 --- a/src/common/lookup_table.cpp +++ b/src/common/lookup_table.cpp @@ -1,5 +1,5 @@ #include "lookup_table.h" -#include "packed_db.h" +#include "split_database.h" #include @@ -41,7 +41,7 @@ fill_ref_index_offsets_func(void* arg) for (j = 0; j < read_size; ++j) { int k = read_start + j; - uint8_t c = PackedDB::get_char(v->data, k); + uint8_t c = GET_CHAR(v->data, k); eit = (eit << 2) | c; assert(eit < index_count); if (j >= kmer_size - 1) @@ -78,7 +78,7 @@ create_ref_index(volume_t* v, int kmer_size, int num_threads) for (int j = 0; j < read_size; ++j) { int k = read_start + j; - uint8_t c = PackedDB::get_char(v->data, k); + uint8_t c = GET_CHAR(v->data, k); assert(c>= 0 && c < 4); eit = (eit << 2) | c; if (j >= kmer_size - 1) diff --git a/src/common/packed_db.cpp b/src/common/packed_db.cpp deleted file mode 100644 index 6ab105f..0000000 --- a/src/common/packed_db.cpp +++ /dev/null @@ -1,198 +0,0 @@ -#include "packed_db.h" - -#include -#include -#include // bzero() - -#include "defs.h" -#include "fasta_reader.h" - -void PackedDB::add_one_seq(const Sequence& seq) { - SeqIndex si; - si.file_offset = -1; - si.memory_offset = db_size; - si.size = seq.size(); - seq_idx.push_back(si); - const idx_t size((si.size + 3) / 4); - const idx_t needed_size(db_size + size); - if (max_db_size < needed_size) { - idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); - for (; new_size < needed_size; new_size *= 2) { } - u1_t* new_pac(NULL); - safe_calloc(new_pac, u1_t, (new_size + 3) / 4); - memcpy(new_pac, pac, (db_size + 3) / 4); - safe_free(pac); - pac = new_pac; - max_db_size = new_size; - } - const Sequence::str_t& org_seq(seq.sequence()); - const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < si.size; ++i, ++db_size) { - const u1_t c(table[static_cast(org_seq[i])]); - set_char(pac, db_size, c < 4 ? c : ++rand_char & 3); - } -} - -void PackedDB::load_fasta_db(const char* const dbname) { - DynamicTimer dtimer(__func__); - FastaReader freader(dbname); - Sequence seq; - while (freader.read_one_seq(seq) != -1) { - add_one_seq(seq); - } -} - -// these routines aren't used anywhere -/* -void PackedDB::dump_pac(const u1_t* const p, const idx_t size, const char* const path) { - std::ofstream out; - open_fstream(out, path, std::ios::out | std::ios::binary); - std::streambuf* sb(out.rdbuf()); - sb_write(sb, p, (size + 3) / 4); - sb_write(sb, &size, sizeof(idx_t)); - close_fstream(out); -} - -u1_t* PackedDB::load_pac(const char* path, idx_t& size) { - std::ifstream in; - open_fstream(in, path, std::ios::in | std::ios::binary); - std::streambuf* sb(in.rdbuf()); - in.seekg(-sizeof(idx_t), std::ios::end); - sb_read(sb, &size, sizeof(idx_t)); - in.seekg(0, std::ios::beg); - u1_t* p; - safe_calloc(p, u1_t, (size + 3) / 4); - sb_read(sb, p, (size + 3) / 4); - close_fstream(in); - return p; -} - -void PackedDB::dump_idx(const PODArray& idx_list, const char* const path) { - std::ofstream out; - open_fstream(out, path, std::ios::out); - const idx_t n(idx_list.size()); - for (idx_t i(0); i < n; ++i) { - out << idx_list[i].memory_offset << "\t" << idx_list[i].size << "\n"; - } - close_fstream(out); -} - -void PackedDB::load_idx(const char* const path, PODArray& idx_list) { - idx_list.clear(); - std::ifstream in; - open_fstream(in, path, std::ios::in); - SeqIndex si; - si.file_offset = -1; - while (in >> si.memory_offset >> si.size) { - idx_list.push_back(si); - } - close_fstream(in); -} - -void PackedDB::dump_packed_db(const char* const path) const { - std::string n; - generate_pac_name(path, n); - dump_pac(pac, db_size, n.c_str()); - generate_idx_name(path, n); - dump_idx(seq_idx, n.c_str()); -} - -void PackedDB::load_packed_db(const char* const path) { - std::string n; - generate_pac_name(path, n); - pac = load_pac(n.c_str(), db_size); - max_db_size = db_size; - generate_idx_name(path, n); - load_idx(n.c_str(), seq_idx); -} - -void PackedDB::pack_fasta_db(const char* const path, const char* const output_prefix, const idx_t min_size) { - u1_t buffer[MAX_SEQ_SIZE]; - const u1_t* const et(get_dna_encode_table()); - FastaReader fr(path); - std::string filename; - generate_pac_name(output_prefix, filename); - std::ofstream pout; - open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); - std::streambuf* psb(pout.rdbuf()); - generate_idx_name(output_prefix, filename); - std::ofstream iout; - open_fstream(iout, filename.c_str(), std::ios::out); - Sequence read; - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - idx_t count(0), tsize(0); - off_t file_offset(0); - for (;;) { - idx_t rsize(fr.read_one_seq(read)); - if (rsize == -1) { - break; - } else if (rsize < min_size) { - continue; - } - Sequence::str_t& s(read.sequence()); - // set_char uses | to set bits, so clear first - bzero(buffer, (rsize + 3) / 4); - for (idx_t i(0); i < rsize; ++i) { - const u1_t c(et[static_cast(s[i])]); - set_char(buffer, i, c < 4 ? c : ++rand_char & 3); - } - iout << file_offset << "\t" << rsize << "\n"; - tsize += rsize; - rsize = (rsize + 3) / 4; - sb_write(psb, buffer, rsize); - file_offset += rsize * 4; - ++count; - } - sb_write(psb, &file_offset, sizeof(idx_t)); - close_fstream(pout); - close_fstream(iout); - LOG(stdout, "pack %ld reads, totally %ld residues", count, tsize); -} - -void PackedDB::add_one_seq(const char* const seq, const idx_t size) { - SeqIndex si; - si.file_offset = -1; - si.memory_offset = db_size; - si.size = size; - seq_idx.push_back(si); - const idx_t needed_size(db_size + si.size); - if (max_db_size < needed_size) { - idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); - for (; new_size < needed_size; new_size *= 2) { } - u1_t* new_pac(NULL); - safe_calloc(new_pac, u1_t, (new_size + 3) / 4); - memcpy(new_pac, pac, (db_size + 3) / 4); - safe_free(pac); - pac = new_pac; - max_db_size = new_size; - } - const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < si.size; ++i, ++db_size) { - const u1_t c(table[static_cast(seq[i])]); - set_char(db_size, c < 4 ? c : ++rand_char & 3); - } -} - -// convert offset into read id -idx_t PackedDB::offset_to_rid(const idx_t offset) const { - if (offset >= db_size) { - return -1; - } - idx_t left(0), mid(0), right(seq_idx.size()); - while (left < right) { - mid = (left + right) >> 1; - if (offset < seq_idx[mid].memory_offset) { - right = mid; - } else if (mid == seq_idx.size() - 1) { - break; - } else if (offset < seq_idx[mid + 1].memory_offset) { - break; - } else { - left = mid + 1; - } - } - return mid; -} -*/ diff --git a/src/common/split_database.cpp b/src/common/split_database.cpp index 30247c7..ffaf86e 100644 --- a/src/common/split_database.cpp +++ b/src/common/split_database.cpp @@ -10,7 +10,6 @@ #include // PATH_MAX #include // unlink() -#include "packed_db.h" #include "fasta_reader.h" #define MSS MAX_SEQ_SIZE @@ -130,7 +129,7 @@ add_one_seq(volume_t* volume, const char* s, const int size) int idx = volume->curr; uint8_t c = s[i]; c = encode_table[c]; - PackedDB::set_char(d, idx, c); + SET_CHAR(d, idx, c); ++volume->curr; } } @@ -145,7 +144,7 @@ extract_one_seq(const volume_t* v, const int id, char* s) for (i = 0; i < size; ++i) { int k = offset + i; - s[i] = PackedDB::get_char(v->data, k); + s[i] = GET_CHAR(v->data, k); } } @@ -245,7 +244,7 @@ void extract_one_seq(ifstream& pac_file, const idx_t offset, const idx_t size, u const char* const dt(get_dna_decode_table()); idx_t i(0); for (; i < size; ++i) { - seq[i] = dt[PackedDB::get_char(buffer, i)]; + seq[i] = dt[GET_CHAR(buffer, i)]; } seq[i] = '\0'; } @@ -389,12 +388,13 @@ int split_raw_dataset(const char* reads, const char* wrk_dir) { void split_dataset(const char* reads, const char* wrk_dir, int* num_vols) { - string name; - PackedDB::generate_idx_name(reads, name); + std::string name(reads); + name += ".idx"; ifstream in_idx_file; open_fstream(in_idx_file, name.c_str(), ios::in); ifstream pac_file; - PackedDB::generate_pac_name(reads, name); + name = reads; + name += ".pac"; open_fstream(pac_file, name.c_str(), ios::in | ios::binary); u1_t* buffer; char* seq; diff --git a/src/common/split_database.h b/src/common/split_database.h index 54fe41e..f27db07 100644 --- a/src/common/split_database.h +++ b/src/common/split_database.h @@ -3,7 +3,8 @@ #include "../common/defs.h" -#define MCS (2140000000L) // max chunk size +// max chunk size +#define MCS (2140000000L) //#define MCS 50000000L typedef struct { @@ -23,6 +24,10 @@ typedef struct { offset_list_t* offset_list; } volume_t; +// moved from PackedDB +#define SET_CHAR(p,i,c) ((p)[(i) >> 2] |= (c) << ((~(i) & 3) << 1)) +#define GET_CHAR(p,i) ((p)[(i) >> 2] >> ((~(i) & 3) << 1) & 3) + volume_t* new_volume_t(int num_reads, int num_bases, int no_allocate = 0); diff --git a/src/filter_reads/filter_reads.cpp b/src/filter_reads/filter_reads.cpp index 8ee14c6..a8dd3da 100644 --- a/src/filter_reads/filter_reads.cpp +++ b/src/filter_reads/filter_reads.cpp @@ -6,7 +6,7 @@ using namespace std; -typedef index_t idx; +typedef idx_t idx; void print_usage(const char* prog) { diff --git a/src/main.mk b/src/main.mk index 3a6b457..2cb52b3 100644 --- a/src/main.mk +++ b/src/main.mk @@ -14,7 +14,6 @@ SOURCES := common/alignment.cpp \ common/fasta_reader.cpp \ common/gapalign.cpp \ common/lookup_table.cpp \ - common/packed_db.cpp \ common/sequence.cpp \ common/split_database.cpp \ common/xdrop_gapalign.cpp diff --git a/src/mecat2cns/argument.h b/src/mecat2cns/argument.h index 828a8b0..a81c086 100644 --- a/src/mecat2cns/argument.h +++ b/src/mecat2cns/argument.h @@ -20,13 +20,13 @@ class Argument class IntegerArgument : public Argument { public: - IntegerArgument(const std::string* an, const std::string* ad, const index_t v) : Argument(an, ad), val(v) {} + IntegerArgument(const std::string* an, const std::string* ad, const idx_t v) : Argument(an, ad), val(v) {} virtual ~IntegerArgument() {} virtual int ProcessArgument(int argc, char** argv); - index_t value() { return val; } + idx_t value() { return val; } private: - index_t val; + idx_t val; }; class DoubleArgument : public Argument diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 03c44bd..c95f13c 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,4 +1,5 @@ #include "dw.h" +#include // memcpy(), memset() namespace ns_banded_sw { diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index a2a844b..927dc9b 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -5,7 +5,6 @@ #include "../common/alignment.h" #include "../common/defs.h" -#include "../common/packed_db.h" namespace ns_banded_sw { diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 66aa5ca..f10ec10 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -2,6 +2,7 @@ #include "reads_correction_m4.h" #include "overlaps_partition.h" #include "options.h" +#include "packed_db.h" // PackedDB #include #include @@ -105,6 +106,7 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { + PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); } else { partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index c268ae8..fb17ae4 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -16,6 +16,7 @@ SOURCES := main.cpp \ reads_correction_aux.cpp \ reads_correction_can.cpp \ reads_correction_m4.cpp \ + packed_db.cpp \ SRC_INCDIRS := . libboost diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index d977bcc..cfd3f35 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -241,6 +241,13 @@ void consensus_worker(const CnsTableItem* const cns_table, uint1* const id_list, } } +static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { + s.reserve(s.size() + end_i - i); + for (; i < end_i; ++i) { + s += "ACGT"[static_cast(seq[i])]; + } +} + // same as consensus_worker, but produces entire read as one entry; // uncorrected sections are just copied as is; @@ -254,9 +261,9 @@ void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const std::vector::const_iterator last_a(eranges.end()); for (; a != end_a; last_a = a++) { if (last_a != end_a) { // add in-between range to cns_result - PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, a->start); + decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, a->start); } else if (a->start > 0) { // add beginning of read - PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), 0, a->start); + decode_and_append_sequence(cns_result.seq, tstr.data(), 0, a->start); } const int begin_i(a->start - 1); const int end_i(a->end); @@ -265,7 +272,7 @@ void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const const idx_t last_end(i != begin_i ? i : a->start); for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } // add low coverage area as-is - PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_end, i); + decode_and_append_sequence(cns_result.seq, tstr.data(), last_end, i); if (i == end_i) { break; } @@ -281,12 +288,12 @@ void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const } } // add uncorrected sequence - PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), start, i); + decode_and_append_sequence(cns_result.seq, tstr.data(), start, i); } } // add end of read if (last_a != end_a) { - PackedDB::decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, tstr.size()); + decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, tstr.size()); } cns_result.range[0] = 0; cns_result.range[1] = cns_result.seq.size(); diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index 6699109..b6ce744 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -30,6 +30,7 @@ static int tech_nanopore = TECH_NANOPORE; static int default_tech = TECH_PACBIO; static int num_partition_files = 0; static int full_reads = 0; +static idx_t read_buffer_size = 0; static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -47,6 +48,7 @@ static const char job_index_n = 'I'; static const char reads_to_correct_n = 'R'; static const char grid_start_delay_n = 'D'; static const char full_reads_n = 'F'; +static const char read_buffer_size_n = 'b'; void print_pacbio_default_options() @@ -118,6 +120,9 @@ make_options(const ConsensusOptions& options) if (options.full_reads) { cmd << " -" << full_reads_n; } + if (options.read_buffer_size) { + cmd << " -" << read_buffer_size_n << " " << options.read_buffer_size; + } cmd << " " << options.m4; cmd << " " << options.reads; cmd << " " << options.corrected_reads; @@ -144,7 +149,8 @@ void print_usage(const char* prog) { << "-" << reads_to_correct_n << " \tnumber of reads to correct [all]\n" << "-" << grid_start_delay_n << " \tseconds to delay between starting grid jobs\n" << "-" << full_reads_n << "\t\toutput full reads, not just the corrected parts\n" - << "-" << usage_n << "\t\tprint usage info.\n" + << "-" << read_buffer_size_n << " \tbytes of memory to buffer reads [no buffer]\n" + << "-" << usage_n << "\t\tprint usage info\n" << "\n" << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; print_pacbio_default_options(); @@ -165,6 +171,7 @@ ConsensusOptions init_consensus_options(const int tech) { t.reads_to_correct = 0; t.grid_start_delay = 0; t.full_reads = full_reads; + t.read_buffer_size = read_buffer_size; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; t.num_threads = num_threads_pacbio; @@ -223,7 +230,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { int opt_char; char err_char; opterr = 0; - while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:F")) != -1) { + while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:")) != -1) { switch (opt_char) { case input_type_n: if (optarg[0] == '0') { @@ -279,6 +286,9 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { case full_reads_n: t.full_reads = 1; break; + case read_buffer_size_n: + t.read_buffer_size = atoll(optarg); + break; case '?': err_char = (char)optopt; fprintf(stderr, "unrecognised option '%c'\n", err_char); @@ -319,6 +329,10 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { std::cerr << "grid start delay must be >= 0\n"; parse_success = false; } + if (t.read_buffer_size < 0) { + std::cerr << "read buffer size must be greater than 0\n"; + parse_success = false; + } if (argc - optind < 3) { return 1; } @@ -338,6 +352,7 @@ print_options(ConsensusOptions& t) if (t.grid_options) std::cout << "grid\t" << t.grid_options << "\n"; if (t.grid_options_split) std::cout << "grid_split\t" << t.grid_options_split << "\n"; if (t.full_reads) std::cout << "full reads\n"; + if (t.read_buffer_size) std::cout << "read_buffer_size\t" << t.read_buffer_size << "\n"; std::cout << "number of threads:\t" << t.num_threads << "\n"; std::cout << "batch size:\t" << t.batch_size << "\n"; std::cout << "mapping ratio:\t" << t.min_mapping_ratio << "\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index 0a7bd7f..f17a3df 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -28,6 +28,7 @@ struct ConsensusOptions int reads_to_correct; int grid_start_delay; int full_reads; + idx_t read_buffer_size; }; void diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp new file mode 100644 index 0000000..49a688f --- /dev/null +++ b/src/mecat2cns/packed_db.cpp @@ -0,0 +1,365 @@ +#include "packed_db.h" + +#include // ifstream, ofstream +#include // set<> +#include // rename() +#include +#include // bzero() +#include // stat(), struct stat +#include // unlink() + +#include "../common/defs.h" +#include "../common/fasta_reader.h" +#include "../common/alignment.h" + +void PackedDB::add_one_seq(const Sequence& seq) { + SeqIndex si; + si.file_offset = -1; + si.memory_offset = db_size; + si.size = seq.size(); + seq_idx.push_back(si); + const idx_t needed_size(db_size + si.size); + if (max_db_size < needed_size) { + idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + for (; new_size < needed_size; new_size *= 2) { } + u1_t* new_pac(NULL); + safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + memcpy(new_pac, pac, (db_size + 3) / 4); + safe_free(pac); + pac = new_pac; + max_db_size = new_size; + } + const Sequence::str_t& org_seq(seq.sequence()); + const u1_t* const table(get_dna_encode_table()); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + for (idx_t i(0); i < si.size; ++i, ++db_size) { + const u1_t c(table[static_cast(org_seq[i])]); + set_char(db_size, c < 4 ? c : ++rand_char & 3); + } +} + +void PackedDB::load_fasta_db(const char* const dbname) { + DynamicTimer dtimer(__func__); + FastaReader freader(dbname); + Sequence seq; + while (freader.read_one_seq(seq) != -1) { + add_one_seq(seq); + } +} + +// XXX - make this restartable +void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const idx_t min_size) { + DynamicTimer dtimer(__func__); + u1_t buffer[MAX_SEQ_SIZE]; + const u1_t* const et(get_dna_encode_table()); + FastaReader fr(fasta.c_str()); + std::string filename(generate_pac_name(output_prefix)); + unlink(filename.c_str()); + filename += ".tmp"; + std::ofstream pout; + open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); + std::streambuf* psb(pout.rdbuf()); + filename = generate_idx_name(output_prefix); + unlink(filename.c_str()); + filename += ".tmp"; + std::ofstream iout; + open_fstream(iout, filename.c_str(), std::ios::out); + Sequence read; + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + off_t file_offset(0); + for (;;) { + const idx_t rsize(fr.read_one_seq(read)); + if (rsize == -1) { + break; + } + if (rsize < min_size) { + // can't skip entries in index or read ids won't + // match ones from candidates, but put dummy size + // in ones we don't use so we error out if they're + // somehow used + iout << file_offset << "\t0\n"; + continue; + } + Sequence::str_t& s(read.sequence()); + const idx_t rbytes((rsize + 3) / 4); + // set_char uses | to set bits, so clear first + bzero(buffer, rbytes); + for (idx_t i(0); i < rsize; ++i) { + const u1_t c(et[static_cast(s[i])]); + set_char(buffer, i, c < 4 ? c : ++rand_char & 3); + } + sb_write(psb, buffer, rbytes); + iout << file_offset << "\t" << rsize << "\n"; + file_offset += rbytes; + } + close_fstream(pout); + close_fstream(iout); + filename = generate_pac_name(output_prefix); + std::string tmp(filename + ".tmp"); + if (rename(tmp.c_str(), filename.c_str()) == -1) { + ERROR("Could not rename tmp database file"); + } + filename = generate_idx_name(output_prefix); + tmp = filename + ".tmp"; + if (rename(tmp.c_str(), filename.c_str()) == -1) { + ERROR("Could not rename tmp database index file"); + } +} + +void PackedDB::open_db(const std::string& path, const idx_t size) { + destroy(); + std::string filename(generate_pac_name(path)); + struct stat buf; + if (stat(filename.c_str(), &buf) == -1) { + ERROR("Could not stat fasta db file: %s", filename.c_str()); + } + max_db_size = std::min(idx_t(buf.st_size), size); + if (max_db_size > 0) { + safe_calloc(pac, u1_t, max_db_size); + } + open_fstream(pstream, filename.c_str(), std::ios::in); + filename = generate_idx_name(path); + std::ifstream index; + open_fstream(index, filename.c_str(), std::ios::in); + SeqIndex si; + if (max_db_size == buf.st_size && max_db_size) { // read it all! + while (index >> si.file_offset >> si.size) { + si.memory_offset = si.file_offset * 4; + seq_idx.push_back(si); + } + close_fstream(index); + std::streambuf* sb(pstream.rdbuf()); + sb_read(sb, pac, max_db_size); + close_fstream(pstream); + } else { + si.memory_offset = -1; + while (index >> si.file_offset >> si.size) { + seq_idx.push_back(si); + } + close_fstream(index); + } +} + +idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t nec) { + if (!pstream.is_open()) { // all in memory already + return nec; + } + DynamicTimer dtimer(__func__); + // not sure if it's faster to just read from disk and not + // worry about obsolete memory_offset values, or if it would be + // better to shift any reads in memory to bottom and then read in + // new ones (which would require going through entire index and + // would leave reads out of order) + // + // get set of reads to read in (most that will fit in memory); + // use sorted set to speed reading (below) + std::set read_ids; + idx_t i(0), total_size(0); + while (i != nec) { + // find all alignments for a given read + // (use set to handle duplicate qids) + const idx_t sid(ec_list[i].sid); + const idx_t start(i); + std::set my_ids; + my_ids.insert(sid); + my_ids.insert(ec_list[i].qid); + for (++i; i != nec && ec_list[i].sid == sid; ++i) { + my_ids.insert(ec_list[i].qid); + } + // find size of new read additions + idx_t size(0); + std::set::const_iterator a(my_ids.begin()); + const std::set::const_iterator end_a(my_ids.end()); + for (; a != end_a; ++a) { + if (read_ids.find(*a) == read_ids.end()) { + size += (seq_idx[*a].size + 3) / 4; + } + } + // if memory is limited, see if we've hit the limit + if (max_db_size > 0 && total_size + size > max_db_size) { + i = start; + break; + } + total_size += size; + read_ids.insert(my_ids.begin(), my_ids.end()); + } + LOG(stderr, "using %ld bytes for %lu reads, %ld aligns (out of %ld)", total_size, read_ids.size(), i, nec); + if (max_db_size == 0) { + max_db_size = total_size; + safe_calloc(pac, u1_t, max_db_size); + } + // now read in the reads + std::set::const_iterator a(read_ids.begin()); + const std::set::const_iterator end_a(read_ids.end()); + idx_t pos(0); + off_t offset(-1); + for (; a != end_a; ++a) { + SeqIndex& si(seq_idx[*a]); + // don't seek if we're already in position + if (offset != si.file_offset && !pstream.seekg(si.file_offset)) { + ERROR("Error seeking on fasta db"); + } + const idx_t bytes((si.size + 3) / 4); + if (!pstream.read((char*)pac + pos, bytes)) { + ERROR("Error reading fasta db"); + } + si.memory_offset = pos * 4; + pos += bytes; + offset = si.file_offset + bytes; + } + if (i == nec) { + pstream.close(); + } + return i; +} + +#if 0 +// these routines aren't used anywhere +void PackedDB::dump_pac(const u1_t* const p, const idx_t size, const char* const path) { + std::ofstream out; + open_fstream(out, path, std::ios::out | std::ios::binary); + std::streambuf* sb(out.rdbuf()); + sb_write(sb, p, (size + 3) / 4); + sb_write(sb, &size, sizeof(idx_t)); + close_fstream(out); +} + +u1_t* PackedDB::load_pac(const char* path, idx_t& size) { + std::ifstream in; + open_fstream(in, path, std::ios::in | std::ios::binary); + std::streambuf* sb(in.rdbuf()); + in.seekg(-sizeof(idx_t), std::ios::end); + sb_read(sb, &size, sizeof(idx_t)); + in.seekg(0, std::ios::beg); + u1_t* p; + safe_calloc(p, u1_t, (size + 3) / 4); + sb_read(sb, p, (size + 3) / 4); + close_fstream(in); + return p; +} + +void PackedDB::dump_idx(const PODArray& idx_list, const char* const path) { + std::ofstream out; + open_fstream(out, path, std::ios::out); + const idx_t n(idx_list.size()); + for (idx_t i(0); i < n; ++i) { + out << idx_list[i].memory_offset << "\t" << idx_list[i].size << "\n"; + } + close_fstream(out); +} + +void PackedDB::load_idx(const char* const path, PODArray& idx_list) { + idx_list.clear(); + std::ifstream in; + open_fstream(in, path, std::ios::in); + SeqIndex si; + si.file_offset = -1; + while (in >> si.memory_offset >> si.size) { + idx_list.push_back(si); + } + close_fstream(in); +} + +void PackedDB::dump_packed_db(const char* const path) const { + std::string(generate_pac_name(path)); + dump_pac(pac, db_size, n.c_str()); + n = generate_idx_name(path); + dump_idx(seq_idx, n.c_str()); +} + +void PackedDB::load_packed_db(const char* const path) { + std::string n(generate_pac_name(path)); + pac = load_pac(n.c_str(), db_size); + max_db_size = db_size; + n = generate_idx_name(path); + load_idx(n.c_str(), seq_idx); +} + +void PackedDB::pack_fasta_db(const char* const path, const char* const output_prefix, const idx_t min_size) { + u1_t buffer[MAX_SEQ_SIZE]; + const u1_t* const et(get_dna_encode_table()); + FastaReader fr(path); + std::string filename(generate_pac_name(output_prefix)); + std::ofstream pout; + open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); + std::streambuf* psb(pout.rdbuf()); + filename = generate_idx_name(output_prefix); + std::ofstream iout; + open_fstream(iout, filename.c_str(), std::ios::out); + Sequence read; + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + idx_t count(0), tsize(0); + off_t file_offset(0); + for (;;) { + idx_t rsize(fr.read_one_seq(read)); + if (rsize == -1) { + break; + } else if (rsize < min_size) { + continue; + } + Sequence::str_t& s(read.sequence()); + // set_char uses | to set bits, so clear first + bzero(buffer, (rsize + 3) / 4); + for (idx_t i(0); i < rsize; ++i) { + const u1_t c(et[static_cast(s[i])]); + set_char(buffer, i, c < 4 ? c : ++rand_char & 3); + } + iout << file_offset << "\t" << rsize << "\n"; + tsize += rsize; + rsize = (rsize + 3) / 4; + sb_write(psb, buffer, rsize); + file_offset += rsize * 4; + ++count; + } + sb_write(psb, &file_offset, sizeof(idx_t)); + close_fstream(pout); + close_fstream(iout); + LOG(stdout, "pack %ld reads, totally %ld residues", count, tsize); +} + +void PackedDB::add_one_seq(const char* const seq, const idx_t size) { + SeqIndex si; + si.file_offset = -1; + si.memory_offset = db_size; + si.size = size; + seq_idx.push_back(si); + const idx_t needed_size(db_size + si.size); + if (max_db_size < needed_size) { + idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + for (; new_size < needed_size; new_size *= 2) { } + u1_t* new_pac(NULL); + safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + memcpy(new_pac, pac, (db_size + 3) / 4); + safe_free(pac); + pac = new_pac; + max_db_size = new_size; + } + const u1_t* const table(get_dna_encode_table()); + unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + for (idx_t i(0); i < si.size; ++i, ++db_size) { + const u1_t c(table[static_cast(seq[i])]); + set_char(db_size, c < 4 ? c : ++rand_char & 3); + } +} + +// convert offset into read id +idx_t PackedDB::offset_to_rid(const idx_t offset) const { + if (offset >= db_size) { + return -1; + } + idx_t left(0), mid(0), right(seq_idx.size()); + while (left < right) { + mid = (left + right) >> 1; + if (offset < seq_idx[mid].memory_offset) { + right = mid; + } else if (mid == seq_idx.size() - 1) { + break; + } else if (offset < seq_idx[mid + 1].memory_offset) { + break; + } else { + left = mid + 1; + } + } + return mid; +} +#endif diff --git a/src/common/packed_db.h b/src/mecat2cns/packed_db.h similarity index 73% rename from src/common/packed_db.h rename to src/mecat2cns/packed_db.h index 1b4e9e1..023892e 100644 --- a/src/common/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -1,8 +1,13 @@ #ifndef PACKED_DB_H #define PACKED_DB_H -#include "defs.h" -#include "sequence.h" +#include // ifstream +#include // set<> +#include // string + +#include "../common/defs.h" +#include "../common/sequence.h" // Sequence +#include "../common/alignment.h" // ExtensionCandidate class PackedDB { private: @@ -17,65 +22,65 @@ class PackedDB { safe_free(pac); } } + // only call one of load_fasta_db and open_db exactly once void load_fasta_db(const char* fasta); + static void convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); + // opens data file, reads in index file + void open_db(const std::string& filename, idx_t memory_footprint); + // returns number of candidates that can be processed + idx_t load_reads(const ExtensionCandidate* ec_list, idx_t nec); void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { r_assert(size == seq_idx[id].size); if (forward) { const idx_t offset(seq_idx[id].memory_offset); for (idx_t i(0); i < size; ++i) { - seq[i] = get_char(pac, offset + i); + seq[i] = get_char(offset + i); } } else { const idx_t offset(seq_idx[id].memory_offset + size - 1); for (idx_t i(0); i < size; ++i) { - seq[i] = 3 - get_char(pac, offset - i); + seq[i] = 3 - get_char(offset - i); } } } - static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { - s.reserve(s.size() + end_i - i); - for (; i < end_i; ++i) { - s += "ACGT"[static_cast(seq[i])]; - } - } - static void decode_sequence(char* const seq, const idx_t seq_size) { - for (idx_t i(0); i < seq_size; ++i) { - const u1_t c(seq[i]); - r_assert(c < 4); // c is unsigned, so always >= 0 - seq[i] = "ACGT"[c]; - } + idx_t num_reads() const { + return seq_idx.size(); } + private: static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { - // use ~x instead of 3 - x for speed p[idx >> 2] |= c << ((~idx & 3) << 1); } - static u1_t get_char(const u1_t* const p, const idx_t idx) { - return p[idx >> 2] >> ((~idx & 3) << 1) & 3; + void set_char(const idx_t idx, const u1_t c) { + // use ~x instead of 3 - x for speed, since we have to & 3 anyway + pac[idx >> 2] |= c << ((~idx & 3) << 1); } - static void generate_pac_name(const char* const prefix, std::string& ret) { - ret = prefix; - ret += ".pac"; + u1_t get_char(const idx_t idx) const { + return pac[idx >> 2] >> ((~idx & 3) << 1) & 3; } - static void generate_idx_name(const char* const prefix, std::string& ret) { - ret = prefix; - ret += ".idx"; + static std::string generate_pac_name(const std::string& prefix) { + return prefix + ".pac"; + } + static std::string generate_idx_name(const std::string& prefix) { + return prefix + ".idx"; } - private: void add_one_seq(const Sequence& seq); - private: - u1_t* pac; - idx_t db_size; - idx_t max_db_size; - PODArray seq_idx; - -/* // these routines aren't used anywhere void destroy() { if (pac) { safe_free(pac); + pac = NULL; } seq_idx.clear(); max_db_size = db_size = 0; } + private: + u1_t* pac; + idx_t db_size; + idx_t max_db_size; + PODArray seq_idx; + std::ifstream pstream; + +#if 0 + // these routines aren't used anywhere void clear() { seq_idx.clear(); db_size = 0; @@ -107,6 +112,13 @@ class PackedDB { const idx_t offset(seq_idx[rid].memory_offset); get_sequence(offset + from, offset + to, forward, seq); } + static void decode_sequence(char* const seq, const idx_t seq_size) { + for (idx_t i(0); i < seq_size; ++i) { + const u1_t c(seq[i]); + r_assert(c < 4); // c is unsigned, so always >= 0 + seq[i] = "ACGT"[c]; + } + } void get_decode_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { get_sequence(rid, from, to, forward, seq); decode_sequence(seq, to - from); @@ -138,8 +150,7 @@ class PackedDB { void dump_packed_db(const char* path) const; void load_packed_db(const char* path); static void pack_fasta_db(const char* fasta, const char* output_prefix, const idx_t min_size); - -*/ +#endif }; diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 939932c..6346e7c 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -80,15 +80,15 @@ void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, st void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, const idx_t nec) { const int n(data.rco.num_threads); - // split by number of ec's, rather than reads, since reads ids are not - // necessarily contiguous and we could get empty lists - for (idx_t i(0), k(0); k < n; ++k) { + // split by number of ec's, rather than reads, since reads ids + // are not contiguous and we could get empty lists + for (idx_t i(0), k(0); k != n; ++k) { const idx_t start(i); // drop fractions here, as we'll likely add a few more ec's below i += (nec - i) / (n - k); - if (i < nec) { // include all ec's for the last read + if (i != nec) { // include all ec's for the last read const int final_sid(ec_list[i].sid); - for (++i; i < nec && ec_list[i].sid == final_sid; ++i) { } + for (++i; i != nec && ec_list[i].sid == final_sid; ++i) { } } data.data[k].num_candidates = i - start; data.data[k].candidates = ec_list + start; diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index dd5ac01..b69d0b7 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -6,7 +6,7 @@ #include #include "dw.h" -#include "../common/packed_db.h" +#include "packed_db.h" #include "options.h" struct CnsTableItem @@ -172,12 +172,13 @@ class ConsensusThreadData { PackedDB& reads; std::ostream& out; pthread_mutex_t out_lock; + idx_t ec_offset; // this doesn't work as a vector - all the pointers end up pointing // to the same values, and eventually it seg faults (probably a // compiler optimization bug) ConsensusPerThreadData* data; public: - ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), data(new ConsensusPerThreadData[prco.num_threads]), last_thread_id_(-1), num_threads_written_(0) { + ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), ec_offset(0), data(new ConsensusPerThreadData[prco.num_threads]), last_thread_id_(-1), num_threads_written_(0) { done_file_ = input_file_name; done_file_ += ".done"; ckpt_file_ = input_file_name; @@ -197,6 +198,12 @@ class ConsensusThreadData { pthread_mutex_unlock(&id_lock_); return tid; } + void reset_threads() { + last_thread_id_ = -1; + for (int i(0); i < rco.num_threads; ++i) { + data[i].next_candidate = 0; + } + } void write_buffer(const int tid, const idx_t i) { ConsensusPerThreadData& pdata(data[tid]); std::vector::const_iterator a(pdata.cns_results.begin()); @@ -227,7 +234,7 @@ class ConsensusThreadData { if (!ckpt_in) { ERROR("Restart failed: could not open checkpoint file: %s", ckpt_file_.c_str()); } - ckpt_in >> rco.job_index >> output_pos; + ckpt_in >> rco.job_index >> output_pos >> ec_offset; if (!ckpt_in) { ERROR("Restart failed: could not read checkpoint file: %s", ckpt_file_.c_str()); } @@ -247,7 +254,7 @@ class ConsensusThreadData { return; } out.flush(); - ckpt_out << rco.job_index << " " << off_t(out.tellp()) << "\n"; + ckpt_out << rco.job_index << " " << off_t(out.tellp()) << " " << ec_offset << "\n"; if (!ckpt_out) { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); return; diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index ce595bd..da7e3ab 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -17,10 +17,10 @@ static void* reads_correction_func_can(void* const arg) { const idx_t min_size(ceil(data.rco.min_size * 0.95)); const int tech_is_pacbio(data.rco.tech == TECH_PACBIO ? 1 : 0); idx_t i(pdata.next_candidate); - while (i < pdata.num_candidates) { + while (i != pdata.num_candidates) { const idx_t start(i); const idx_t sid(candidates[start].sid); - for (++i; i < pdata.num_candidates && candidates[i].sid == sid; ++i) { } + for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } if (i - start < data.rco.min_cov || candidates[start].ssize < min_size) { continue; } @@ -37,18 +37,88 @@ static void* reads_correction_func_can(void* const arg) { return NULL; } +class EC_Index { + public: + idx_t offset, count; + explicit EC_Index() : offset(0), count(0) { } + explicit EC_Index(idx_t i, idx_t j) : offset(i), count(j) { } + ~EC_Index() { } +}; + +// reorder list so reads are more concentrated and we can process more +// ecs per pass, with limited read space + +static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, const idx_t nec, const idx_t num_reads) { + ExtensionCandidate* new_list(new ExtensionCandidate[nec]); + std::vector index(num_reads); + // index existing list by sid + for (idx_t i(0); i != nec;) { + const idx_t start(i); + const idx_t sid(ec_list[i].sid); + for (++i; i != nec && ec_list[i].sid == sid; ++i) { } + index[sid] = EC_Index(start, i - start); + } + // generate the new read order + std::vector used(index.size(), 0); + std::vector new_order; + new_order.reserve(index.size()); + idx_t next_unused(0), next_search(0); + for (;;) { + // skip over used reads, reads with no alignments + for (; static_cast(next_unused) != used.size() && (used[next_unused] || index[next_unused].count == 0); ++next_unused) { } + if (static_cast(next_unused) == used.size()) { + break; + } + used[next_unused] = 1; + new_order.push_back(next_unused); + // add all reads aligned to, and aligned to those, and so on + for (; static_cast(next_search) != new_order.size(); ++next_search) { + const idx_t sid(new_order[next_search]); + idx_t i(index[sid].offset); + const idx_t end_i(i + index[sid].count); + for (; i != end_i; ++i) { + const idx_t qid(ec_list[i].qid); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); + } + } + } + } + // copy over the ecs to the new list in the new order + idx_t pos(0); + std::vector::const_iterator a(new_order.begin()); + const std::vector::const_iterator end_a(new_order.end()); + for (; a != end_a; ++a) { + const EC_Index& b(index[*a]); + memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidate) * b.count); + pos += b.count; + } + delete[] ec_list; + return new_list; +} + // load and sort partition data, assign to threads, start threads static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { idx_t nec; - ExtensionCandidate* const ec_list(load_partition_data(m4_file_name, nec)); + ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); - allocate_ecs(data, ec_list, nec); + ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads()); pthread_t thread_ids[data.rco.num_threads]; - for (int i(0); i < data.rco.num_threads; ++i) { - pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(&data)); - } - for (int i(0); i < data.rco.num_threads; ++i) { - pthread_join(thread_ids[i], NULL); + while (data.ec_offset != nec) { + // see how many candidates we can run, given + // how much read sequence we can load into memory + const idx_t ecs(data.reads.load_reads(ec_list + data.ec_offset, nec - data.ec_offset)); + allocate_ecs(data, ec_list + data.ec_offset, ecs); + for (int i(0); i != data.rco.num_threads; ++i) { + pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(&data)); + } + for (int i(0); i != data.rco.num_threads; ++i) { + pthread_join(thread_ids[i], NULL); + } + // don't update offset before threading to prevent checkpointing with new value + data.ec_offset += ecs; + data.reset_threads(); } delete[] ec_list; } @@ -87,7 +157,8 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; - reads.load_fasta_db(rco.reads); + //reads.load_fasta_db(rco.reads); + reads.open_db("fasta.db", rco.read_buffer_size); if (rco.job_index != -1) { return reads_correction_can_p(rco, partition_file_vec, reads); } else { @@ -108,7 +179,7 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { } char process_info[1024]; const int job_end(partition_file_vec.size()); - for (; rco.job_index < job_end; ++rco.job_index) { + for (; rco.job_index != job_end; ++rco.job_index) { const std::string& p(partition_file_vec[rco.job_index]); sprintf(process_info, "processing %s", p.c_str()); DynamicTimer dtimer(process_info); diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index 6818e55..a1534f0 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -5,7 +5,6 @@ #include "pw_options.h" #include "../common/diff_gapalign.h" #include "../common/xdrop_gapalign.h" -#include "../common/packed_db.h" #include "../common/lookup_table.h" #include "pw_impl.h" diff --git a/src/mecat2pw/pw_impl.h b/src/mecat2pw/pw_impl.h index f915e0c..8c0d590 100644 --- a/src/mecat2pw/pw_impl.h +++ b/src/mecat2pw/pw_impl.h @@ -2,7 +2,6 @@ #define PW_IMPL_H #include "../common/alignment.h" -#include "../common/packed_db.h" #include "../common/lookup_table.h" #include "pw_options.h" // options_t From ef131a7f245d8b45babf0ae5b058268bb3ef9adc Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 1 Apr 2019 12:11:38 -0500 Subject: [PATCH 09/56] refactored candidate thread loop also removed unneeded aserts from dw --- src/mecat2cns/dw.cpp | 10 ++--- src/mecat2cns/reads_correction_can.cpp | 57 ++++++++++++++++---------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index c95f13c..5164d01 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -389,14 +389,14 @@ int dw(const char* query, const int query_size, const int query_start, const cha const char* encode2char("ACGT-"); for (k = result->left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { unsigned char ch(result->left_store1[k]); - r_assert(ch >= 0 && ch <= 4); + r_assert(ch <= 4); ch = encode2char[ch]; result->out_store1[idx] = ch; if (ch != '-') { ++i; } ch = result->left_store2[k]; - r_assert(ch >= 0 && ch <= 4); + r_assert(ch <= 4); ch = encode2char[ch]; result->out_store2[idx] = ch; if (ch != '-') { @@ -406,20 +406,20 @@ int dw(const char* query, const int query_size, const int query_start, const cha result->query_start = query_start - i; if (result->query_start < 0) { std::cerr << "query_start = " << query_start << ", i = " << i << "\n"; + r_assert(result->query_start >= 0); } - r_assert(result->query_start >= 0); result->target_start = target_start - j; r_assert(result->target_start >= 0); for (k = 0, i = 0, j = 0; k < result->right_store_size; ++k, ++idx) { unsigned char ch(result->right_store1[k]); - r_assert(ch >= 0 && ch <= 4); + r_assert(ch <= 4); ch = encode2char[ch]; result->out_store1[idx] = ch; if (ch != '-') { ++i; } ch = result->right_store2[k]; - r_assert(ch >= 0 && ch <= 4); + r_assert(ch <= 4); ch = encode2char[ch]; result->out_store2[idx] = ch; if (ch != '-') { diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index da7e3ab..6c2e2b7 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -14,30 +14,33 @@ static void* reads_correction_func_can(void* const arg) { const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); const ExtensionCandidate* const candidates(pdata.candidates); - const idx_t min_size(ceil(data.rco.min_size * 0.95)); - const int tech_is_pacbio(data.rco.tech == TECH_PACBIO ? 1 : 0); idx_t i(pdata.next_candidate); - while (i != pdata.num_candidates) { - const idx_t start(i); - const idx_t sid(candidates[start].sid); - for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } - if (i - start < data.rco.min_cov || candidates[start].ssize < min_size) { - continue; - } - if (tech_is_pacbio) { + if (data.rco.tech == TECH_PACBIO) { + while (i != pdata.num_candidates) { + const idx_t start(i); + const idx_t sid(candidates[start].sid); + for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } ns_meap_cns::consensus_one_read_can_pacbio(data, pdata, sid, start, i); - } else { - ns_meap_cns::consensus_one_read_can_nanopore(data, pdata, sid, start, i); + if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { + data.write_buffer(tid, i); + } } - if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { - data.write_buffer(tid, i); + } else { + while (i != pdata.num_candidates) { + const idx_t start(i); + const idx_t sid(candidates[start].sid); + for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } + ns_meap_cns::consensus_one_read_can_nanopore(data, pdata, sid, start, i); + if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { + data.write_buffer(tid, i); + } } } data.write_buffer(tid, i); return NULL; } -class EC_Index { +class EC_Index { // offset into ec_list (and number of ecs) for each read id public: idx_t offset, count; explicit EC_Index() : offset(0), count(0) { } @@ -46,18 +49,26 @@ class EC_Index { }; // reorder list so reads are more concentrated and we can process more -// ecs per pass, with limited read space +// ecs per pass, with limited read space; also filters list to exclude +// candidates of reads with low size or coverage -static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, const idx_t nec, const idx_t num_reads) { - ExtensionCandidate* new_list(new ExtensionCandidate[nec]); +static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov, const idx_t min_size) { + idx_t total_ec(0); std::vector index(num_reads); // index existing list by sid for (idx_t i(0); i != nec;) { const idx_t start(i); const idx_t sid(ec_list[i].sid); for (++i; i != nec && ec_list[i].sid == sid; ++i) { } - index[sid] = EC_Index(start, i - start); + // skip candidates that aren't up to snuff + const idx_t count(i - start); + if (count >= min_cov && ec_list[start].ssize >= min_size) { + index[sid] = EC_Index(start, count); + total_ec += count; + } } + nec = total_ec; + ExtensionCandidate* new_list(new ExtensionCandidate[nec]); // generate the new read order std::vector used(index.size(), 0); std::vector new_order; @@ -91,8 +102,10 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, const std::vector::const_iterator end_a(new_order.end()); for (; a != end_a; ++a) { const EC_Index& b(index[*a]); - memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidate) * b.count); - pos += b.count; + if (b.count) { + memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidate) * b.count); + pos += b.count; + } } delete[] ec_list; return new_list; @@ -103,7 +116,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu idx_t nec; ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); - ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads()); + ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov, ceil(data.rco.min_size * 0.95)); pthread_t thread_ids[data.rco.num_threads]; while (data.ec_offset != nec) { // see how many candidates we can run, given From c74943cf400782339c51e4f5f042da713879ca94 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 1 Apr 2019 14:37:12 -0500 Subject: [PATCH 10/56] made fasta to pac conversion restartable --- .gitignore | 3 + src/mecat2cns/packed_db.cpp | 262 +++++++++++------------------------- src/mecat2cns/packed_db.h | 80 ----------- 3 files changed, 80 insertions(+), 265 deletions(-) diff --git a/.gitignore b/.gitignore index 9808873..1f78b82 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ Temporary Items # Manual backup files *.orig + +# binaries directory +Linux-amd64 diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 49a688f..44f84e0 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -6,7 +6,7 @@ #include #include // bzero() #include // stat(), struct stat -#include // unlink() +#include // F_OK, unlink() #include "../common/defs.h" #include "../common/fasta_reader.h" @@ -47,26 +47,66 @@ void PackedDB::load_fasta_db(const char* const dbname) { } } -// XXX - make this restartable +static int check_conversion_restart(const std::string& ckpt_file, off_t& fasta_offset, off_t& pac_offset, off_t& index_offset, unsigned int& rand_char) { + std::ifstream in(ckpt_file.c_str()); + if (!in) { + return 0; + } + in >> fasta_offset >> pac_offset >> index_offset >> rand_char; + if (!in) { + ERROR("Read error while restoring checkpoint from %s", ckpt_file.c_str()); + } + return 1; +} + +static void checkpoint_conversion(const std::string& ckpt_file, const std::string& ckpt_file_tmp, const off_t fasta_offset, const off_t pac_offset, const off_t index_offset, const unsigned int rand_char) { + std::ofstream out(ckpt_file_tmp.c_str()); + if (!out) { + LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp.c_str()); + return; + } + out << fasta_offset << " " << pac_offset << " " << index_offset << " " << rand_char << "\n"; + if (!out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp.c_str()); + return; + } + out.close(); + if (rename(ckpt_file_tmp.c_str(), ckpt_file.c_str()) == -1) { + LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file.c_str()); + } +} + void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const idx_t min_size) { + const std::string pac_name(output_prefix + ".pac"); + const std::string index_name(output_prefix + ".idx"); + // see if we already did this + if (access(pac_name.c_str(), F_OK) == 0 && access(index_name.c_str(), F_OK) == 0) { + return; + } DynamicTimer dtimer(__func__); u1_t buffer[MAX_SEQ_SIZE]; const u1_t* const et(get_dna_encode_table()); FastaReader fr(fasta.c_str()); - std::string filename(generate_pac_name(output_prefix)); - unlink(filename.c_str()); - filename += ".tmp"; - std::ofstream pout; - open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); - std::streambuf* psb(pout.rdbuf()); - filename = generate_idx_name(output_prefix); - unlink(filename.c_str()); - filename += ".tmp"; - std::ofstream iout; - open_fstream(iout, filename.c_str(), std::ios::out); - Sequence read; + const std::string pac_name_tmp(pac_name + ".tmp"); + const std::string index_name_tmp(index_name + ".tmp"); + const std::string ckpt_name(pac_name + ".ckpt"); + const std::string ckpt_name_tmp(ckpt_name + ".tmp"); + std::ofstream pout, iout; unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - off_t file_offset(0); + off_t file_offset(0), fasta_offset, index_offset; + if (check_conversion_restart(ckpt_name, fasta_offset, file_offset, index_offset, rand_char)) { + // don't truncate on restart + open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); + pout.seekp(file_offset); + open_fstream(iout, index_name_tmp.c_str(), std::ios::out | std::ios::in); + iout.seekp(index_offset); + fr.seekg(fasta_offset); + } else { + open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::binary); + open_fstream(iout, index_name_tmp.c_str(), std::ios::out); + } + Sequence read; + time_t next_checkpoint_time(time(0) + 300); for (;;) { const idx_t rsize(fr.read_one_seq(read)); if (rsize == -1) { @@ -74,9 +114,8 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& } if (rsize < min_size) { // can't skip entries in index or read ids won't - // match ones from candidates, but put dummy size - // in ones we don't use so we error out if they're - // somehow used + // match ones from candidates, so put dummy size + // in ones we don't use iout << file_offset << "\t0\n"; continue; } @@ -88,39 +127,43 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& const u1_t c(et[static_cast(s[i])]); set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } - sb_write(psb, buffer, rbytes); + if (!pout.write((char*)buffer, rbytes)) { + ERROR("Write error to file %s", pac_name_tmp.c_str()); + } iout << file_offset << "\t" << rsize << "\n"; file_offset += rbytes; + if (time(0) >= next_checkpoint_time) { + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), file_offset, iout.tellp(), rand_char); + next_checkpoint_time = time(0) + 300; + } } + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), file_offset, iout.tellp(), rand_char); close_fstream(pout); close_fstream(iout); - filename = generate_pac_name(output_prefix); - std::string tmp(filename + ".tmp"); - if (rename(tmp.c_str(), filename.c_str()) == -1) { + if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { ERROR("Could not rename tmp database file"); } - filename = generate_idx_name(output_prefix); - tmp = filename + ".tmp"; - if (rename(tmp.c_str(), filename.c_str()) == -1) { + if (rename(index_name_tmp.c_str(), index_name.c_str()) == -1) { ERROR("Could not rename tmp database index file"); } + unlink(ckpt_name.c_str()); } void PackedDB::open_db(const std::string& path, const idx_t size) { destroy(); - std::string filename(generate_pac_name(path)); + const std::string pac_name(path + ".pac"); struct stat buf; - if (stat(filename.c_str(), &buf) == -1) { - ERROR("Could not stat fasta db file: %s", filename.c_str()); + if (stat(pac_name.c_str(), &buf) == -1) { + ERROR("Could not stat fasta db file: %s", pac_name.c_str()); } max_db_size = std::min(idx_t(buf.st_size), size); - if (max_db_size > 0) { + if (max_db_size) { safe_calloc(pac, u1_t, max_db_size); } - open_fstream(pstream, filename.c_str(), std::ios::in); - filename = generate_idx_name(path); + open_fstream(pstream, pac_name.c_str(), std::ios::in); + const std::string index_name(path + ".idx"); std::ifstream index; - open_fstream(index, filename.c_str(), std::ios::in); + open_fstream(index, index_name.c_str(), std::ios::in); SeqIndex si; if (max_db_size == buf.st_size && max_db_size) { // read it all! while (index >> si.file_offset >> si.size) { @@ -149,7 +192,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t // worry about obsolete memory_offset values, or if it would be // better to shift any reads in memory to bottom and then read in // new ones (which would require going through entire index and - // would leave reads out of order) + // would leave reads out of order); past experience says the former // // get set of reads to read in (most that will fit in memory); // use sorted set to speed reading (below) @@ -176,7 +219,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t } } // if memory is limited, see if we've hit the limit - if (max_db_size > 0 && total_size + size > max_db_size) { + if (max_db_size && total_size + size > max_db_size) { i = start; break; } @@ -212,154 +255,3 @@ idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t } return i; } - -#if 0 -// these routines aren't used anywhere -void PackedDB::dump_pac(const u1_t* const p, const idx_t size, const char* const path) { - std::ofstream out; - open_fstream(out, path, std::ios::out | std::ios::binary); - std::streambuf* sb(out.rdbuf()); - sb_write(sb, p, (size + 3) / 4); - sb_write(sb, &size, sizeof(idx_t)); - close_fstream(out); -} - -u1_t* PackedDB::load_pac(const char* path, idx_t& size) { - std::ifstream in; - open_fstream(in, path, std::ios::in | std::ios::binary); - std::streambuf* sb(in.rdbuf()); - in.seekg(-sizeof(idx_t), std::ios::end); - sb_read(sb, &size, sizeof(idx_t)); - in.seekg(0, std::ios::beg); - u1_t* p; - safe_calloc(p, u1_t, (size + 3) / 4); - sb_read(sb, p, (size + 3) / 4); - close_fstream(in); - return p; -} - -void PackedDB::dump_idx(const PODArray& idx_list, const char* const path) { - std::ofstream out; - open_fstream(out, path, std::ios::out); - const idx_t n(idx_list.size()); - for (idx_t i(0); i < n; ++i) { - out << idx_list[i].memory_offset << "\t" << idx_list[i].size << "\n"; - } - close_fstream(out); -} - -void PackedDB::load_idx(const char* const path, PODArray& idx_list) { - idx_list.clear(); - std::ifstream in; - open_fstream(in, path, std::ios::in); - SeqIndex si; - si.file_offset = -1; - while (in >> si.memory_offset >> si.size) { - idx_list.push_back(si); - } - close_fstream(in); -} - -void PackedDB::dump_packed_db(const char* const path) const { - std::string(generate_pac_name(path)); - dump_pac(pac, db_size, n.c_str()); - n = generate_idx_name(path); - dump_idx(seq_idx, n.c_str()); -} - -void PackedDB::load_packed_db(const char* const path) { - std::string n(generate_pac_name(path)); - pac = load_pac(n.c_str(), db_size); - max_db_size = db_size; - n = generate_idx_name(path); - load_idx(n.c_str(), seq_idx); -} - -void PackedDB::pack_fasta_db(const char* const path, const char* const output_prefix, const idx_t min_size) { - u1_t buffer[MAX_SEQ_SIZE]; - const u1_t* const et(get_dna_encode_table()); - FastaReader fr(path); - std::string filename(generate_pac_name(output_prefix)); - std::ofstream pout; - open_fstream(pout, filename.c_str(), std::ios::out | std::ios::binary); - std::streambuf* psb(pout.rdbuf()); - filename = generate_idx_name(output_prefix); - std::ofstream iout; - open_fstream(iout, filename.c_str(), std::ios::out); - Sequence read; - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - idx_t count(0), tsize(0); - off_t file_offset(0); - for (;;) { - idx_t rsize(fr.read_one_seq(read)); - if (rsize == -1) { - break; - } else if (rsize < min_size) { - continue; - } - Sequence::str_t& s(read.sequence()); - // set_char uses | to set bits, so clear first - bzero(buffer, (rsize + 3) / 4); - for (idx_t i(0); i < rsize; ++i) { - const u1_t c(et[static_cast(s[i])]); - set_char(buffer, i, c < 4 ? c : ++rand_char & 3); - } - iout << file_offset << "\t" << rsize << "\n"; - tsize += rsize; - rsize = (rsize + 3) / 4; - sb_write(psb, buffer, rsize); - file_offset += rsize * 4; - ++count; - } - sb_write(psb, &file_offset, sizeof(idx_t)); - close_fstream(pout); - close_fstream(iout); - LOG(stdout, "pack %ld reads, totally %ld residues", count, tsize); -} - -void PackedDB::add_one_seq(const char* const seq, const idx_t size) { - SeqIndex si; - si.file_offset = -1; - si.memory_offset = db_size; - si.size = size; - seq_idx.push_back(si); - const idx_t needed_size(db_size + si.size); - if (max_db_size < needed_size) { - idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); - for (; new_size < needed_size; new_size *= 2) { } - u1_t* new_pac(NULL); - safe_calloc(new_pac, u1_t, (new_size + 3) / 4); - memcpy(new_pac, pac, (db_size + 3) / 4); - safe_free(pac); - pac = new_pac; - max_db_size = new_size; - } - const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < si.size; ++i, ++db_size) { - const u1_t c(table[static_cast(seq[i])]); - set_char(db_size, c < 4 ? c : ++rand_char & 3); - } -} - -// convert offset into read id -idx_t PackedDB::offset_to_rid(const idx_t offset) const { - if (offset >= db_size) { - return -1; - } - idx_t left(0), mid(0), right(seq_idx.size()); - while (left < right) { - mid = (left + right) >> 1; - if (offset < seq_idx[mid].memory_offset) { - right = mid; - } else if (mid == seq_idx.size() - 1) { - break; - } else if (offset < seq_idx[mid + 1].memory_offset) { - break; - } else { - left = mid + 1; - } - } - return mid; -} -#endif diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 023892e..86d85a3 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -57,12 +57,6 @@ class PackedDB { u1_t get_char(const idx_t idx) const { return pac[idx >> 2] >> ((~idx & 3) << 1) & 3; } - static std::string generate_pac_name(const std::string& prefix) { - return prefix + ".pac"; - } - static std::string generate_idx_name(const std::string& prefix) { - return prefix + ".idx"; - } void add_one_seq(const Sequence& seq); void destroy() { if (pac) { @@ -78,80 +72,6 @@ class PackedDB { idx_t max_db_size; PODArray seq_idx; std::ifstream pstream; - -#if 0 - // these routines aren't used anywhere - void clear() { - seq_idx.clear(); - db_size = 0; - memset(pac, 0, (max_db_size + 3) / 4); - } - void reserve(const idx_t size) { - destroy(); - max_db_size = size; - safe_calloc(pac, u1_t, (max_db_size + 3) / 4); - } - void get_sequence(const idx_t from, const idx_t to, const bool forward, char* const seq) const { - if (forward) { - idx_t idx(0); - for (idx_t i(from); i < to; ++i, ++idx) { - seq[idx] = get_char(pac, i); - } - } else { - idx_t idx(to - from - 1); - for (idx_t i(from); i < to; ++i, --idx) { - seq[idx] = 3 - get_char(pac, i); - } - } - } - void get_sequence(const idx_t rid, const bool forward, char* const seq) const { - const SeqIndex& a(seq_idx[rid]); - get_sequence(a.memory_offset, a.memory_offset + a.size, forward, seq); - } - void get_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { - const idx_t offset(seq_idx[rid].memory_offset); - get_sequence(offset + from, offset + to, forward, seq); - } - static void decode_sequence(char* const seq, const idx_t seq_size) { - for (idx_t i(0); i < seq_size; ++i) { - const u1_t c(seq[i]); - r_assert(c < 4); // c is unsigned, so always >= 0 - seq[i] = "ACGT"[c]; - } - } - void get_decode_sequence(const idx_t rid, const idx_t from, const idx_t to, const bool forward, char* const seq) const { - get_sequence(rid, from, to, forward, seq); - decode_sequence(seq, to - from); - } - void set_char(const idx_t idx, const u1_t c) { - set_char(pac, idx, c); - } - u1_t get_char(const idx_t idx) const { - return get_char(pac, idx); - } - idx_t size() const { - return db_size; - } - idx_t num_seqs() const { - return seq_idx.size(); - } - idx_t seq_offset(const idx_t rid) const { - return seq_idx[rid].memory_offset; - } - idx_t seq_size(const idx_t rid) const { - return seq_idx[rid].size; - } - void add_one_seq(const char* seq, const idx_t size); - idx_t offset_to_rid(const idx_t offset) const; - static void dump_pac(const u1_t* p, idx_t size, const char* path); - static u1_t* load_pac(const char* path, idx_t& size); - static void dump_idx(const PODArray& idx_list, const char* path); - static void load_idx(const char* path, PODArray& idx_list); - void dump_packed_db(const char* path) const; - void load_packed_db(const char* path); - static void pack_fasta_db(const char* fasta, const char* output_prefix, const idx_t min_size); -#endif - }; #endif // PACKED_DB_H From b118c88afe359d6ddac0315aa76f0fd76705ac33 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 2 Apr 2019 22:16:18 -0500 Subject: [PATCH 11/56] testing reordering reads slightly worried about the memory footprint so currently using u4_t to hold the read index which limits total reads to 2^32-1, rather than 2^63-1 for the rest of the program. Easy to change, but will up the memory footprint of the reordering by taking 32 bytes per candidate rather than 16 --- src/mecat2cns/main.cpp | 16 ++- src/mecat2cns/overlaps_partition.cpp | 186 +++++++++++++++++++++++-- src/mecat2cns/overlaps_partition.h | 8 +- src/mecat2cns/packed_db.cpp | 121 ++++++++++++++-- src/mecat2cns/packed_db.h | 6 + src/mecat2cns/reads_correction_can.cpp | 29 ++-- 6 files changed, 330 insertions(+), 36 deletions(-) diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index f10ec10..e127e4b 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -4,13 +4,14 @@ #include "options.h" #include "packed_db.h" // PackedDB +#include // S_IRUSR, S_IXUSR #include #include #include // string -#include -#include // ... unlink() -#include // S_IRUSR, S_IXUSR #include // chmod() +#include // ... unlink() +#include // pair<> +#include static void grid_start(const char* const prog, const ReadsCorrectionOptions &options, const int i) { // create grid script, have grid run it @@ -106,8 +107,13 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { - PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); - partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); + //PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); + //partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); + std::vector read_order; + std::vector > read_info; + make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info); + PackedDB::convert_fasta_to_ordered_db(rco.reads, "fasta.db", read_info, read_order); + partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, read_order); } else { partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); } diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 620f238..b65d89e 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -3,10 +3,13 @@ #include #include #include +#include // stat(), struct stat #include +#include // make_pair(), pair<> #include "overlaps_store.h" #include "reads_correction_aux.h" +#include "packed_db.h" // PackedDB #define error_and_exit(msg) { std::cerr << msg << "\n"; abort(); } @@ -108,7 +111,7 @@ static idx_t get_num_reads(const char* const candidates_file) { return max_id + 1; } -static void normalise_candidate(const ExtensionCandidate& src, ExtensionCandidate& dst, const bool subject_is_target) { +static void normalize_candidate(const ExtensionCandidate& src, ExtensionCandidate& dst, const bool subject_is_target) { if (subject_is_target) { dst = src; } else { @@ -116,14 +119,10 @@ static void normalise_candidate(const ExtensionCandidate& src, ExtensionCandidat dst.qid = src.sid; dst.qext = src.sext; dst.qsize = src.ssize; - dst.qoff = src.soff; - dst.qend = src.send; dst.sdir = src.qdir; dst.sid = src.qid; dst.sext = src.qext; dst.ssize = src.qsize; - dst.soff = src.qoff; - dst.send = src.qend; dst.score = src.score; } if (dst.sdir == REV) { @@ -147,6 +146,9 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec, nec; + // not set by >> + ec.qoff = ec.soff = ec.qend = ec.send = 0; + nec.qoff = nec.soff = nec.qend = nec.send = 0; // and here we go through the input file num_batches times, // being limited by the number of open output files we can have for (; i < num_batches; i += prw.kNumFiles) { @@ -169,16 +171,81 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m if (ec.qsize < min_read_size || ec.ssize < min_read_size) { continue; } - // not set by >> - ec.qoff = ec.soff = ec.qend = ec.send = 0; if (L <= ec.qid && ec.qid < R) { - normalise_candidate(ec, nec, false); + normalize_candidate(ec, nec, false); if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { prw.checkpoint(in.tellg()); } } if (L <= ec.sid && ec.sid < R) { - normalise_candidate(ec, nec, true); + normalize_candidate(ec, nec, true); + if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { + prw.checkpoint(in.tellg()); + } + } + } + for (int k(0); k < nf; ++k) { + fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); + if (prw.counts[k] != 0) { + idx_file << prw.file_names[k] << "\n"; + } + } + prw.CloseFiles(); + } + close_fstream(idx_file); + prw.finalize(); +} + +void partition_candidates_reorder(const std::string& input, const idx_t batch_size, const int num_files, const std::vector& read_order) { + DynamicTimer dtimer(__func__); + PartitionResultsWriter prw(num_files); + idx_t i(0); + off_t input_pos; + int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); + if (!is_restart) { + prw.num_reads = read_order.size(); + } + const idx_t num_batches((prw.num_reads + batch_size - 1) / batch_size); + std::string idx_file_name; + generate_partition_index_file_name(input.c_str(), idx_file_name); + std::ofstream idx_file; + open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); + ExtensionCandidate ec, nec; + // not set by >> + ec.qoff = ec.soff = ec.qend = ec.send = 0; + nec.qoff = nec.soff = nec.qend = nec.send = 0; + // and here we go through the input file num_batches times, + // being limited by the number of open output files we can have + for (; i < num_batches; i += prw.kNumFiles) { + const idx_t sfid(i); + const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + const int nf(efid - sfid); + const idx_t L(batch_size * sfid); + const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + std::ifstream in; + open_fstream(in, input.c_str(), std::ios::in); + if (is_restart) { + if (!in.seekg(input_pos)) { + ERROR("Input seek failed while restoring checkpoint: %s", input.c_str()); + } + is_restart = 0; + } else { + prw.OpenFiles(sfid, efid, input, generate_partition_file_name, "partition.done"); + } + while (in >> ec) { + ec.qid = read_order[ec.qid]; + ec.sid = read_order[ec.sid]; + if (ec.qid == -1 || ec.sid == -1) { + continue; + } + if (L <= ec.qid && ec.qid < R) { + normalize_candidate(ec, nec, false); + if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { + prw.checkpoint(in.tellg()); + } + } + if (L <= ec.sid && ec.sid < R) { + normalize_candidate(ec, nec, true); if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { prw.checkpoint(in.tellg()); } @@ -258,3 +325,104 @@ void load_partition_files_info(const char* const idx_file_name, std::vector& read_order, std::vector >& read_info) { + std::ifstream in; + // if file already exists, just read it in + if (access(sort_file_name.c_str(), F_OK) == 0) { + struct stat buf; + if (stat(sort_file_name.c_str(), &buf) == -1) { + ERROR("Could not stat read reorder file: %s", sort_file_name.c_str()); + } + read_order.resize(buf.st_size / sizeof(idx_t)); + open_fstream(in, sort_file_name.c_str(), std::ios::in); + if (!in.read((char *)(&read_order[0]), buf.st_size)) { + ERROR("Error reading read reorder file: %s", sort_file_name.c_str()); + } + close_fstream(in); + PackedDB::read_index(pac_prefix, read_info); + return; + } + DynamicTimer dtimer(__func__); + // first read in candidates and find all read-read pairings + // (using u4_t to reduce memory footprint, but limits us to 2^32 reads) + std::vector > aligns; + std::vector read_sizes(num_reads); + ExtensionCandidate ec; + open_fstream(in, input.c_str(), std::ios::in); + while (in >> ec) { + // screen out small reads + if (ec.qsize >= min_size && ec.ssize >= min_size) { + read_sizes[ec.qid] = ec.qsize; + read_sizes[ec.sid] = ec.ssize; + aligns.push_back(std::make_pair(ec.qid, ec.sid)); + aligns.push_back(std::make_pair(ec.sid, ec.qid)); + } + } + close_fstream(in); + std::sort(aligns.begin(), aligns.end()); + // generate index into aligns + std::vector aligns_index(num_reads, -1); + const idx_t end_i(aligns.size()); + for (idx_t i(0); i != end_i;) { + const idx_t start(i); + const u4_t read_id(aligns[i].first); + for (++i; i != end_i && aligns[i].first == read_id; ++i) { } + if (i - start > min_cov) { + aligns_index[read_id] = start; + } + } + // generate the new read order + std::vector used(num_reads, 0); + std::vector new_order; // [new_read_id] = old_read_id + new_order.reserve(num_reads); + for (size_t next_unused(0), next_search(0);;) { + // skip over used reads, reads with too few alignments + for (; next_unused != used.size() && (used[next_unused] || aligns_index[next_unused] == -1); ++next_unused) { } + if (next_unused == used.size()) { + break; + } + used[next_unused] = 1; + new_order.push_back(next_unused); + // add all reads aligned to, and aligned to those, and so on + for (; next_search != new_order.size(); ++next_search) { + const idx_t sid(new_order[next_search]); + idx_t i(aligns_index[sid]); + if (i != -1) { + for (; i != end_i && aligns[i].first == sid; ++i) { + const u4_t qid(aligns[i].second); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); + } + } + } + } + } + // now reverse new_order into read_order; + // also, generate index for reordered read database + read_order.assign(num_reads, -1); + read_info.resize(new_order.size()); + idx_t total_size(0); + size_t new_rid(0); + const size_t end_new_rid(new_order.size()); + for (; new_rid != end_new_rid; ++new_rid) { + const idx_t old_rid(new_order[new_rid]); + read_order[old_rid] = new_rid; + std::pair& b(read_info[new_rid]); + b.first = total_size; + b.second = read_sizes[old_rid]; + total_size += (b.second + 3) / 4; + } + PackedDB::create_index(pac_prefix, read_info); + const std::string sort_file_name_tmp(sort_file_name + ".tmp"); + std::ofstream out; + open_fstream(out, sort_file_name_tmp.c_str(), std::ios::out | std::ios::binary); + if (!out.write((char *)(&read_order[0]), sizeof(idx_t) * read_order.size())) { + ERROR("Error writing to read reorder file: %s", sort_file_name_tmp.c_str()); + } + close_fstream(out); + if (rename(sort_file_name_tmp.c_str(), sort_file_name.c_str()) == -1) { + ERROR("Could not rename read reorder file: %s", sort_file_name_tmp.c_str()); + } +} diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 607bff3..3e06a56 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -1,8 +1,9 @@ #ifndef OVERLAPS_PARTITION_H #define OVERLAPS_PARTITION_H -#include // vector<> #include // string +#include // pair<> +#include // vector<> #include "../common/alignment.h" @@ -14,6 +15,11 @@ void partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t b void partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); +void partition_candidates_reorder(const std::string& input, idx_t batch_size, int num_files, const std::vector& read_order); + void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); +void make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_info); + + #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 44f84e0..ec0b41e 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -31,7 +31,7 @@ void PackedDB::add_one_seq(const Sequence& seq) { } const Sequence::str_t& org_seq(seq.sequence()); const u1_t* const table(get_dna_encode_table()); - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion + unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion for (idx_t i(0); i < si.size; ++i, ++db_size) { const u1_t c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); @@ -92,12 +92,12 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& const std::string ckpt_name(pac_name + ".ckpt"); const std::string ckpt_name_tmp(ckpt_name + ".tmp"); std::ofstream pout, iout; - unsigned int rand_char(-1); // spread out unknown sequence in a repeatable fashion - off_t file_offset(0), fasta_offset, index_offset; - if (check_conversion_restart(ckpt_name, fasta_offset, file_offset, index_offset, rand_char)) { + unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion + off_t pac_offset(0), fasta_offset, index_offset; + if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, index_offset, rand_char)) { // don't truncate on restart open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); - pout.seekp(file_offset); + pout.seekp(pac_offset); open_fstream(iout, index_name_tmp.c_str(), std::ios::out | std::ios::in); iout.seekp(index_offset); fr.seekg(fasta_offset); @@ -116,7 +116,10 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& // can't skip entries in index or read ids won't // match ones from candidates, so put dummy size // in ones we don't use - iout << file_offset << "\t0\n"; + iout << pac_offset << "\t0\n"; + if (!iout) { + ERROR("Error writing to pac index file: %s", index_name_tmp.c_str()); + } continue; } Sequence::str_t& s(read.sequence()); @@ -130,14 +133,17 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& if (!pout.write((char*)buffer, rbytes)) { ERROR("Write error to file %s", pac_name_tmp.c_str()); } - iout << file_offset << "\t" << rsize << "\n"; - file_offset += rbytes; + iout << pac_offset << "\t" << rsize << "\n"; + if (!iout) { + ERROR("Error writing to pac index file: %s", index_name_tmp.c_str()); + } + pac_offset += rbytes; if (time(0) >= next_checkpoint_time) { - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), file_offset, iout.tellp(), rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char); next_checkpoint_time = time(0) + 300; } } - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), file_offset, iout.tellp(), rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char); close_fstream(pout); close_fstream(iout); if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { @@ -255,3 +261,98 @@ idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t } return i; } + +void PackedDB::create_index(const std::string& output_prefix, const std::vector >& index) { + const std::string index_name(output_prefix + ".idx"); + const std::string index_name_tmp(index_name + ".tmp"); + std::ofstream out; + open_fstream(out, index_name_tmp.c_str(), std::ios::out); + std::vector >::const_iterator a(index.begin()); + const std::vector >::const_iterator end_a(index.end()); + for (; a != end_a; ++a) { + out << a->first << "\t" << a->second << "\n"; + if (!out) { + ERROR("Error writing to pac index file: %s", index_name_tmp.c_str()); + } + } + close_fstream(out); + if (rename(index_name_tmp.c_str(), index_name.c_str()) == -1) { + ERROR("Error renaming pac index file: %s", index_name_tmp.c_str()); + } +} + +void PackedDB::read_index(const std::string& output_prefix, std::vector >& index) { + const std::string index_name(output_prefix + ".idx"); + std::ifstream in; + open_fstream(in, index_name.c_str(), std::ios::in); + idx_t i, j; + while (in >> i >> j) { + index.push_back(std::make_pair(i, j)); + } +} + +// we don't need to generate the index, and we'll be writing to the output +// file somewhat randomly + +void PackedDB::convert_fasta_to_ordered_db(const std::string& fasta, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order) { + DynamicTimer dtimer(__func__); + const std::string pac_name(output_prefix + ".pac"); + // see if we already did this + if (access(pac_name.c_str(), F_OK) == 0) { + return; + } + u1_t buffer[MAX_SEQ_SIZE]; + const u1_t* const et(get_dna_encode_table()); + FastaReader fr(fasta.c_str()); + const std::string pac_name_tmp(pac_name + ".tmp"); + const std::string ckpt_name(pac_name + ".ckpt"); + const std::string ckpt_name_tmp(ckpt_name + ".tmp"); + std::ofstream pout; + unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion + off_t pac_offset, fasta_offset, old_rid(0); + if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, old_rid, rand_char)) { + // don't truncate, but file does need to exist already + open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); + fr.seekg(fasta_offset); + } else { + open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::binary); + } + Sequence read; + time_t next_checkpoint_time(time(0) + 300); + for (;; ++old_rid) { + const idx_t rsize(fr.read_one_seq(read)); + if (rsize == -1) { + break; + } + const idx_t new_rid(read_order[old_rid]); + if (new_rid == -1) { // we don't use this read, so skip + continue; + } + const std::pair& info(index[new_rid]); + assert(info.second == rsize); + Sequence::str_t& s(read.sequence()); + const idx_t rbytes((rsize + 3) / 4); + // set_char uses | to set bits, so clear first + bzero(buffer, rbytes); + for (idx_t i(0); i < rsize; ++i) { + const u1_t c(et[static_cast(s[i])]); + set_char(buffer, i, c < 4 ? c : ++rand_char & 3); + } + if (!pout.seekp(info.first)) { + ERROR("Failed to seek to position: %s", pac_name_tmp.c_str()); + } + if (!pout.write((char*)buffer, rbytes)) { + ERROR("Write error to file %s", pac_name_tmp.c_str()); + } + if (time(0) >= next_checkpoint_time) { + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char); + next_checkpoint_time = time(0) + 300; + } + } + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char); + close_fstream(pout); + if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { + ERROR("Could not rename tmp database file"); + } + unlink(ckpt_name.c_str()); +} diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 86d85a3..809905f 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -4,6 +4,8 @@ #include // ifstream #include // set<> #include // string +#include // pair<> +#include // vector<> #include "../common/defs.h" #include "../common/sequence.h" // Sequence @@ -25,6 +27,10 @@ class PackedDB { // only call one of load_fasta_db and open_db exactly once void load_fasta_db(const char* fasta); static void convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); + // create a prospective index file for pac file to be written in random order + static void create_index(const std::string& output_prefix, const std::vector >& index); + static void read_index(const std::string& output_prefix, std::vector >& index); + static void convert_fasta_to_ordered_db(const std::string& fasta, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order); // opens data file, reads in index file void open_db(const std::string& filename, idx_t memory_footprint); // returns number of candidates that can be processed diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 6c2e2b7..7e933ff 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -52,7 +52,7 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // ecs per pass, with limited read space; also filters list to exclude // candidates of reads with low size or coverage -static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov, const idx_t min_size) { +static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov) { idx_t total_ec(0); std::vector index(num_reads); // index existing list by sid @@ -60,9 +60,10 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, const idx_t start(i); const idx_t sid(ec_list[i].sid); for (++i; i != nec && ec_list[i].sid == sid; ++i) { } - // skip candidates that aren't up to snuff + // make sure we have enough coverage + // (don't need to check size, that happened during partition) const idx_t count(i - start); - if (count >= min_cov && ec_list[start].ssize >= min_size) { + if (count >= min_cov) { index[sid] = EC_Index(start, count); total_ec += count; } @@ -85,13 +86,19 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, // add all reads aligned to, and aligned to those, and so on for (; static_cast(next_search) != new_order.size(); ++next_search) { const idx_t sid(new_order[next_search]); - idx_t i(index[sid].offset); - const idx_t end_i(i + index[sid].count); - for (; i != end_i; ++i) { - const idx_t qid(ec_list[i].qid); - if (!used[qid]) { - used[qid] = 1; - new_order.push_back(qid); + const EC_Index& a(index[sid]); + // only include reads that align to this one if we plan to + // actually run those alignments (that is, if it has sufficient + // coverage) + if (a.count >= min_cov) { + idx_t i(a.offset); + const idx_t end_i(i + a.count); + for (; i != end_i; ++i) { + const idx_t qid(ec_list[i].qid); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); + } } } } @@ -116,7 +123,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu idx_t nec; ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); - ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov, ceil(data.rco.min_size * 0.95)); + ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); pthread_t thread_ids[data.rco.num_threads]; while (data.ec_offset != nec) { // see how many candidates we can run, given From 7ef87a82c42ff18b1e2ad84d38cb53d9a0d4f817 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 4 Apr 2019 01:34:04 -0500 Subject: [PATCH 12/56] added branch for number of reads vs u4_t if there are too many reads to reorder up front, check for it and fail back to the older method (i.e., splitting candidates by read id and reordering inside each partition) --- src/mecat2cns/main.cpp | 15 ++- src/mecat2cns/overlaps_partition.cpp | 176 ++++++++++++++++++------- src/mecat2cns/overlaps_partition.h | 4 +- src/mecat2cns/overlaps_store.h | 2 +- src/mecat2cns/reads_correction_can.cpp | 5 +- 5 files changed, 143 insertions(+), 59 deletions(-) diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index e127e4b..47786de 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -107,13 +107,18 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { - //PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); - //partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); std::vector read_order; std::vector > read_info; - make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info); - PackedDB::convert_fasta_to_ordered_db(rco.reads, "fasta.db", read_info, read_order); - partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, read_order); + std::vector align_counts; + const int reorder_worked(make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info, align_counts)); + if (reorder_worked) { + PackedDB::convert_fasta_to_ordered_db(rco.reads, "fasta.db", read_info, read_order); + partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, rco.reads_to_correct, read_order, align_counts); + } else { + LOG(stderr, "Too many reads to reorder, failing back to old method\n"); + PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); + partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); + } } else { partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); } diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index b65d89e..5b802da 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -131,7 +131,7 @@ static void normalize_candidate(const ExtensionCandidate& src, ExtensionCandidat } } -void partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, const int num_files, idx_t num_reads) { +void partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, const int num_files, const idx_t num_reads) { DynamicTimer dtimer(__func__); PartitionResultsWriter prw(num_files); idx_t i(0); @@ -196,32 +196,55 @@ void partition_candidates(const char* input, const idx_t batch_size, const int m prw.finalize(); } -void partition_candidates_reorder(const std::string& input, const idx_t batch_size, const int num_files, const std::vector& read_order) { +// assign reads to files in read order so that each file +// has about the same number of candidates + +static void allocate_reads_to_files(const idx_t num_batches, const std::vector& align_counts, std::vector& read_to_file) { + idx_t total_aligns(0); + std::vector::const_iterator a(align_counts.begin()); + const std::vector::const_iterator end_a(align_counts.end()); + for (; a != end_a; ++a) { + total_aligns += *a; + } + idx_t read_id(0), align_count(0); + for (int batch(0); batch != num_batches; ++batch) { + const idx_t want_aligns((total_aligns - align_count) / (num_batches - batch) + align_count); + for (; align_count < want_aligns; ++read_id) { + // skip over reads we're not correcting, as they might + // actually have alignments (which we want to ignore) + if (align_counts[read_id]) { + align_count += align_counts[read_id]; + read_to_file[read_id] = batch; + } + } + } +} + +void partition_candidates_reorder(const std::string& input, const idx_t batch_size, const int num_files, const idx_t num_reads, const std::vector& read_order, const std::vector& align_counts) { DynamicTimer dtimer(__func__); PartitionResultsWriter prw(num_files); idx_t i(0); off_t input_pos; int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); if (!is_restart) { - prw.num_reads = read_order.size(); + prw.num_reads = num_reads ? num_reads : read_order.size(); } - const idx_t num_batches((prw.num_reads + batch_size - 1) / batch_size); + const int num_batches((prw.num_reads + batch_size - 1) / batch_size); + std::vector read_to_file(prw.num_reads, -1); + allocate_reads_to_files(num_batches, align_counts, read_to_file); std::string idx_file_name; generate_partition_index_file_name(input.c_str(), idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec, nec; - // not set by >> + // not set by >> or normalize_candidate() ec.qoff = ec.soff = ec.qend = ec.send = 0; nec.qoff = nec.soff = nec.qend = nec.send = 0; // and here we go through the input file num_batches times, // being limited by the number of open output files we can have for (; i < num_batches; i += prw.kNumFiles) { - const idx_t sfid(i); - const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); - const int nf(efid - sfid); - const idx_t L(batch_size * sfid); - const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + const int sfid(i); + const int efid(std::min(sfid + prw.kNumFiles, num_batches)); std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); if (is_restart) { @@ -238,19 +261,22 @@ void partition_candidates_reorder(const std::string& input, const idx_t batch_si if (ec.qid == -1 || ec.sid == -1) { continue; } - if (L <= ec.qid && ec.qid < R) { + const int qfile(read_to_file[ec.qid]); + if (sfid <= qfile && qfile < efid) { normalize_candidate(ec, nec, false); - if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { + if (prw.WriteOneResult(qfile, ec.qid, nec)) { prw.checkpoint(in.tellg()); } } - if (L <= ec.sid && ec.sid < R) { + const int sfile(read_to_file[ec.sid]); + if (sfid <= sfile && sfile < efid) { normalize_candidate(ec, nec, true); - if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { + if (prw.WriteOneResult(sfile, ec.sid, nec)) { prw.checkpoint(in.tellg()); } } } + const int nf(efid - sfid); for (int k(0); k < nf; ++k) { fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); if (prw.counts[k] != 0) { @@ -326,60 +352,79 @@ void load_partition_files_info(const char* const idx_file_name, std::vector& read_order, std::vector >& read_info) { - std::ifstream in; - // if file already exists, just read it in - if (access(sort_file_name.c_str(), F_OK) == 0) { - struct stat buf; - if (stat(sort_file_name.c_str(), &buf) == -1) { - ERROR("Could not stat read reorder file: %s", sort_file_name.c_str()); - } - read_order.resize(buf.st_size / sizeof(idx_t)); - open_fstream(in, sort_file_name.c_str(), std::ios::in); - if (!in.read((char *)(&read_order[0]), buf.st_size)) { - ERROR("Error reading read reorder file: %s", sort_file_name.c_str()); - } - close_fstream(in); - PackedDB::read_index(pac_prefix, read_info); - return; +// encapsulate the huge vector (aligns) so it goes out of scope and the +// memory space recovered as soon as possible + +static int find_new_read_order(const std::string& input, const idx_t num_reads, const int min_size, const int min_cov, std::vector& new_order, std::vector& read_sizes, std::vector& align_counts) { + const idx_t max_reads(std::numeric_limits::max()); + if (max_reads < num_reads) { + return 0; } - DynamicTimer dtimer(__func__); // first read in candidates and find all read-read pairings - // (using u4_t to reduce memory footprint, but limits us to 2^32 reads) + // (using u4_t to halve memory footprint, but limits us + // to 2^32-1 reads) std::vector > aligns; - std::vector read_sizes(num_reads); + struct stat buf; + if (stat(input.c_str(), &buf) == -1) { + ERROR("Could not stat candidate file: %s", input.c_str()); + } + // likely underestimate (by a lot), but a better start than nothing + aligns.reserve(buf.st_size / 64); ExtensionCandidate ec; + std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); + // Note: expanding read_sizes on the fly can be slow, so we + // might want to put in some way to get the full number of reads + // and allocate up front + // Note: it'd be nice if we could add checkpointing to this loop + // as it'll take a while to process the entire candidates file while (in >> ec) { - // screen out small reads - if (ec.qsize >= min_size && ec.ssize >= min_size) { - read_sizes[ec.qid] = ec.qsize; + // screen out small reads, aligns between + // reads we don't care about + if (ec.qsize >= min_size && ec.ssize >= min_size && (ec.qid < num_reads || ec.sid < num_reads)) { + if (ec.qid < num_reads) { + aligns.push_back(std::make_pair(ec.qid, ec.sid)); + } else if (max_reads < ec.qid) { + return 0; + } else { + read_sizes.resize(ec.qid + 1); + } + if (ec.sid < num_reads) { + aligns.push_back(std::make_pair(ec.sid, ec.qid)); + } else if (max_reads < ec.sid) { + return 0; + } else { + read_sizes.resize(ec.sid + 1); + } read_sizes[ec.sid] = ec.ssize; - aligns.push_back(std::make_pair(ec.qid, ec.sid)); - aligns.push_back(std::make_pair(ec.sid, ec.qid)); + read_sizes[ec.qid] = ec.qsize; } } close_fstream(in); std::sort(aligns.begin(), aligns.end()); // generate index into aligns + align_counts.assign(num_reads, 0); std::vector aligns_index(num_reads, -1); const idx_t end_i(aligns.size()); for (idx_t i(0); i != end_i;) { const idx_t start(i); const u4_t read_id(aligns[i].first); for (++i; i != end_i && aligns[i].first == read_id; ++i) { } - if (i - start > min_cov) { + if (i - start >= min_cov) { aligns_index[read_id] = start; + align_counts[read_id] = i - start; } } - // generate the new read order - std::vector used(num_reads, 0); - std::vector new_order; // [new_read_id] = old_read_id - new_order.reserve(num_reads); - for (size_t next_unused(0), next_search(0);;) { + // generate the new read order; + // make sure to make space for all reads that + // could be used, not just those we're correcting + std::vector used(read_sizes.size(), 0); + new_order.reserve(read_sizes.size()); + size_t next_search(0); + for (idx_t next_unused(0);;) { // skip over used reads, reads with too few alignments - for (; next_unused != used.size() && (used[next_unused] || aligns_index[next_unused] == -1); ++next_unused) { } - if (next_unused == used.size()) { + for (; next_unused != num_reads && (used[next_unused] || aligns_index[next_unused] == -1); ++next_unused) { } + if (next_unused == num_reads) { break; } used[next_unused] = 1; @@ -399,20 +444,50 @@ void make_read_sort_order(const std::string& input, const std::string& sort_file } } } + return 1; +} + +int make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_size, const int min_cov, std::vector& read_order, std::vector >& read_info, std::vector& align_counts) { + // if file already exists, just read it in + if (access(sort_file_name.c_str(), F_OK) == 0) { + struct stat buf; + if (stat(sort_file_name.c_str(), &buf) == -1) { + ERROR("Could not stat read reorder file: %s", sort_file_name.c_str()); + } + read_order.resize(buf.st_size / sizeof(idx_t)); + std::ifstream in; + open_fstream(in, sort_file_name.c_str(), std::ios::in); + if (!in.read((char *)(&read_order[0]), buf.st_size)) { + ERROR("Error reading read reorder file: %s", sort_file_name.c_str()); + } + close_fstream(in); + PackedDB::read_index(pac_prefix, read_info); + return 1; + } + DynamicTimer dtimer(__func__); + std::vector new_order; // [new_read_id] = old_read_id + std::vector read_sizes(num_reads); + std::vector presort_align_counts; + if (!find_new_read_order(input, num_reads, min_size, min_cov, new_order, read_sizes, presort_align_counts)) { + return 0; + } // now reverse new_order into read_order; // also, generate index for reordered read database - read_order.assign(num_reads, -1); + read_order.assign(read_sizes.size(), -1); read_info.resize(new_order.size()); + align_counts.assign(new_order.size(), 0); idx_t total_size(0); - size_t new_rid(0); - const size_t end_new_rid(new_order.size()); - for (; new_rid != end_new_rid; ++new_rid) { + const idx_t end_new_rid(new_order.size()); + for (idx_t new_rid(0); new_rid != end_new_rid; ++new_rid) { const idx_t old_rid(new_order[new_rid]); read_order[old_rid] = new_rid; std::pair& b(read_info[new_rid]); b.first = total_size; b.second = read_sizes[old_rid]; total_size += (b.second + 3) / 4; + if (old_rid < num_reads) { + align_counts[new_rid] = presort_align_counts[old_rid]; + } } PackedDB::create_index(pac_prefix, read_info); const std::string sort_file_name_tmp(sort_file_name + ".tmp"); @@ -425,4 +500,5 @@ void make_read_sort_order(const std::string& input, const std::string& sort_file if (rename(sort_file_name_tmp.c_str(), sort_file_name.c_str()) == -1) { ERROR("Could not rename read reorder file: %s", sort_file_name_tmp.c_str()); } + return 1; } diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 3e06a56..090f1d8 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -15,11 +15,11 @@ void partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t b void partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); -void partition_candidates_reorder(const std::string& input, idx_t batch_size, int num_files, const std::vector& read_order); +void partition_candidates_reorder(const std::string& input, idx_t batch_size, int num_files, idx_t num_reads, const std::vector& read_order, const std::vector& align_counts); void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); -void make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_info); +int make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_info, std::vector& align_counts); #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 9ed5e6e..ca02e58 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -30,7 +30,7 @@ template class PartitionResultsWriter { ~PartitionResultsWriter() { CloseFiles(); } - void OpenFiles(const idx_t sfid, const idx_t efid, const std::string& prefix, file_name_generator fng, const std::string& done_file) { + void OpenFiles(const int sfid, const int efid, const std::string& prefix, file_name_generator fng, const std::string& done_file) { CloseFiles(); if (efid <= sfid) { return; diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 7e933ff..ac4cba6 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -123,7 +123,10 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu idx_t nec; ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); - ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); + if (access("reads.order", F_OK) != 0) { + // if we didn't reorder already, do it here + ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); + } pthread_t thread_ids[data.rco.num_threads]; while (data.ec_offset != nec) { // see how many candidates we can run, given From c7bee6c853b3d36ff7e250320f5df862b545b275 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 9 Apr 2019 13:17:55 -0500 Subject: [PATCH 13/56] added options for reordering and preprocessing mainly to make reordering optional for now, as it needs more work - it's too memory intensive for something that's supposed to mainly be used when memory is low. Also made sure checks for minimum coverage were always applied regardless of what processing options were chosen. --- src/mecat2cns/main.cpp | 22 ++++++++---- src/mecat2cns/mecat_correction.cpp | 2 +- src/mecat2cns/options.cpp | 23 +++++++++++- src/mecat2cns/options.h | 2 ++ src/mecat2cns/overlaps_partition.cpp | 16 +++++++++ src/mecat2cns/overlaps_store.h | 23 ++++++------ src/mecat2cns/packed_db.cpp | 43 +++++++++++++++------- src/mecat2cns/packed_db.h | 3 +- src/mecat2cns/reads_correction_can.cpp | 49 +++++++++++++++++--------- 9 files changed, 133 insertions(+), 50 deletions(-) diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 47786de..97de172 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -107,16 +107,24 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { - std::vector read_order; - std::vector > read_info; - std::vector align_counts; - const int reorder_worked(make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info, align_counts)); - if (reorder_worked) { + if (rco.preprocess_reads) { + const idx_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); + if (rco.reads_to_correct <= 0 || n_reads < rco.reads_to_correct) { + rco.reads_to_correct = n_reads; + } + } + if (rco.reorder_reads) { + std::vector read_order; + std::vector > read_info; + std::vector align_counts; + // XXX - all these need to be rewritten - account for + // preprocessing (do size checks up front), don't use + // excessive memory, write to disk as you go to allow + // checkpointing + make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info, align_counts); PackedDB::convert_fasta_to_ordered_db(rco.reads, "fasta.db", read_info, read_order); partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, rco.reads_to_correct, read_order, align_counts); } else { - LOG(stderr, "Too many reads to reorder, failing back to old method\n"); - PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size); partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); } } else { diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index cfd3f35..3032ac4 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -456,7 +456,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); const idx_t sext(ec.sext); const idx_t qext(ec.qdir == FWD ? ec.qext : ec.qsize - 1 - ec.qext); - const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr.data(), qext, ec.qsize, tstr.data(), sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) { if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { ++num_added; diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index b6ce744..f1b798f 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -31,6 +31,8 @@ static int default_tech = TECH_PACBIO; static int num_partition_files = 0; static int full_reads = 0; static idx_t read_buffer_size = 0; +static int preprocess_reads = 0; +static int reorder_reads = 0; static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -49,6 +51,8 @@ static const char reads_to_correct_n = 'R'; static const char grid_start_delay_n = 'D'; static const char full_reads_n = 'F'; static const char read_buffer_size_n = 'b'; +static const char preprocess_reads_n = 'P'; +static const char reorder_reads_n = 'O'; void print_pacbio_default_options() @@ -123,6 +127,12 @@ make_options(const ConsensusOptions& options) if (options.read_buffer_size) { cmd << " -" << read_buffer_size_n << " " << options.read_buffer_size; } + if (options.preprocess_reads) { + cmd << " -" << preprocess_reads_n; + } + if (options.reorder_reads) { + cmd << " -" << reorder_reads_n; + } cmd << " " << options.m4; cmd << " " << options.reads; cmd << " " << options.corrected_reads; @@ -150,6 +160,8 @@ void print_usage(const char* prog) { << "-" << grid_start_delay_n << " \tseconds to delay between starting grid jobs\n" << "-" << full_reads_n << "\t\toutput full reads, not just the corrected parts\n" << "-" << read_buffer_size_n << " \tbytes of memory to buffer reads [no buffer]\n" + << "-" << preprocess_reads_n << "\t\tconvert reads from fasta to fasta db before processing (implied by -" << read_buffer_size_n << " and -" << reorder_reads_n << ")\n" + << "-" << reorder_reads_n << "\t\treorder reads before processing to improve memory efficiency\n" << "-" << usage_n << "\t\tprint usage info\n" << "\n" << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; @@ -172,6 +184,8 @@ ConsensusOptions init_consensus_options(const int tech) { t.grid_start_delay = 0; t.full_reads = full_reads; t.read_buffer_size = read_buffer_size; + t.preprocess_reads = preprocess_reads; + t.reorder_reads = reorder_reads; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; t.num_threads = num_threads_pacbio; @@ -230,7 +244,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { int opt_char; char err_char; opterr = 0; - while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:")) != -1) { + while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:PO")) != -1) { switch (opt_char) { case input_type_n: if (optarg[0] == '0') { @@ -288,6 +302,13 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { break; case read_buffer_size_n: t.read_buffer_size = atoll(optarg); + // implies preprocess_reads, so fall through here + case preprocess_reads_n: + t.preprocess_reads = 1; + break; + case reorder_reads_n: + t.preprocess_reads = 1; + t.reorder_reads = 1; break; case '?': err_char = (char)optopt; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index f17a3df..6711fa4 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -29,6 +29,8 @@ struct ConsensusOptions int grid_start_delay; int full_reads; idx_t read_buffer_size; + int preprocess_reads; + int reorder_reads; }; void diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 5b802da..2e402ca 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -352,6 +352,15 @@ void load_partition_files_info(const char* const idx_file_name, std::vector fasta db to get the read sizes first, so we don't have to create a read +// size index here (and checkpoint it), but instead can read in the fasta db index and +// double check the read sizes as we go + // encapsulate the huge vector (aligns) so it goes out of scope and the // memory space recovered as soon as possible @@ -447,6 +456,13 @@ static int find_new_read_order(const std::string& input, const idx_t num_reads, return 1; } +// create a new order for reads to speed up pulling in reads for alignment +// processing (for conditions where memory is not sufficient to hold all +// reads in memory); generates a file with the old -> new read ordering, +// also generates an index for a fasta db file with the reads in the new +// order (to allow easy conversion of the original fasta to a fasta db +// with the new read ordering) + int make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_size, const int min_cov, std::vector& read_order, std::vector >& read_info, std::vector& align_counts) { // if file already exists, just read it in if (access(sort_file_name.c_str(), F_OK) == 0) { diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index ca02e58..2b37ef5 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -83,7 +83,7 @@ template class PartitionResultsWriter { if (results[i].size() == kStoreSize) { write_buffer_to_disk(i); results[i].clear(); - if (is_checkpoint_time()) { + if (time(0) >= next_checkpoint_time_) { return 1; } } @@ -145,6 +145,16 @@ template class PartitionResultsWriter { if (rename(ckpt_file_tmp_.c_str(), ckpt_file_.c_str()) == -1) { LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file_.c_str()); } + next_checkpoint_time_ = time(0) + 300; + } + idx_t total_count() const { + idx_t total(0); + std::vector::const_iterator a(counts.begin()); + const std::vector::const_iterator end_a(counts.end()); + for (; a != end_a; ++a) { + total += *a; + } + return total; } private: idx_t batch_start_ ; @@ -153,15 +163,6 @@ template class PartitionResultsWriter { std::string ckpt_file_; std::string ckpt_file_tmp_; private: - int is_checkpoint_time() { - const time_t current_time(time(0)); - if (current_time < next_checkpoint_time_ || current_time == static_cast(-1)) { - return 0; - } else { - next_checkpoint_time_ = current_time + 300; - return 1; - } - } void allocate_data(const std::string& prefix, file_name_generator fng, const int is_restart) { // allocate about a gb of memory as buffer, split among num_open_files kStoreSize = (1 << 30) / sizeof(T) / num_open_files; @@ -174,7 +175,7 @@ template class PartitionResultsWriter { const std::string tmp_file(file_names[i] + ".tmp"); if (is_restart) { if (access(file_names[i].c_str(), F_OK) == 0) { - // already finished + // mark as already finished file_names[i].clear(); // use /dev/null to prevent write errors open_fstream(files[i], "/dev/null", std::ios::binary); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index ec0b41e..491637b 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -47,25 +47,25 @@ void PackedDB::load_fasta_db(const char* const dbname) { } } -static int check_conversion_restart(const std::string& ckpt_file, off_t& fasta_offset, off_t& pac_offset, off_t& index_offset, unsigned int& rand_char) { +static int check_conversion_restart(const std::string& ckpt_file, off_t& fasta_offset, off_t& pac_offset, off_t& index_offset, unsigned int& rand_char, size_t& read_count) { std::ifstream in(ckpt_file.c_str()); if (!in) { return 0; } - in >> fasta_offset >> pac_offset >> index_offset >> rand_char; + in >> fasta_offset >> pac_offset >> index_offset >> rand_char >> read_count; if (!in) { ERROR("Read error while restoring checkpoint from %s", ckpt_file.c_str()); } return 1; } -static void checkpoint_conversion(const std::string& ckpt_file, const std::string& ckpt_file_tmp, const off_t fasta_offset, const off_t pac_offset, const off_t index_offset, const unsigned int rand_char) { +static void checkpoint_conversion(const std::string& ckpt_file, const std::string& ckpt_file_tmp, const off_t fasta_offset, const off_t pac_offset, const off_t index_offset, const unsigned int rand_char, size_t read_count) { std::ofstream out(ckpt_file_tmp.c_str()); if (!out) { LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp.c_str()); return; } - out << fasta_offset << " " << pac_offset << " " << index_offset << " " << rand_char << "\n"; + out << fasta_offset << " " << pac_offset << " " << index_offset << " " << rand_char << " " << read_count << "\n"; if (!out) { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp.c_str()); return; @@ -76,12 +76,23 @@ static void checkpoint_conversion(const std::string& ckpt_file, const std::strin } } -void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const idx_t min_size) { +size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const idx_t min_size) { const std::string pac_name(output_prefix + ".pac"); const std::string index_name(output_prefix + ".idx"); + size_t read_count(0); // see if we already did this if (access(pac_name.c_str(), F_OK) == 0 && access(index_name.c_str(), F_OK) == 0) { - return; + // get number of reads from end of database + std::ifstream pin; + open_fstream(pin, pac_name.c_str(), std::ios::in); + if (!pin.seekg(-sizeof(size_t), std::ios_base::end)) { + ERROR("Could not seek to end of fasta db to get size\n"); + } + if (!pin.read((char*)&read_count, sizeof(size_t))) { + ERROR("Could not read fasta db to get size\n"); + } + close_fstream(pin); + return read_count; } DynamicTimer dtimer(__func__); u1_t buffer[MAX_SEQ_SIZE]; @@ -94,7 +105,7 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& std::ofstream pout, iout; unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion off_t pac_offset(0), fasta_offset, index_offset; - if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, index_offset, rand_char)) { + if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, index_offset, rand_char, read_count)) { // don't truncate on restart open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); pout.seekp(pac_offset); @@ -112,6 +123,7 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& if (rsize == -1) { break; } + ++read_count; if (rsize < min_size) { // can't skip entries in index or read ids won't // match ones from candidates, so put dummy size @@ -139,11 +151,14 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& } pac_offset += rbytes; if (time(0) >= next_checkpoint_time) { - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char, read_count); next_checkpoint_time = time(0) + 300; } } - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), pac_offset, iout.tellp(), rand_char, read_count); + if (!pout.write((char*)&read_count, sizeof(size_t))) { + ERROR("Write error to file %s", pac_name_tmp.c_str()); + } close_fstream(pout); close_fstream(iout); if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { @@ -153,6 +168,7 @@ void PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& ERROR("Could not rename tmp database index file"); } unlink(ckpt_name.c_str()); + return read_count; } void PackedDB::open_db(const std::string& path, const idx_t size) { @@ -295,12 +311,12 @@ void PackedDB::read_index(const std::string& output_prefix, std::vector >& index, const std::vector& read_order) { - DynamicTimer dtimer(__func__); const std::string pac_name(output_prefix + ".pac"); // see if we already did this if (access(pac_name.c_str(), F_OK) == 0) { return; } + DynamicTimer dtimer(__func__); u1_t buffer[MAX_SEQ_SIZE]; const u1_t* const et(get_dna_encode_table()); FastaReader fr(fasta.c_str()); @@ -310,7 +326,8 @@ void PackedDB::convert_fasta_to_ordered_db(const std::string& fasta, const std:: std::ofstream pout; unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion off_t pac_offset, fasta_offset, old_rid(0); - if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, old_rid, rand_char)) { + size_t read_count(0); + if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, old_rid, rand_char, read_count)) { // don't truncate, but file does need to exist already open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); fr.seekg(fasta_offset); @@ -345,11 +362,11 @@ void PackedDB::convert_fasta_to_ordered_db(const std::string& fasta, const std:: ERROR("Write error to file %s", pac_name_tmp.c_str()); } if (time(0) >= next_checkpoint_time) { - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char, read_count); next_checkpoint_time = time(0) + 300; } } - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char); + checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char, read_count); close_fstream(pout); if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { ERROR("Could not rename tmp database file"); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 809905f..3d6b5cf 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -26,7 +26,8 @@ class PackedDB { } // only call one of load_fasta_db and open_db exactly once void load_fasta_db(const char* fasta); - static void convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); + // returns number of reads + static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); // create a prospective index file for pac file to be written in random order static void create_index(const std::string& output_prefix, const std::vector >& index); static void read_index(const std::string& output_prefix, std::vector >& index); diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index ac4cba6..73a5e28 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -20,6 +20,10 @@ static void* reads_correction_func_can(void* const arg) { const idx_t start(i); const idx_t sid(candidates[start].sid); for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } + // still have to check this here, as it's not always checked earlier + if (i - start < data.rco.min_cov) { + continue; + } ns_meap_cns::consensus_one_read_can_pacbio(data, pdata, sid, start, i); if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { data.write_buffer(tid, i); @@ -30,6 +34,9 @@ static void* reads_correction_func_can(void* const arg) { const idx_t start(i); const idx_t sid(candidates[start].sid); for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } + if (i - start < data.rco.min_cov) { + continue; + } ns_meap_cns::consensus_one_read_can_nanopore(data, pdata, sid, start, i); if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { data.write_buffer(tid, i); @@ -50,7 +57,7 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // reorder list so reads are more concentrated and we can process more // ecs per pass, with limited read space; also filters list to exclude -// candidates of reads with low size or coverage +// candidates of reads with low coverage static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov) { idx_t total_ec(0); @@ -68,23 +75,23 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, total_ec += count; } } - nec = total_ec; - ExtensionCandidate* new_list(new ExtensionCandidate[nec]); + ExtensionCandidate* new_list(new ExtensionCandidate[total_ec]); // generate the new read order - std::vector used(index.size(), 0); + std::vector used(num_reads, 0); std::vector new_order; - new_order.reserve(index.size()); - idx_t next_unused(0), next_search(0); + new_order.reserve(num_reads); + idx_t next_unused(0); + size_t next_search(0); for (;;) { // skip over used reads, reads with no alignments - for (; static_cast(next_unused) != used.size() && (used[next_unused] || index[next_unused].count == 0); ++next_unused) { } - if (static_cast(next_unused) == used.size()) { + for (; next_unused != num_reads && (used[next_unused] || index[next_unused].count == 0); ++next_unused) { } + if (next_unused == num_reads) { break; } used[next_unused] = 1; new_order.push_back(next_unused); // add all reads aligned to, and aligned to those, and so on - for (; static_cast(next_search) != new_order.size(); ++next_search) { + for (; next_search != new_order.size(); ++next_search) { const idx_t sid(new_order[next_search]); const EC_Index& a(index[sid]); // only include reads that align to this one if we plan to @@ -115,6 +122,8 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, } } delete[] ec_list; + // only do this at the end, once we know we'll successfully complete + nec = total_ec; return new_list; } @@ -123,15 +132,20 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu idx_t nec; ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); - if (access("reads.order", F_OK) != 0) { - // if we didn't reorder already, do it here - ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); + // if we're memory limited and we didn't already reorder, do it here; + // spend some cpu time to reduce number of passes + if (data.rco.read_buffer_size && !data.rco.reorder_reads) { + // don't die if we run out of memory, just do it the slow way + try { + ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); + } catch (std::bad_alloc &except) { } } pthread_t thread_ids[data.rco.num_threads]; while (data.ec_offset != nec) { // see how many candidates we can run, given // how much read sequence we can load into memory - const idx_t ecs(data.reads.load_reads(ec_list + data.ec_offset, nec - data.ec_offset)); + // (unless we're not limited, in which case load 'em all) + const idx_t ecs(data.rco.read_buffer_size ? data.reads.load_reads(ec_list + data.ec_offset, nec - data.ec_offset) : nec); allocate_ecs(data, ec_list + data.ec_offset, ecs); for (int i(0); i != data.rco.num_threads; ++i) { pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(&data)); @@ -139,7 +153,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu for (int i(0); i != data.rco.num_threads; ++i) { pthread_join(thread_ids[i], NULL); } - // don't update offset before threading to prevent checkpointing with new value + // update offset after threading to avoid checkpointing with new value data.ec_offset += ecs; data.reset_threads(); } @@ -180,8 +194,11 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; - //reads.load_fasta_db(rco.reads); - reads.open_db("fasta.db", rco.read_buffer_size); + if (rco.preprocess_reads) { // load from converted fasta file + reads.open_db("fasta.db", rco.read_buffer_size); + } else { // load from fasta file + reads.load_fasta_db(rco.reads); + } if (rco.job_index != -1) { return reads_correction_can_p(rco, partition_file_vec, reads); } else { From 4a2f1d6c25568a9c8080a7b41054c529e1fd51c2 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 10 Apr 2019 11:47:52 -0500 Subject: [PATCH 14/56] more prep towards reordering properly --- src/mecat2cns/main.cpp | 13 ++- src/mecat2cns/overlaps_partition.cpp | 106 +++++++----------- src/mecat2cns/overlaps_partition.h | 3 +- src/mecat2cns/packed_db.cpp | 149 +++++++++++++++++++------ src/mecat2cns/packed_db.h | 4 +- src/mecat2cns/reads_correction_can.cpp | 2 +- 6 files changed, 165 insertions(+), 112 deletions(-) diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 97de172..d1afcf0 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -114,15 +114,18 @@ int main(int argc, char** argv) { } } if (rco.reorder_reads) { - std::vector read_order; - std::vector > read_info; - std::vector align_counts; + std::vector read_order; // [old_rid] = new_rid + // [read id] = (offset, size) + std::vector > read_index; + std::vector align_counts; // [new_rid] = aligns // XXX - all these need to be rewritten - account for // preprocessing (do size checks up front), don't use // excessive memory, write to disk as you go to allow // checkpointing - make_read_sort_order(rco.m4, "reads.order", "fasta.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_info, align_counts); - PackedDB::convert_fasta_to_ordered_db(rco.reads, "fasta.db", read_info, read_order); + // + // XXX - does not set aligns_count on restart + make_read_sort_order(rco.m4, "fasta.db", "reads.order", "fasta_ordered.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_index, align_counts); + PackedDB::convert_db_to_ordered_db("fasta.db", "fasta_ordered.db", read_index, read_order); partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, rco.reads_to_correct, read_order, align_counts); } else { partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 2e402ca..b9d807c 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -206,9 +206,11 @@ static void allocate_reads_to_files(const idx_t num_batches, const std::vector> ec) { - ec.qid = read_order[ec.qid]; + ec.qid = read_order[ec.qid]; // convert to new read order ec.sid = read_order[ec.sid]; if (ec.qid == -1 || ec.sid == -1) { continue; @@ -356,57 +359,26 @@ void load_partition_files_info(const char* const idx_file_name, std::vector fasta db to get the read sizes first, so we don't have to create a read -// size index here (and checkpoint it), but instead can read in the fasta db index and -// double check the read sizes as we go +// gamble on having enough memory to hold the array? -// encapsulate the huge vector (aligns) so it goes out of scope and the -// memory space recovered as soon as possible - -static int find_new_read_order(const std::string& input, const idx_t num_reads, const int min_size, const int min_cov, std::vector& new_order, std::vector& read_sizes, std::vector& align_counts) { - const idx_t max_reads(std::numeric_limits::max()); - if (max_reads < num_reads) { - return 0; - } +static void generate_new_read_order(const std::string& input, const idx_t num_reads, const int min_size, const int min_cov, const std::vector& old_read_sizes, std::vector& new_order, std::vector& align_counts) { // first read in candidates and find all read-read pairings - // (using u4_t to halve memory footprint, but limits us - // to 2^32-1 reads) - std::vector > aligns; - struct stat buf; - if (stat(input.c_str(), &buf) == -1) { - ERROR("Could not stat candidate file: %s", input.c_str()); - } - // likely underestimate (by a lot), but a better start than nothing - aligns.reserve(buf.st_size / 64); + std::vector > aligns; ExtensionCandidate ec; std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); - // Note: expanding read_sizes on the fly can be slow, so we - // might want to put in some way to get the full number of reads - // and allocate up front - // Note: it'd be nice if we could add checkpointing to this loop - // as it'll take a while to process the entire candidates file while (in >> ec) { - // screen out small reads, aligns between - // reads we don't care about + // screen out small reads, aligns between reads we don't care about; + // we could use old_read_sizes[id] instead, but as ec is right here, + // should be faster to do the comparison with it if (ec.qsize >= min_size && ec.ssize >= min_size && (ec.qid < num_reads || ec.sid < num_reads)) { + r_assert(old_read_sizes[ec.sid] == ec.ssize && old_read_sizes[ec.qid] == ec.qsize); if (ec.qid < num_reads) { - aligns.push_back(std::make_pair(ec.qid, ec.sid)); - } else if (max_reads < ec.qid) { - return 0; - } else { - read_sizes.resize(ec.qid + 1); + aligns.push_back(std::make_pair(ec.qid, ec.sid)); } if (ec.sid < num_reads) { - aligns.push_back(std::make_pair(ec.sid, ec.qid)); - } else if (max_reads < ec.sid) { - return 0; - } else { - read_sizes.resize(ec.sid + 1); + aligns.push_back(std::make_pair(ec.sid, ec.qid)); } - read_sizes[ec.sid] = ec.ssize; - read_sizes[ec.qid] = ec.qsize; } } close_fstream(in); @@ -417,7 +389,7 @@ static int find_new_read_order(const std::string& input, const idx_t num_reads, const idx_t end_i(aligns.size()); for (idx_t i(0); i != end_i;) { const idx_t start(i); - const u4_t read_id(aligns[i].first); + const idx_t read_id(aligns[i].first); for (++i; i != end_i && aligns[i].first == read_id; ++i) { } if (i - start >= min_cov) { aligns_index[read_id] = start; @@ -427,8 +399,8 @@ static int find_new_read_order(const std::string& input, const idx_t num_reads, // generate the new read order; // make sure to make space for all reads that // could be used, not just those we're correcting - std::vector used(read_sizes.size(), 0); - new_order.reserve(read_sizes.size()); + std::vector used(old_read_sizes.size(), 0); + new_order.reserve(old_read_sizes.size()); size_t next_search(0); for (idx_t next_unused(0);;) { // skip over used reads, reads with too few alignments @@ -441,19 +413,21 @@ static int find_new_read_order(const std::string& input, const idx_t num_reads, // add all reads aligned to, and aligned to those, and so on for (; next_search != new_order.size(); ++next_search) { const idx_t sid(new_order[next_search]); - idx_t i(aligns_index[sid]); - if (i != -1) { - for (; i != end_i && aligns[i].first == sid; ++i) { - const u4_t qid(aligns[i].second); - if (!used[qid]) { - used[qid] = 1; - new_order.push_back(qid); + // we only use aligns to first num_reads reads + if (sid < num_reads) { + idx_t i(aligns_index[sid]); + if (i != -1) { + for (; i != end_i && aligns[i].first == sid; ++i) { + const idx_t qid(aligns[i].second); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); + } } } } } } - return 1; } // create a new order for reads to speed up pulling in reads for alignment @@ -463,7 +437,7 @@ static int find_new_read_order(const std::string& input, const idx_t num_reads, // order (to allow easy conversion of the original fasta to a fasta db // with the new read ordering) -int make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_size, const int min_cov, std::vector& read_order, std::vector >& read_info, std::vector& align_counts) { +void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_size, const int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts) { // if file already exists, just read it in if (access(sort_file_name.c_str(), F_OK) == 0) { struct stat buf; @@ -477,35 +451,34 @@ int make_read_sort_order(const std::string& input, const std::string& sort_file_ ERROR("Error reading read reorder file: %s", sort_file_name.c_str()); } close_fstream(in); - PackedDB::read_index(pac_prefix, read_info); - return 1; + PackedDB::read_index(pac_prefix, read_index); + return; } DynamicTimer dtimer(__func__); + std::vector old_read_sizes; // pre-reorder sizes + PackedDB::read_sizes(old_pac_prefix, old_read_sizes); std::vector new_order; // [new_read_id] = old_read_id - std::vector read_sizes(num_reads); std::vector presort_align_counts; - if (!find_new_read_order(input, num_reads, min_size, min_cov, new_order, read_sizes, presort_align_counts)) { - return 0; - } + generate_new_read_order(input, num_reads, min_size, min_cov, old_read_sizes, new_order, presort_align_counts); // now reverse new_order into read_order; // also, generate index for reordered read database - read_order.assign(read_sizes.size(), -1); - read_info.resize(new_order.size()); + read_order.assign(old_read_sizes.size(), -1); + read_index.resize(new_order.size()); align_counts.assign(new_order.size(), 0); idx_t total_size(0); const idx_t end_new_rid(new_order.size()); for (idx_t new_rid(0); new_rid != end_new_rid; ++new_rid) { const idx_t old_rid(new_order[new_rid]); read_order[old_rid] = new_rid; - std::pair& b(read_info[new_rid]); + std::pair& b(read_index[new_rid]); b.first = total_size; - b.second = read_sizes[old_rid]; + b.second = old_read_sizes[old_rid]; total_size += (b.second + 3) / 4; if (old_rid < num_reads) { align_counts[new_rid] = presort_align_counts[old_rid]; } } - PackedDB::create_index(pac_prefix, read_info); + PackedDB::create_index(pac_prefix, read_index); const std::string sort_file_name_tmp(sort_file_name + ".tmp"); std::ofstream out; open_fstream(out, sort_file_name_tmp.c_str(), std::ios::out | std::ios::binary); @@ -516,5 +489,4 @@ int make_read_sort_order(const std::string& input, const std::string& sort_file_ if (rename(sort_file_name_tmp.c_str(), sort_file_name.c_str()) == -1) { ERROR("Could not rename read reorder file: %s", sort_file_name_tmp.c_str()); } - return 1; } diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 090f1d8..5961a15 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -19,7 +19,6 @@ void partition_candidates_reorder(const std::string& input, idx_t batch_size, in void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); -int make_read_sort_order(const std::string& input, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_info, std::vector& align_counts); - +void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts); #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 491637b..7085b40 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -174,27 +174,38 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string void PackedDB::open_db(const std::string& path, const idx_t size) { destroy(); const std::string pac_name(path + ".pac"); - struct stat buf; - if (stat(pac_name.c_str(), &buf) == -1) { - ERROR("Could not stat fasta db file: %s", pac_name.c_str()); + open_fstream(pstream, pac_name.c_str(), std::ios::in); + // get number of reads from end of database + if (!pstream.seekg(-sizeof(size_t), std::ios_base::end)) { + ERROR("Could not seek to end of fasta db to get size\n"); } - max_db_size = std::min(idx_t(buf.st_size), size); + const idx_t file_size(pstream.tellg()); + max_db_size = std::min(file_size, size); if (max_db_size) { safe_calloc(pac, u1_t, max_db_size); } - open_fstream(pstream, pac_name.c_str(), std::ios::in); + size_t read_count; + if (!pstream.read((char*)&read_count, sizeof(size_t))) { + ERROR("Could not read fasta db to get size\n"); + } + seq_idx.reserve(read_count); + pstream.clear(); // clear eof + if (!pstream.seekg(0, std::ios_base::beg)) { + ERROR("Could not reset to start of fast db file\n"); + } const std::string index_name(path + ".idx"); std::ifstream index; open_fstream(index, index_name.c_str(), std::ios::in); SeqIndex si; - if (max_db_size == buf.st_size && max_db_size) { // read it all! + if (max_db_size == file_size && max_db_size) { // read it all! while (index >> si.file_offset >> si.size) { si.memory_offset = si.file_offset * 4; seq_idx.push_back(si); } close_fstream(index); - std::streambuf* sb(pstream.rdbuf()); - sb_read(sb, pac, max_db_size); + if (!pstream.read((char*)pac, max_db_size)) { + ERROR("Error reading fasta database\n"); + } close_fstream(pstream); } else { si.memory_offset = -1; @@ -205,6 +216,21 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { } } +const char* PackedDB::load_read(const idx_t read_id) { + const SeqIndex& si(seq_idx[read_id]); + if (!pstream.is_open()) { // all in memory already + return (char*)pac + si.memory_offset / 4; + } + if (!pstream.seekg(si.file_offset)) { + ERROR("Error seeking on fasta db"); + } + const idx_t bytes((si.size + 3) / 4); + if (!pstream.read((char*)pac, bytes)) { + ERROR("Error reading fasta db"); + } + return (char*)pac; +} + idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t nec) { if (!pstream.is_open()) { // all in memory already return nec; @@ -298,8 +324,22 @@ void PackedDB::create_index(const std::string& output_prefix, const std::vector< } void PackedDB::read_index(const std::string& output_prefix, std::vector >& index) { + const std::string pac_name(output_prefix + ".pac"); + std::ifstream in(pac_name.c_str()); + // pre-allocate index if possible + if (in.is_open()) { + // get number of reads from end of database + if (!in.seekg(-sizeof(size_t), std::ios_base::end)) { + ERROR("Could not seek to end of fasta db to get size\n"); + } + size_t read_count; + if (!in.read((char*)&read_count, sizeof(size_t))) { + ERROR("Could not read fasta db to get size\n"); + } + in.close(); + index.reserve(read_count); + } const std::string index_name(output_prefix + ".idx"); - std::ifstream in; open_fstream(in, index_name.c_str(), std::ios::in); idx_t i, j; while (in >> i >> j) { @@ -307,66 +347,103 @@ void PackedDB::read_index(const std::string& output_prefix, std::vector& sizes) { + const std::string pac_name(output_prefix + ".pac"); + std::ifstream in(pac_name.c_str()); + // pre-allocate index if possible + if (in.is_open()) { + // get number of reads from end of database + if (!in.seekg(-sizeof(size_t), std::ios_base::end)) { + ERROR("Could not seek to end of fasta db to get size\n"); + } + size_t read_count; + if (!in.read((char*)&read_count, sizeof(size_t))) { + ERROR("Could not read fasta db to get size\n"); + } + in.close(); + sizes.reserve(read_count); + } + const std::string index_name(output_prefix + ".idx"); + open_fstream(in, index_name.c_str(), std::ios::in); + idx_t i, j; + while (in >> i >> j) { + sizes.push_back(j); + } +} + +static int check_reorder_restart(const std::string& ckpt_file, size_t& rid) { + std::ifstream in(ckpt_file.c_str()); + if (!in) { + return 0; + } + in >> rid; + if (!in) { + ERROR("Read error while restoring checkpoint from %s", ckpt_file.c_str()); + } + return 1; +} + +static void checkpoint_reorder(const std::string& ckpt_file, const std::string& ckpt_file_tmp, const size_t rid) { + std::ofstream out(ckpt_file_tmp.c_str()); + if (!out) { + LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp.c_str()); + return; + } + out << rid << "\n"; + if (!out) { + LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp.c_str()); + return; + } + out.close(); + if (rename(ckpt_file_tmp.c_str(), ckpt_file.c_str()) == -1) { + LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file.c_str()); + } +} + // we don't need to generate the index, and we'll be writing to the output // file somewhat randomly -void PackedDB::convert_fasta_to_ordered_db(const std::string& fasta, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order) { +void PackedDB::convert_db_to_ordered_db(const std::string& input_prefix , const std::string& output_prefix, const std::vector >& index, const std::vector& read_order) { const std::string pac_name(output_prefix + ".pac"); // see if we already did this if (access(pac_name.c_str(), F_OK) == 0) { return; } DynamicTimer dtimer(__func__); - u1_t buffer[MAX_SEQ_SIZE]; - const u1_t* const et(get_dna_encode_table()); - FastaReader fr(fasta.c_str()); const std::string pac_name_tmp(pac_name + ".tmp"); const std::string ckpt_name(pac_name + ".ckpt"); const std::string ckpt_name_tmp(ckpt_name + ".tmp"); + PackedDB old_reads; + old_reads.open_db(input_prefix, MAX_SEQ_SIZE); std::ofstream pout; - unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion - off_t pac_offset, fasta_offset, old_rid(0); - size_t read_count(0); - if (check_conversion_restart(ckpt_name, fasta_offset, pac_offset, old_rid, rand_char, read_count)) { + size_t old_rid(0); + if (check_reorder_restart(ckpt_name, old_rid)) { // don't truncate, but file does need to exist already open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); - fr.seekg(fasta_offset); } else { open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::binary); } - Sequence read; time_t next_checkpoint_time(time(0) + 300); - for (;; ++old_rid) { - const idx_t rsize(fr.read_one_seq(read)); - if (rsize == -1) { - break; - } + for (; old_rid != read_order.size(); ++old_rid) { const idx_t new_rid(read_order[old_rid]); if (new_rid == -1) { // we don't use this read, so skip continue; } + const char* seq(old_reads.load_read(old_rid)); const std::pair& info(index[new_rid]); - assert(info.second == rsize); - Sequence::str_t& s(read.sequence()); - const idx_t rbytes((rsize + 3) / 4); - // set_char uses | to set bits, so clear first - bzero(buffer, rbytes); - for (idx_t i(0); i < rsize; ++i) { - const u1_t c(et[static_cast(s[i])]); - set_char(buffer, i, c < 4 ? c : ++rand_char & 3); - } + const idx_t rbytes((info.second + 3) / 4); if (!pout.seekp(info.first)) { ERROR("Failed to seek to position: %s", pac_name_tmp.c_str()); } - if (!pout.write((char*)buffer, rbytes)) { + if (!pout.write(seq, rbytes)) { ERROR("Write error to file %s", pac_name_tmp.c_str()); } if (time(0) >= next_checkpoint_time) { - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char, read_count); + checkpoint_reorder(ckpt_name, ckpt_name_tmp, old_rid); next_checkpoint_time = time(0) + 300; } } - checkpoint_conversion(ckpt_name, ckpt_name_tmp, fr.tellg(), 0, old_rid, rand_char, read_count); + checkpoint_reorder(ckpt_name, ckpt_name_tmp, old_rid); close_fstream(pout); if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { ERROR("Could not rename tmp database file"); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 3d6b5cf..7f2d1a6 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -31,9 +31,11 @@ class PackedDB { // create a prospective index file for pac file to be written in random order static void create_index(const std::string& output_prefix, const std::vector >& index); static void read_index(const std::string& output_prefix, std::vector >& index); - static void convert_fasta_to_ordered_db(const std::string& fasta, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order); + static void read_sizes(const std::string& output_prefix, std::vector& sizes); + static void convert_db_to_ordered_db(const std::string& input_prefix, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order); // opens data file, reads in index file void open_db(const std::string& filename, idx_t memory_footprint); + const char* load_read(idx_t read_id); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidate* ec_list, idx_t nec); void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 73a5e28..8e3662c 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -195,7 +195,7 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; if (rco.preprocess_reads) { // load from converted fasta file - reads.open_db("fasta.db", rco.read_buffer_size); + reads.open_db(rco.reorder_reads ? "fasta_ordered.db" : "fasta.db", rco.read_buffer_size); } else { // load from fasta file reads.load_fasta_db(rco.reads); } From c272e9d2e37dc4e7f3555ca01128ba0c28b63e4e Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 10 Apr 2019 12:34:40 -0500 Subject: [PATCH 15/56] added option for binary output to mecat2pw --- src/mecat2pw/pw_impl.cpp | 34 +++-- src/mecat2pw/pw_options.cpp | 285 ++++++++++++++++++------------------ src/mecat2pw/pw_options.h | 32 ++-- 3 files changed, 184 insertions(+), 167 deletions(-) diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index a1534f0..9034052 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -764,12 +764,19 @@ static void candidate_detect(PWThreadData* const data, const int tid) { } if (++nec == PWThreadData::kResultListSize) { pthread_mutex_lock(&data->result_write_lock); - for (int i(0); i < nec; ++i) { - (*data->out) << eclist[i]; - if (!(*data->out)) { + if (data->options->binary_output) { + if (!data->out->write((char*)&eclist, sizeof(ExtensionCandidate) * nec)) { std::cerr << "Error writing output\n"; exit(1); } + } else { + for (int i(0); i < nec; ++i) { + (*data->out) << eclist[i]; + if (!(*data->out)) { + std::cerr << "Error writing output\n"; + exit(1); + } + } } pthread_mutex_unlock(&data->result_write_lock); nec = 0; @@ -779,12 +786,19 @@ static void candidate_detect(PWThreadData* const data, const int tid) { } if (nec) { pthread_mutex_lock(&data->result_write_lock); - for (int i(0); i < nec; ++i) { - (*data->out) << eclist[i]; - if (!(*data->out)) { + if (data->options->binary_output) { + if (!data->out->write((char*)&eclist, sizeof(ExtensionCandidate) * nec)) { std::cerr << "Error writing output\n"; exit(1); } + } else { + for (int i(0); i < nec; ++i) { + (*data->out) << eclist[i]; + if (!(*data->out)) { + std::cerr << "Error writing output\n"; + exit(1); + } + } } pthread_mutex_unlock(&data->result_write_lock); } @@ -816,13 +830,13 @@ class ProcessState { std::cerr << "Error: failed to restore from checkpoint: " << ckpt_file_name_ << "\n"; exit(1); } - // set ios_base::in to prevent truncation - out.open(volume_results_name_tmp_.c_str(), std::ios_base::out | std::ios_base::in); + // set ios::in to prevent truncation + out.open(volume_results_name_tmp_.c_str(), std::ios::out | std::ios::in | std::ios::binary); out.seekp(out_pos_); } else { vid = svid; - // can't set ios_base::in, as we need to create the file - out.open(volume_results_name_tmp_.c_str()); + // can't set ios::in, as we need to create the file + out.open(volume_results_name_tmp_.c_str(), std::ios::out | std::ios::binary); } if (!out) { std::cerr << "Could not open volume results file: " << volume_results_name_tmp_ << "\n"; diff --git a/src/mecat2pw/pw_options.cpp b/src/mecat2pw/pw_options.cpp index 1c5ad4a..ff5ee30 100644 --- a/src/mecat2pw/pw_options.cpp +++ b/src/mecat2pw/pw_options.cpp @@ -79,27 +79,28 @@ make_options(options_t* options) if (options->reads_to_correct) { cmd << " -R " << options->reads_to_correct; } + if (options->binary_output) { + cmd << " -b"; + } return cmd.str(); } -void -init_options(options_t* options, int tech) -{ - assert(options); +void init_options(options_t* const options, const int tech) { + assert(options); options->task = TASK_ALN; - options->reads = NULL; - options->output = NULL; - options->wrk_dir = NULL; - options->grid_options = NULL; - options->grid_options_split = NULL; - options->num_threads = 1; - options->num_candidates = 100; - options->output_gapped_start_point = 0; + options->reads = NULL; + options->output = NULL; + options->wrk_dir = NULL; + options->grid_options = NULL; + options->grid_options_split = NULL; + options->num_threads = 1; + options->num_candidates = 100; + options->output_gapped_start_point = 0; options->tech = tech; options->job_index = -1; options->num_vols = -1; options->reads_to_correct = 0; - + options->binary_output = 0; if (tech == TECH_PACBIO) { options->min_align_size = kDefaultAlignSizePacbio; options->min_kmer_match = kDefaultKmerMatchPacbio; @@ -131,18 +132,16 @@ void print_usage(const char* prog) fprintf(stderr, "-R \tnumber of reads to error correct [all]\n"); fprintf(stderr, "-G \tscheduler command/options\n"); fprintf(stderr, "-S \tscheduler command/options (for split)\n"); + fprintf(stderr, "-b\t\toutput file is binary rather than text\n"); fprintf(stderr, "(note that grid command/options have to start with \"qsub\" and end with \"-N\")\n"); fprintf(stderr, "(note that slurm command/options have to start with \"sbatch\" and end with \"-J\")\n"); } -int -parse_arguments(int argc, char* argv[], options_t* options) -{ - int opt_char; - char err_char; - opterr = 0; - int ret = 0; - +int parse_arguments(int argc, char* argv[], options_t* options) { + int opt_char; + char err_char; + opterr = 0; + int ret = 0; int task = -1; const char* reads = NULL; const char* output = NULL; @@ -158,61 +157,58 @@ parse_arguments(int argc, char* argv[], options_t* options) int job_index = -1; int num_vols = -1; int reads_to_correct = 0; - - while((opt_char = getopt(argc, argv, "j:d:o:w:t:n:g:x:a:k:G:i:N:R:S:")) != -1) - { - switch(opt_char) - { + int binary_output = 0; + while((opt_char = getopt(argc, argv, "j:d:o:w:t:n:g:x:a:k:G:i:N:R:S:b")) != -1) { + switch(opt_char) { case 'j': task = atoi(optarg); break; - case 'd': - reads = optarg; - break; - case 'o': - output = optarg; - break; - case 'w': - wrk_dir = optarg; - break; - case 'G': - grid_options = optarg; - break; - case 'S': - grid_options_split = optarg; - break; - case 't': - num_threads = atoi(optarg); - break; - case 'i': - job_index = atoi(optarg); - break; - case 'N': - num_vols = atoi(optarg); - break; - case 'R': - reads_to_correct = atoi(optarg); - break; - case 'n': - num_candidates = atoi(optarg); - break; + case 'd': + reads = optarg; + break; + case 'o': + output = optarg; + break; + case 'w': + wrk_dir = optarg; + break; + case 'G': + grid_options = optarg; + break; + case 'S': + grid_options_split = optarg; + break; + case 't': + num_threads = atoi(optarg); + break; + case 'i': + job_index = atoi(optarg); + break; + case 'N': + num_vols = atoi(optarg); + break; + case 'R': + reads_to_correct = atoi(optarg); + break; + case 'n': + num_candidates = atoi(optarg); + break; case 'a': min_align_size = atoi(optarg); break; case 'k': min_kmer_match = atoi(optarg); break; - case 'g': - if (optarg[0] == '0') - output_gapped_start_point = 0; - else if (optarg[0] == '1') - output_gapped_start_point = 1; - else - { - LOG(stderr, "argument to option \'-g\' must be either \'0\' or \'1\'"); - return 1; - } - break; + case 'g': + if (optarg[0] == '0') { + output_gapped_start_point = 0; + } else if (optarg[0] == '1') { + output_gapped_start_point = 1; + } else { + LOG(stderr, "argument to option \'-g\' must be either \'0\' or \'1\'"); + return 1; + } + break; case 'x': if (optarg[0] == '0') { tech = TECH_PACBIO; @@ -222,85 +218,92 @@ parse_arguments(int argc, char* argv[], options_t* options) ERROR("invalid argument to option 'x': %s", optarg); } break; - case '?': - err_char = (char)optopt; - LOG(stderr, "unrecognised option \'%c\'", err_char); - return 1; - break; - case ':': - err_char = (char)optopt; - LOG(stderr, "argument to option \'%c\' is not provided!", err_char); - return 1; - break; - } - } - + case '?': + err_char = (char)optopt; + LOG(stderr, "unrecognised option \'%c\'", err_char); + return 1; + break; + case ':': + err_char = (char)optopt; + LOG(stderr, "argument to option \'%c\' is not provided!", err_char); + return 1; + break; + } + } init_options(options, tech); - if (task != -1) options->task = task; + if (task != -1) { + options->task = task; + } options->reads = reads; options->output = output; options->wrk_dir = wrk_dir; - if (grid_options != NULL) options->grid_options = grid_options; - if (grid_options_split != NULL) options->grid_options_split = grid_options_split; - if (num_threads != -1) options->num_threads = num_threads; - if (num_candidates != -1) options->num_candidates = num_candidates; - if (min_align_size != -1) options->min_align_size = min_align_size; - if (min_kmer_match != -1) options->min_kmer_match = min_kmer_match; - if (output_gapped_start_point != -1) options->output_gapped_start_point = output_gapped_start_point; - if (job_index != -1) options->job_index = job_index; - if (num_vols != -1) options->num_vols = num_vols; - if (reads_to_correct) options->reads_to_correct = reads_to_correct; - - if (options->task != TASK_SEED && options->task != TASK_ALN) - { + if (grid_options != NULL) { + options->grid_options = grid_options; + } + if (grid_options_split != NULL) { + options->grid_options_split = grid_options_split; + } + if (num_threads != -1) { + options->num_threads = num_threads; + } + if (num_candidates != -1) { + options->num_candidates = num_candidates; + } + if (min_align_size != -1) { + options->min_align_size = min_align_size; + } + if (min_kmer_match != -1) { + options->min_kmer_match = min_kmer_match; + } + if (output_gapped_start_point != -1) { + options->output_gapped_start_point = output_gapped_start_point; + } + if (job_index != -1) { + options->job_index = job_index; + } + if (num_vols != -1) { + options->num_vols = num_vols; + } + if (reads_to_correct) { + options->reads_to_correct = reads_to_correct; + } + if (binary_output) { + options->binary_output = 1; + } + if (options->task != TASK_SEED && options->task != TASK_ALN) { LOG(stderr, "task (-j) must be %d or %d, not %d.", TASK_SEED, TASK_ALN, options->task); ret = 1; } - - if (!options->reads) - { - LOG(stderr, "dataset must be specified."); - ret = 1; - } - else if (!options->output) - { - LOG(stderr, "output must be specified."); - ret = 1; - } - else if (!options->wrk_dir) - { - LOG(stderr, "working directory must be specified."); - ret = 1; - } - else if (options->num_threads < 1) - { - LOG(stderr, "number of cpu threads must be > 0."); - ret = 1; - } - else if (options->num_candidates < 1) - { - LOG(stderr, "number of candidates must be > 0."); - ret = 1; - } - else if (options->reads_to_correct < 0) - { - LOG(stderr, "reads to correct must be > 0."); - ret = 1; - } - - if (ret) return ret; - - DIR* dir = opendir(options->wrk_dir); - if (dir == NULL) - { - int t = mkdir(options->wrk_dir, S_IRWXU); - if (t == -1) - { - LOG(stderr, "fail to create folder \'%s\'!", options->wrk_dir); - exit(1); - } - } - else closedir(dir); - - return ret; + if (!options->reads) { + LOG(stderr, "dataset must be specified."); + ret = 1; + } else if (!options->output) { + LOG(stderr, "output must be specified."); + ret = 1; + } else if (!options->wrk_dir) { + LOG(stderr, "working directory must be specified."); + ret = 1; + } else if (options->num_threads < 1) { + LOG(stderr, "number of cpu threads must be > 0."); + ret = 1; + } else if (options->num_candidates < 1) { + LOG(stderr, "number of candidates must be > 0."); + ret = 1; + } else if (options->reads_to_correct < 0) { + LOG(stderr, "reads to correct must be > 0."); + ret = 1; + } + if (ret) { + return ret; + } + DIR* dir = opendir(options->wrk_dir); + if (dir == NULL) { + if (mkdir(options->wrk_dir, S_IRWXU) == -1) { + LOG(stderr, "fail to create folder \'%s\'!", options->wrk_dir); + exit(1); + } + } else { + closedir(dir); + } + return ret; } diff --git a/src/mecat2pw/pw_options.h b/src/mecat2pw/pw_options.h index 5c5fa8c..75f902f 100644 --- a/src/mecat2pw/pw_options.h +++ b/src/mecat2pw/pw_options.h @@ -8,23 +8,23 @@ #define TASK_SEED 0 #define TASK_ALN 1 -typedef struct -{ +typedef struct { int task; - const char* reads; - const char* output; - const char* wrk_dir; - const char* grid_options; - const char* grid_options_split; - int num_threads; - int num_candidates; - int min_align_size; - int min_kmer_match; - int output_gapped_start_point; - int tech; - int num_vols; - int job_index; - int reads_to_correct; // reads must lead fasta file + const char* reads; + const char* output; + const char* wrk_dir; + const char* grid_options; + const char* grid_options_split; + int num_threads; + int num_candidates; + int min_align_size; + int min_kmer_match; + int output_gapped_start_point; + int tech; + int num_vols; + int job_index; + int reads_to_correct; // reads must lead fasta file + int binary_output; } options_t; std::string From 04003aaf4788059d4fc982cd78d7cbec860cf378 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 12 Apr 2019 18:34:06 -0500 Subject: [PATCH 16/56] shifting candidate handling to new format Changed structures to help lower memory usage, some refactoring to help add read sorting to also help with memory usage --- src/common/alignment.cpp | 3 + src/common/alignment.h | 38 ++++++ src/mecat2cns/main.cpp | 12 +- src/mecat2cns/mecat_correction.cpp | 110 ++++++++--------- src/mecat2cns/options.cpp | 17 +-- src/mecat2cns/overlaps_partition.cpp | 159 ++++++++++++------------- src/mecat2cns/overlaps_partition.h | 8 +- src/mecat2cns/overlaps_store.h | 7 +- src/mecat2cns/packed_db.cpp | 2 +- src/mecat2cns/packed_db.h | 16 ++- src/mecat2cns/reads_correction_aux.cpp | 17 +++ src/mecat2cns/reads_correction_aux.h | 23 +++- src/mecat2cns/reads_correction_can.cpp | 18 ++- src/mecat2cns/reads_correction_m4.cpp | 2 +- 14 files changed, 241 insertions(+), 191 deletions(-) diff --git a/src/common/alignment.cpp b/src/common/alignment.cpp index be8678c..aeea467 100644 --- a/src/common/alignment.cpp +++ b/src/common/alignment.cpp @@ -148,3 +148,6 @@ std::ostream& operator<<(std::ostream& out, const ReferenceMapping& rm) return out; } */ + +const int64_t ExtensionCandidateCompressed::max_value = std::numeric_limits::max(); +const int64_t ExtensionCandidateCompressed::max_qext = std::numeric_limits::max() >> 1; diff --git a/src/common/alignment.h b/src/common/alignment.h index 4046d3a..e4ae6c1 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -2,6 +2,7 @@ #define ALIGNMENT_H #include +#include // numeric_limits<> #include "defs.h" @@ -12,6 +13,43 @@ struct ExtensionCandidate int score; }; +// candidates only use a few of these values, so make a smaller structure for them +// sid and qid are used a lot, and we sort on score, so stash qdir inside msb of qext +struct ExtensionCandidateCompressed { + uint32_t sid, qid, sext, qext_, score; + int qdir() const { + return qext_ & MSB_ ? REV : FWD; + } + int qext() const { + return qext_ & ~MSB_; + } + void set_qext(const uint32_t new_qext, const int new_qdir) { + qext_ = new_qdir == FWD ? new_qext : new_qext | MSB_; + } + void set(const ExtensionCandidate& a) { + sid = a.sid; + qid = a.qid; + sext = a.sext; + // sdir is forced to FWD, so swap qdir is sdir is REV + qext_ = a.qdir == a.sdir ? a.qext : a.qext | MSB_; + score = a.score; + } + void set_swap(const ExtensionCandidate& a) { // swap s and q + sid = a.qid; + qid = a.sid; + sext = a.qext; + qext_ = a.sdir == a.qdir ? a.sext : a.sext | MSB_; + score = a.score; + } + // to check conversions from type int; need to use int64_t in case + // int is only int32_t in size (which wouldn't hold uint32_t max) + static const int64_t max_value; + // account for using MSB for qdir + static const int64_t max_qext; + private: + static const uint32_t MSB_ = 1 << (sizeof(uint32_t) * 8 - 1); +}; + std::istream& operator>>(std::istream& in, ExtensionCandidate& ec); diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index d1afcf0..fd73351 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -107,11 +107,9 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { - if (rco.preprocess_reads) { - const idx_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); - if (rco.reads_to_correct <= 0 || n_reads < rco.reads_to_correct) { - rco.reads_to_correct = n_reads; - } + const idx_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); + if (rco.reads_to_correct <= 0 || n_reads < rco.reads_to_correct) { + rco.reads_to_correct = n_reads; } if (rco.reorder_reads) { std::vector read_order; // [old_rid] = new_rid @@ -124,11 +122,11 @@ int main(int argc, char** argv) { // checkpointing // // XXX - does not set aligns_count on restart - make_read_sort_order(rco.m4, "fasta.db", "reads.order", "fasta_ordered.db", rco.reads_to_correct, rco.min_size, rco.min_cov, read_order, read_index, align_counts); + make_read_sort_order(rco.m4, "fasta.db", "reads.order", "fasta_ordered.db", rco.reads_to_correct, rco.min_cov, read_order, read_index, align_counts); PackedDB::convert_db_to_ordered_db("fasta.db", "fasta_ordered.db", read_index, read_order); partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, rco.reads_to_correct, read_order, align_counts); } else { - partition_candidates(rco.m4, rco.batch_size, rco.min_size, rco.num_partition_files, rco.reads_to_correct); + partition_candidates(rco.m4, "fasta.db", rco.batch_size, rco.num_partition_files, rco.reads_to_correct); } } else { partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 3032ac4..b7bd2bf 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -302,7 +302,7 @@ void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads(ctd.reads); - ExtensionCandidate* overlaps(pctd.candidates); + ExtensionCandidate* overlaps((ExtensionCandidate*)pctd.candidates); DiffRunningData* const drd_s(pctd.drd_s); DiffRunningData* drd(NULL); M5Record& m5(pctd.m5); @@ -349,7 +349,7 @@ void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads = ctd.reads; - ExtensionCandidate* overlaps = pctd.candidates; + ExtensionCandidate* overlaps = (ExtensionCandidate*)pctd.candidates; DiffRunningData* drd_s = pctd.drd_s; DiffRunningData* drd = NULL; M5Record& m5 = pctd.m5; @@ -423,16 +423,16 @@ check_cov_stats(u1_t* cov_stats, int soff, int send) void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { PackedDB& reads(ctd.reads); - ExtensionCandidate* candidates(pctd.candidates); + ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); DiffRunningData* const drd_s(pctd.drd_s); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t read_size(candidates[sid].ssize); + const idx_t ssize(reads.read_size(sid)); std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); - tstr.resize(read_size); - reads.GetSequence(read_id, true, tstr.data(), read_size); + tstr.resize(ssize); + reads.GetSequence(read_id, true, tstr.data(), ssize); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -441,23 +441,22 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const int max_added(60); eid = std::min(eid, sid + 200); // max of 200 extents CnsTableItem* cns_table(pctd.cns_table); - std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); + std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; u1_t* cov_stats(pctd.id_list); std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); for (idx_t i(sid); i < eid && num_added < max_added; ++i) { - ExtensionCandidate& ec(candidates[i]); - r_assert(ec.sdir == FWD); + const ExtensionCandidateCompressed& ec(candidates[i]); if (used_ids.find(ec.qid) != used_ids.end()) { continue; } - qstr.resize(ec.qsize); - reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); - const idx_t sext(ec.sext); - const idx_t qext(ec.qdir == FWD ? ec.qext : ec.qsize - 1 - ec.qext); - const bool r(GetAlignment(qstr.data(), qext, ec.qsize, tstr.data(), sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); - if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) { + const idx_t qsize(reads.read_size(ec.qid)); + qstr.resize(qsize); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data(), qsize); + const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); + const bool r(GetAlignment(qstr.data(), qext, qsize, tstr.data(), ec.sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); + if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); @@ -469,7 +468,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD } std::vector mranges, eranges; cns_vec.get_mapping_ranges(mranges); - get_effective_ranges(mranges, eranges, read_size, ctd.rco.min_size); + get_effective_ranges(mranges, eranges, ssize, ctd.rco.min_size); if (ctd.rco.full_reads) { consensus_worker_one_read(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, tstr, cns_results); } else { @@ -477,52 +476,47 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD } } -void -consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) -{ - PackedDB& reads = ctd.reads; - ExtensionCandidate* candidates = pctd.candidates; - DiffRunningData* drd_s = pctd.drd_s; - DiffRunningData* drd = NULL; - M5Record& m5 = pctd.m5; - CnsAlns& cns_vec = pctd.cns_alns; - std::vector& cns_results = pctd.cns_results; - const idx_t read_size = candidates[sid].ssize; +void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { + PackedDB& reads(ctd.reads); + ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); + DiffRunningData* const drd_s(pctd.drd_s); + DiffRunningData* drd(NULL); + M5Record& m5(pctd.m5); + CnsAlns& cns_vec(pctd.cns_alns); + std::vector& cns_results(pctd.cns_results); + const idx_t ssize(reads.read_size(sid)); std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; - tstr.resize(read_size); - reads.GetSequence(read_id, true, tstr.data(), read_size); - std::string& nqstr = pctd.qaln; - std::string& ntstr = pctd.saln; - const int min_align_size = ctd.rco.min_align_size; - const double min_mapping_ratio = ctd.rco.min_mapping_ratio - 0.02; - - int num_added = 0; - int num_ext = 0; - const int max_ext = 200; - CnsTableItem* cns_table = pctd.cns_table; - std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); + tstr.resize(ssize); + reads.GetSequence(read_id, true, tstr.data(), ssize); + std::string& nqstr(pctd.qaln); + std::string& ntstr(pctd.saln); + const int min_align_size(ctd.rco.min_align_size); + const double min_mapping_ratio(ctd.rco.min_mapping_ratio - 0.02); + int num_added(0); + int num_ext(0); + const int max_ext(200); + CnsTableItem* cns_table(pctd.cns_table); + std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; - u1_t* cov_stats = pctd.id_list; + u1_t* const cov_stats(pctd.id_list); std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); - for (idx_t i = sid; i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) - { - ++num_ext; - ExtensionCandidate& ec = candidates[i]; - r_assert(ec.sdir == FWD); - if (used_ids.find(ec.qid) != used_ids.end()) continue; - qstr.resize(ec.qsize); - reads.GetSequence(ec.qid, ec.qdir == FWD, qstr.data(), ec.qsize); - idx_t qext = ec.qext; - idx_t sext = ec.sext; - if (ec.qdir == REV) qext = ec.qsize - 1 - qext; + for (idx_t i(sid); i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { + ++num_ext; + const ExtensionCandidateCompressed& ec(candidates[i]); + if (used_ids.find(ec.qid) != used_ids.end()) { + continue; + } + const idx_t qsize(reads.read_size(ec.qid)); + qstr.resize(qsize); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data(), qsize); + const idx_t sext(ec.sext); + const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); drd = drd_s; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); - if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ec.qsize, m5soff(m5), m5send(m5), ec.ssize, min_mapping_ratio)) - { - if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) - { + const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size)); + if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { + if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); @@ -531,10 +525,8 @@ consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData } } } - std::vector mranges, eranges; - eranges.push_back(MappingRange(0, read_size)); - + eranges.push_back(MappingRange(0, ssize)); consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index f1b798f..0f6aadd 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -9,7 +9,6 @@ static int input_type_pacbio = 1; static int num_threads_pacbio = 1; -static idx_t batch_size_pacbio = 100000; static double mapping_ratio_pacbio = 0.9; static int align_size_pacbio = 2000; static int cov_pacbio = 6; @@ -19,7 +18,6 @@ static int tech_pacbio = TECH_PACBIO; static int input_type_nanopore = 1; static int num_threads_nanopore = 1; -static idx_t batch_size_nanopore = 100000; static double mapping_ratio_nanopore = 0.4; static int align_size_nanopore = 400; static int cov_nanopore = 6; @@ -33,6 +31,7 @@ static int full_reads = 0; static idx_t read_buffer_size = 0; static int preprocess_reads = 0; static int reorder_reads = 0; +static size_t batch_size = 8589934592; // 8 GB static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -59,7 +58,6 @@ print_pacbio_default_options() { std::cerr << "-" << input_type_n << " " << input_type_pacbio << " -" << num_threads_n << " " << num_threads_pacbio - << " -" << batch_size_n << " " << batch_size_pacbio << " -" << mapping_ratio_n << " " << mapping_ratio_pacbio << " -" << align_size_n << " " << align_size_pacbio << " -" << cov_n << " " << cov_pacbio << " " @@ -71,7 +69,6 @@ void print_nanopore_default_options() { std::cerr << "-" << input_type_n << " " << input_type_nanopore << " -" << num_threads_n << " " << num_threads_nanopore - << " -" << batch_size_n << " " << batch_size_nanopore << " -" << mapping_ratio_n << " " << mapping_ratio_nanopore << " -" << align_size_n << " " << align_size_nanopore << " -" << cov_n << " " << cov_nanopore @@ -88,7 +85,7 @@ make_options(const ConsensusOptions& options) if (options.num_threads > -1) { cmd << " -" << num_threads_n << " " << options.num_threads; } - if (options.batch_size > 0) { + if (options.batch_size) { cmd << " -" << batch_size_n << " " << options.batch_size; } if (options.min_mapping_ratio >= 0) { @@ -144,11 +141,10 @@ void print_usage(const char* prog) { << prog << " [options] input reads output\n" << "\n" << "OPTIONS:\n" - << "-" << tech_n << " <0/1>\tsequencing platform: 0 = PACBIO, 1 = NANOPORE\n" - << "\t\tdefault: 0\n" + << "-" << tech_n << " <0/1>\tsequencing platform: 0 = PACBIO, 1 = NANOPORE [0]\n" << "-" << input_type_n << " <0/1>\tinput type: 0 = candidate, 1 = m4\n" << "-" << num_threads_n << " \tnumber of threads (CPUs)\n" - << "-" << batch_size_n << " \tbatch size that the reads will be partitioned\n" + << "-" << batch_size_n << " \tmemory used for holding candidates [8 GB]\n" << "-" << mapping_ratio_n << " \tminimum mapping ratio\n" << "-" << align_size_n << " \tminimum overlap size\n" << "-" << cov_n << " \tminimum coverage under consideration\n" @@ -186,10 +182,10 @@ ConsensusOptions init_consensus_options(const int tech) { t.read_buffer_size = read_buffer_size; t.preprocess_reads = preprocess_reads; t.reorder_reads = reorder_reads; + t.batch_size = batch_size; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; t.num_threads = num_threads_pacbio; - t.batch_size = batch_size_pacbio; t.min_mapping_ratio = mapping_ratio_pacbio; t.min_align_size = align_size_pacbio; t.min_cov = cov_pacbio; @@ -199,7 +195,6 @@ ConsensusOptions init_consensus_options(const int tech) { } else { t.input_type = input_type_nanopore; t.num_threads = num_threads_nanopore; - t.batch_size = batch_size_nanopore; t.min_mapping_ratio = mapping_ratio_nanopore; t.min_align_size = align_size_nanopore; t.min_cov = cov_nanopore; @@ -326,7 +321,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { std::cerr << "cpu threads must be greater than 0\n"; parse_success = false; } - if (t.batch_size <= 0) { + if (t.batch_size == 0) { std::cerr << "batch size must be greater than 0\n"; parse_success = false; } diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index b9d807c..dbed64b 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -98,88 +98,65 @@ void generate_partition_file_name(const char* const m4_file_name, const idx_t pa ret += os.str(); } -static idx_t get_num_reads(const char* const candidates_file) { - std::ifstream in; - open_fstream(in, candidates_file, std::ios::in); - ExtensionCandidate ec; - int max_id(-1); - while (in >> ec) { - max_id = std::max(ec.qid, max_id); - max_id = std::max(ec.sid, max_id); - } - close_fstream(in); - return max_id + 1; -} - -static void normalize_candidate(const ExtensionCandidate& src, ExtensionCandidate& dst, const bool subject_is_target) { - if (subject_is_target) { - dst = src; - } else { - dst.qdir = src.sdir; - dst.qid = src.sid; - dst.qext = src.sext; - dst.qsize = src.ssize; - dst.sdir = src.qdir; - dst.sid = src.qid; - dst.sext = src.qext; - dst.ssize = src.qsize; - dst.score = src.score; - } - if (dst.sdir == REV) { - dst.qdir = REVERSE_STRAND(dst.qdir); - dst.sdir = REVERSE_STRAND(dst.sdir); - } -} - -void partition_candidates(const char* input, const idx_t batch_size, const int min_read_size, const int num_files, const idx_t num_reads) { +void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t batch_size, const int num_files, const idx_t num_reads) { DynamicTimer dtimer(__func__); - PartitionResultsWriter prw(num_files); + struct stat buf; + if (stat(input.c_str(), &buf) == -1) { + ERROR("Could not get file size: %s", input.c_str()); + } + // each candidate line takes approximately 44 characters, but go with 32; + // each one produces two candidates (forward and reverse) + const idx_t num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size); + const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); + std::vector read_sizes; + PackedDB::read_sizes(pac_prefix, read_sizes); + PartitionResultsWriter prw(num_files); idx_t i(0); off_t input_pos; int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); - if (!is_restart) { - prw.num_reads = num_reads ? num_reads : get_num_reads(input); - } - const idx_t num_batches((prw.num_reads + batch_size - 1) / batch_size); std::string idx_file_name; - generate_partition_index_file_name(input, idx_file_name); + generate_partition_index_file_name(input.c_str(), idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - ExtensionCandidate ec, nec; - // not set by >> - ec.qoff = ec.soff = ec.qend = ec.send = 0; - nec.qoff = nec.soff = nec.qend = nec.send = 0; + ExtensionCandidate ec; + ExtensionCandidateCompressed nec; // and here we go through the input file num_batches times, // being limited by the number of open output files we can have for (; i < num_batches; i += prw.kNumFiles) { const idx_t sfid(i); const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); - const idx_t L(batch_size * sfid); - const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + const idx_t L(sfid * reads_per_batch); + const idx_t R(efid < num_batches ? efid * reads_per_batch : num_reads); std::ifstream in; - open_fstream(in, input, std::ios::in); + open_fstream(in, input.c_str(), std::ios::in); if (is_restart) { if (!in.seekg(input_pos)) { - ERROR("Input seek failed while restoring checkpoint: %s", input); + ERROR("Input seek failed while restoring checkpoint: %s", input.c_str()); } is_restart = 0; } else { prw.OpenFiles(sfid, efid, input, generate_partition_file_name, "partition.done"); } while (in >> ec) { - if (ec.qsize < min_read_size || ec.ssize < min_read_size) { + if (!read_sizes[ec.sid] || !read_sizes[ec.qid]) { continue; } - if (L <= ec.qid && ec.qid < R) { - normalize_candidate(ec, nec, false); - if (prw.WriteOneResult((ec.qid - L) / batch_size, ec.qid, nec)) { + r_assert(ec.ssize == read_sizes[ec.sid] && ec.qsize == read_sizes[ec.qid]); + r_assert(ec.sid <= ExtensionCandidateCompressed::max_value); + r_assert(ec.qid <= ExtensionCandidateCompressed::max_value); + r_assert(ec.sext <= ExtensionCandidateCompressed::max_value); + r_assert(ec.qext <= ExtensionCandidateCompressed::max_qext); + r_assert(ec.score <= ExtensionCandidateCompressed::max_value); + if (L <= ec.sid && ec.sid < R) { + nec.set(ec); + if (prw.WriteOneResult((nec.sid - L) / reads_per_batch, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } - if (L <= ec.sid && ec.sid < R) { - normalize_candidate(ec, nec, true); - if (prw.WriteOneResult((ec.sid - L) / batch_size, ec.sid, nec)) { + if (L <= ec.qid && ec.qid < R) { + nec.set_swap(ec); + if (prw.WriteOneResult((nec.sid - L) / reads_per_batch, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } @@ -223,26 +200,27 @@ static void allocate_reads_to_files(const idx_t num_batches, const std::vector& read_order, const std::vector& align_counts) { +void partition_candidates_reorder(const std::string& input, const size_t batch_size, const int num_files, const idx_t num_reads, const std::vector& read_order, const std::vector& align_counts) { DynamicTimer dtimer(__func__); - PartitionResultsWriter prw(num_files); + struct stat buf; + if (stat(input.c_str(), &buf) == -1) { + ERROR("Could not get file size: %s", input.c_str()); + } + // each candidate line takes approximately 44 characters, but go with 32; + // each one produces two candidates (forward and reverse) + const int num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size); + PartitionResultsWriter prw(num_files); idx_t i(0); off_t input_pos; int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); - if (!is_restart) { - prw.num_reads = num_reads ? num_reads : read_order.size(); - } - const int num_batches((prw.num_reads + batch_size - 1) / batch_size); - std::vector read_to_file(prw.num_reads, -1); + std::vector read_to_file(num_reads, -1); allocate_reads_to_files(num_batches, align_counts, read_to_file); std::string idx_file_name; generate_partition_index_file_name(input.c_str(), idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - ExtensionCandidate ec, nec; - // not set by >> or normalize_candidate() - ec.qoff = ec.soff = ec.qend = ec.send = 0; - nec.qoff = nec.soff = nec.qend = nec.send = 0; + ExtensionCandidate ec; + ExtensionCandidateCompressed nec; // and here we go through the input file num_batches times, // being limited by the number of open output files we can have for (; i < num_batches; i += prw.kNumFiles) { @@ -264,17 +242,22 @@ void partition_candidates_reorder(const std::string& input, const idx_t batch_si if (ec.qid == -1 || ec.sid == -1) { continue; } - const int qfile(read_to_file[ec.qid]); - if (sfid <= qfile && qfile < efid) { - normalize_candidate(ec, nec, false); - if (prw.WriteOneResult(qfile, ec.qid, nec)) { + r_assert(ec.sid <= ExtensionCandidateCompressed::max_value); + r_assert(ec.qid <= ExtensionCandidateCompressed::max_value); + r_assert(ec.sext <= ExtensionCandidateCompressed::max_value); + r_assert(ec.qext <= ExtensionCandidateCompressed::max_qext); + r_assert(ec.score <= ExtensionCandidateCompressed::max_value); + const int sfile(read_to_file[ec.sid]); + if (sfid <= sfile && sfile < efid) { + nec.set(ec); + if (prw.WriteOneResult(sfile, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } - const int sfile(read_to_file[ec.sid]); - if (sfid <= sfile && sfile < efid) { - normalize_candidate(ec, nec, true); - if (prw.WriteOneResult(sfile, ec.sid, nec)) { + const int qfile(read_to_file[ec.qid]); + if (sfid <= qfile && qfile < efid) { + nec.set_swap(ec); + if (prw.WriteOneResult(qfile, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } @@ -292,12 +275,17 @@ void partition_candidates_reorder(const std::string& input, const idx_t batch_si prw.finalize(); } -void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const idx_t batch_size, const int min_read_size, const int num_files) { +void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const size_t batch_size, const int min_read_size, const int num_files) { DynamicTimer dtimer(__func__); idx_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); std::set repeat_reads; //get_repeat_reads(m4_file_name, min_cov_ratio, num_reads, repeat_reads); - const idx_t num_batches((num_reads + batch_size - 1) / batch_size); + struct stat buf; + if (stat(m4_file_name, &buf) == -1) { + ERROR("Could not get file size: %s", m4_file_name); + } + const idx_t num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidate) / batch_size); + const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); std::string idx_file_name; generate_partition_index_file_name(m4_file_name, idx_file_name); std::ofstream idx_file; @@ -309,8 +297,8 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra const idx_t sfid(i); const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); - const idx_t L(batch_size * sfid); - const idx_t R(efid < num_batches ? batch_size * efid : prw.num_reads); + const idx_t L(reads_per_batch * sfid); + const idx_t R(efid < num_batches ? reads_per_batch * efid : num_reads); std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name, "partition.done"); @@ -325,12 +313,12 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra if (m4qid(m4) >= L && m4qid(m4) < R) { normalize_m4record(m4, false, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4qid(m4) - L) / batch_size, m4qid(m4), ec); + prw.WriteOneResult((m4qid(m4) - L) / reads_per_batch , m4qid(m4), ec); } if (m4sid(m4) >= L && m4sid(m4) < R) { normalize_m4record(m4, true, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4sid(m4) - L) / batch_size, m4sid(m4), ec); + prw.WriteOneResult((m4sid(m4) - L) / reads_per_batch , m4sid(m4), ec); } } for (int k(0); k < nf; ++k) { @@ -361,7 +349,7 @@ void load_partition_files_info(const char* const idx_file_name, std::vector& old_read_sizes, std::vector& new_order, std::vector& align_counts) { +static void generate_new_read_order(const std::string& input, const idx_t num_reads, const int min_cov, const std::vector& old_read_sizes, std::vector& new_order, std::vector& align_counts) { // first read in candidates and find all read-read pairings std::vector > aligns; ExtensionCandidate ec; @@ -369,9 +357,8 @@ static void generate_new_read_order(const std::string& input, const idx_t num_re open_fstream(in, input.c_str(), std::ios::in); while (in >> ec) { // screen out small reads, aligns between reads we don't care about; - // we could use old_read_sizes[id] instead, but as ec is right here, - // should be faster to do the comparison with it - if (ec.qsize >= min_size && ec.ssize >= min_size && (ec.qid < num_reads || ec.sid < num_reads)) { + // use old_read_sizes as we've already screened out the small reads + if (old_read_sizes[ec.sid] && old_read_sizes[ec.qid] && (ec.qid < num_reads || ec.sid < num_reads)) { r_assert(old_read_sizes[ec.sid] == ec.ssize && old_read_sizes[ec.qid] == ec.qsize); if (ec.qid < num_reads) { aligns.push_back(std::make_pair(ec.qid, ec.sid)); @@ -437,7 +424,7 @@ static void generate_new_read_order(const std::string& input, const idx_t num_re // order (to allow easy conversion of the original fasta to a fasta db // with the new read ordering) -void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_size, const int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts) { +void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts) { // if file already exists, just read it in if (access(sort_file_name.c_str(), F_OK) == 0) { struct stat buf; @@ -459,7 +446,7 @@ void make_read_sort_order(const std::string& input, const std::string& old_pac_p PackedDB::read_sizes(old_pac_prefix, old_read_sizes); std::vector new_order; // [new_read_id] = old_read_id std::vector presort_align_counts; - generate_new_read_order(input, num_reads, min_size, min_cov, old_read_sizes, new_order, presort_align_counts); + generate_new_read_order(input, num_reads, min_cov, old_read_sizes, new_order, presort_align_counts); // now reverse new_order into read_order; // also, generate index for reordered read database read_order.assign(old_read_sizes.size(), -1); diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 5961a15..3ee276d 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -11,14 +11,14 @@ void generate_partition_index_file_name(const char* m4_file_name, std::string& r void generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret); -void partition_m4records(const char* m4_file_name, double min_cov_ratio, idx_t batch_size, int min_read_size, int num_files); +void partition_m4records(const char* m4_file_name, double min_cov_ratio, size_t batch_size, int min_read_size, int num_files); -void partition_candidates(const char* input, idx_t batch_size, int min_read_size, int num_files, idx_t num_reads = 0); +void partition_candidates(const std::string& input, const std::string& pac_prefix, size_t batch_size, int num_files, idx_t num_reads); -void partition_candidates_reorder(const std::string& input, idx_t batch_size, int num_files, idx_t num_reads, const std::vector& read_order, const std::vector& align_counts); +void partition_candidates_reorder(const std::string& input, size_t batch_size, int num_files, idx_t num_reads, const std::vector& read_order, const std::vector& align_counts); void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); -void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_size, int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts); +void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts); #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 2b37ef5..d954581 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -18,7 +18,6 @@ template class PartitionResultsWriter { const int kNumFiles; // effective open file limit int kStoreSize; int num_open_files; - int num_reads; PODArray* results; // can't use vector<>, causes memory corruption std::ofstream* files; // can't use vector<>, non-copyable std::vector file_names; @@ -26,7 +25,7 @@ template class PartitionResultsWriter { public: // can't make kNumFiles static, as sysconf() is run-time only; // leave room for stdin, stdout, stderr, a few others - explicit PartitionResultsWriter(const int num_files) : kNumFiles(num_files > 0 ? num_files : sysconf(_SC_OPEN_MAX) - 10), kStoreSize(0), num_open_files(0), num_reads(0), results(0), files(0) { } + explicit PartitionResultsWriter(const int num_files) : kNumFiles(num_files > 0 ? num_files : sysconf(_SC_OPEN_MAX) - 10), kStoreSize(0), num_open_files(0), results(0), files(0) { } ~PartitionResultsWriter() { CloseFiles(); } @@ -98,7 +97,7 @@ template class PartitionResultsWriter { if (!in) { return 0; } - in >> batch_start_ >> num_open_files >> num_reads >> input_pos; + in >> batch_start_ >> num_open_files >> input_pos; if (!in) { ERROR("Read error while restoring checkpoint from %s", ckpt_file_.c_str()); } @@ -123,7 +122,7 @@ template class PartitionResultsWriter { LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp_.c_str()); return; } - out << batch_start_ << " " << num_open_files << " " << num_reads << " " << input_pos << "\n"; + out << batch_start_ << " " << num_open_files << " " << input_pos << "\n"; if (!out) { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); return; diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 7085b40..a65d57c 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -231,7 +231,7 @@ const char* PackedDB::load_read(const idx_t read_id) { return (char*)pac; } -idx_t PackedDB::load_reads(const ExtensionCandidate* const ec_list, const idx_t nec) { +idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, const idx_t nec) { if (!pstream.is_open()) { // all in memory already return nec; } diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 7f2d1a6..f2ca674 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -37,17 +37,18 @@ class PackedDB { void open_db(const std::string& filename, idx_t memory_footprint); const char* load_read(idx_t read_id); // returns number of candidates that can be processed - idx_t load_reads(const ExtensionCandidate* ec_list, idx_t nec); + idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { - r_assert(size == seq_idx[id].size); + const SeqIndex &si(seq_idx[id]); + r_assert(size == si.size); if (forward) { - const idx_t offset(seq_idx[id].memory_offset); - for (idx_t i(0); i < size; ++i) { + const idx_t offset(si.memory_offset); + for (idx_t i(0); i < si.size; ++i) { seq[i] = get_char(offset + i); } } else { - const idx_t offset(seq_idx[id].memory_offset + size - 1); - for (idx_t i(0); i < size; ++i) { + const idx_t offset(si.memory_offset + si.size - 1); + for (idx_t i(0); i < si.size; ++i) { seq[i] = 3 - get_char(offset - i); } } @@ -55,6 +56,9 @@ class PackedDB { idx_t num_reads() const { return seq_idx.size(); } + idx_t read_size(const idx_t read_id) const { + return seq_idx[read_id].size; + } private: static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { p[idx >> 2] |= c << ((~idx & 3) << 1); diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 6346e7c..20d66cf 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -94,3 +94,20 @@ void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, data.data[k].candidates = ec_list + start; } } + +void allocate_ecs(ConsensusThreadData& data, ExtensionCandidateCompressed* const ec_list, const idx_t nec) { + const int n(data.rco.num_threads); + // split by number of ec's, rather than reads, since reads ids + // are not contiguous and we could get empty lists + for (idx_t i(0), k(0); k != n; ++k) { + const idx_t start(i); + // drop fractions here, as we'll likely add a few more ec's below + i += (nec - i) / (n - k); + if (i != nec) { // include all ec's for the last read + const int final_sid(ec_list[i].sid); + for (++i; i != nec && ec_list[i].sid == final_sid; ++i) { } + } + data.data[k].num_candidates = i - start; + data.data[k].candidates = ec_list + start; + } +} diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index b69d0b7..e2b0ad7 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -135,12 +135,32 @@ struct CmpExtensionCandidateBySidAndScore { } }; +struct CmpExtensionCandidateCompressedBySidAndScore { + bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { + if (a.sid != b.sid) { // primary sort + return a.sid < b.sid; // for splitting up in allocate_ecs() + } else if (a.score != b.score) { // secondary sort + return b.score < a.score; // process best ones first + } else if (a.qid != b.qid) { + return a.qid < b.qid; + } else if (a.qext() != b.qext()) { + return a.qext() < b.qext(); + } else if (a.sext != b.sext) { // tertiary sort + return a.sext < b.sext; // make sorting consistent + } else { + return a.qdir() < b.qdir(); + } + } +}; + class ConsensusPerThreadData { public: // num_candidates, candidates initialized by allocate_ecs() // next_candidate initialized by ConsensusThreadData::restart() idx_t num_candidates, next_candidate; - ExtensionCandidate* candidates; + // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed + // for candidate runs (to reduce memory usage) + void* candidates; ns_banded_sw::DiffRunningData* drd_s; ns_banded_sw::DiffRunningData* drd_l; CnsTableItem cns_table[MAX_SEQ_SIZE]; @@ -280,5 +300,6 @@ class ConsensusThreadData { void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, std::string& qnorm, std::string& tnorm, bool push); void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, idx_t nec); +void allocate_ecs(ConsensusThreadData &data, ExtensionCandidateCompressed* ec_list, idx_t nec); #endif // _READS_CORRECTION_AUX_H diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 8e3662c..125ac77 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -13,7 +13,7 @@ static void* reads_correction_func_can(void* const arg) { ConsensusThreadData& data(*(static_cast(arg))); const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); - const ExtensionCandidate* const candidates(pdata.candidates); + const ExtensionCandidateCompressed* const candidates((ExtensionCandidateCompressed*)pdata.candidates); idx_t i(pdata.next_candidate); if (data.rco.tech == TECH_PACBIO) { while (i != pdata.num_candidates) { @@ -59,7 +59,7 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // ecs per pass, with limited read space; also filters list to exclude // candidates of reads with low coverage -static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov) { +static ExtensionCandidateCompressed* reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov) { idx_t total_ec(0); std::vector index(num_reads); // index existing list by sid @@ -75,7 +75,7 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, total_ec += count; } } - ExtensionCandidate* new_list(new ExtensionCandidate[total_ec]); + ExtensionCandidateCompressed* new_list(new ExtensionCandidateCompressed[total_ec]); // generate the new read order std::vector used(num_reads, 0); std::vector new_order; @@ -117,7 +117,7 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, for (; a != end_a; ++a) { const EC_Index& b(index[*a]); if (b.count) { - memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidate) * b.count); + memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidateCompressed) * b.count); pos += b.count; } } @@ -130,8 +130,8 @@ static ExtensionCandidate* reorder_candidates(ExtensionCandidate* const ec_list, // load and sort partition data, assign to threads, start threads static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { idx_t nec; - ExtensionCandidate* ec_list(load_partition_data(m4_file_name, nec)); - std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); + ExtensionCandidateCompressed* ec_list(load_partition_data(m4_file_name, nec)); + std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySidAndScore()); // if we're memory limited and we didn't already reorder, do it here; // spend some cpu time to reduce number of passes if (data.rco.read_buffer_size && !data.rco.reorder_reads) { @@ -194,11 +194,7 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; - if (rco.preprocess_reads) { // load from converted fasta file - reads.open_db(rco.reorder_reads ? "fasta_ordered.db" : "fasta.db", rco.read_buffer_size); - } else { // load from fasta file - reads.load_fasta_db(rco.reads); - } + reads.open_db(rco.reorder_reads ? "fasta_ordered.db" : "fasta.db", rco.read_buffer_size); if (rco.job_index != -1) { return reads_correction_can_p(rco, partition_file_vec, reads); } else { diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index 369f10e..a7a0a97 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -12,7 +12,7 @@ void* reads_correction_func_m4(void* arg) { ConsensusThreadData& data(*(static_cast(arg))); const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); - ExtensionCandidate* const overlaps(pdata.candidates); + ExtensionCandidate* const overlaps((ExtensionCandidate*)pdata.candidates); const idx_t num_ovlps(pdata.num_candidates); idx_t i = 0, j; while (i < num_ovlps) { From e57eb43d88192c341af17579432a1390763a2d2c Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 16 Apr 2019 13:00:00 -0500 Subject: [PATCH 17/56] added MECAT2 defaults, fixed memory reduction bugs Also changed spun off processes to not bother listing options if they're defaults --- README.MD | 1360 +++++++++++++------------- src/mecat2cns/dw.cpp | 8 +- src/mecat2cns/mecat_correction.cpp | 12 +- src/mecat2cns/options.cpp | 22 +- src/mecat2cns/overlaps_partition.cpp | 9 +- src/mecat2cns/packed_db.cpp | 2 +- src/mecat2pw/pw_impl.cpp | 2 +- src/mecat2pw/pw_options.cpp | 4 +- 8 files changed, 713 insertions(+), 706 deletions(-) diff --git a/README.MD b/README.MD index aab0e20..58350ac 100644 --- a/README.MD +++ b/README.MD @@ -1,680 +1,680 @@ -**We have released a new version [MECAT2](https://github.com/xiaochuanle/MECAT2). Please go and download that new version. This version will not be updated any more.** - -# Contents - -* [Introduction](#S-introduction) - -* [Installation](#S-installation) - -* [Quick Start](#S-quick-start) - -* [Input Format](#S-input-format) - -* [Program Descriptions](#S-program-description) - -* [Citation](#S-citation) - -* [Contact](#S-contact) - -* [Update Information](#S-update) - -# Introdction - - - -MECAT is an ultra-fast Mapping, Error Correction and de novo Assembly Tools for single molecula sequencing (SMRT) reads. MECAT employs novel alignment and error correction algorithms that are much more efficient than the state of art of aligners and error correction tools. MECAT can be used for effectively de novo assemblying large genomes. For example, on a 32-thread computer with 2.0 GHz CPU , MECAT takes 9.5 days to assemble a human genome based on 54x SMRT data, which is 40 times faster than the current [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/). We also use MECAT to assemble a diploid human genome based on 102x SMRT data only in 25 days. The latter assembly leads a great improvement of quality to the previous genome assembled from the 54x haploid SMRT data. MECAT performance were compared with [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/), [FALCON](https://github.com/PacificBiosciences/falcon) and [Canu(v1.3)](http://canu.readthedocs.io/en/latest/) in five real datasets. The quality of assembled contigs produced by MECAT is the same or better than that of the [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/) and [FALCON](https://github.com/PacificBiosciences/falcon). Here are some comparisons on the 32-thread computer with 2.0 GHz CPU and 512 GB RAM memory: - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
GenomePipelineThread numberTotal running time (h)NG50 of genome
E.coliFALCON161.214,635,129
PBcR-MHAP161.294,652,272
Canu160.714,648,002
MECAT160.244,649,626
YeastFALCON162.16587,169
PBcR-MHAP164.2818,229
Canu165.11739,902
MECAT160.91929,350
A.thalianaFALCON16223.847,583,032
PBcR-MHAP16188.79,610,192
Canu16118.578,315,338
MECAT1610.7312600961
D.melanogasterFALCON16140.7215,664,372
PBcR-MHAP16101.2213,627,256
Canu1669.3414,179,324
MECAT169.5818,111,159
Human(54X)PBcR-MHAH(f)3257501,857,788
PBcR-MHAH(s)32130004,320,471
MECAT32230.544,878,957
-
- - -MECAT consists of four modules: - -* `mecat2pw`, a fast and accurate pairwise mapping tool for SMRT reads - -* `mecat2ref`, a fast and accurate reference mapping tool for SMRT reads - -* `mecat2cns`, correct noisy reads based on their pairwise overlaps - -* `mecat2canu`, a modified and more efficient version of the [Canu pipeline](https://github.com/marbl/canu). [Canu](https://github.com/marbl/canu) is a customized version of the Celera Assembler that designed for high-noise single-molecule sequencing - - - -MEAP is written in C, C++, and perl. It is open source and distributed under the [GPLv3](http://www.gnu.org/licenses/gpl-3.0.html) license. - - - -# Installation - -The current directory is `/public/users/chenying/smrt_asm`. - -* Install `MECAT`: -``` shell -git clone https://github.com/xiaochuanle/MECAT.git -cd MECAT -make -cd .. -``` -After installation, all the executables are found in `MECAT/Linux-amd64/bin`. The folder name `Linux-amd64` will vary in operating systems. For example, in MAC, the executables are put in `MECAT/Darwin-amd64/bin`. - -* Install `HDF5`: -``` shell -wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.15-patch1/src/hdf5-1.8.15-patch1.tar.gz -tar xzvf hdf5-1.8.15-patch1.tar.gz -mkdir hdf5 -cd hdf5-1.8.15-patch1 -./configure --enable-cxx --prefix=/public/users/chenying/smrt_asm/hdf5 -make -make install -cd .. -``` -The header files of `HDF5` are in `hdf5/include`. The library files of `HDF5` are in `hdf5/lib` (in some systems, they are put in `hdf5/lib64`, check it!). - -* Install `dextract` -``` shell -git clone https://github.com/PacificBiosciences/DEXTRACTOR.git -cp MECAT/dextract_makefile DEXTRACTOR -cd DEXTRACTOR -export HDF5_INCLUDE=/public/users/chenying/smrt_asm/hdf5/include -export HDF5_LIB=/public/users/chenying/smrt_asm/hdf5/lib -make -f dextract_makefile -cd .. -``` - -* Add relative pathes -``` shell -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/public/users/chenying/smrt_asm/hdf5/lib -export PATH=/public/users/chenying/smrt_asm/MECAT/Linux-amd64/bin:$PATH -export PATH=/public/users/chenying/smrt_asm/DEXTRACTOR:$PATH -``` - - -# Quick Start - - - -Using MECAT to assemble a genome involves 4 steps. Here we take assemblying the genome of Ecoli as an example, to go through each step in order. Options of each command will be given in next section. - -## Assemblying Pacbio Data - -We download the reads [ecoli_filtered.fastq.gz](http://gembox.cbcb.umd.edu/mhap/raw/ecoli_filtered.fastq.gz) from the MHAP website. By decompressing it we obtain `ecoli_filtered.fastq`. - -* Step 1, using `mecat2pw` to detect overlapping candidates - -``` shell - -mecat2pw -j 0 -d ecoli_filtered.fastq -o ecoli_filtered.fastq.pm.can -w wrk_dir -t 16 - -``` - -* Step 2, correct the noisy reads based on their pairwise overlapping candidates. - -```shell - -mecat2cns -i 0 -t 16 ecoli_filtered.fastq.pm.can ecoli_filtered.fastq corrected_ecoli_filtered - -``` - -* Step 3, extract the longest 25X corrected reads - -```shell - -extract_sequences corrected_ecoli_filtered.fasta corrected_ecoli_25x.fasta 4800000 25 - -``` - -* Step 4, assemble the longest 25X corrected reads using `mecat2cacu` - -```shell - -mecat2canu -trim-assemble -p ecoli -d ecoli genomeSize=4800000 ErrorRate=0.02 maxMemory=40 maxThreads=16 useGrid=0 Overlapper=mecat2asmpw -pacbio-corrected corrected_ecoli_25x.fasta - -``` - -## Assemblying Nanopore Data - -Download [MAP006-PCR-1_2D_pass.fasta](http://nanopore.s3.climb.ac.uk/MAP006-PCR-1_2D_pass.fasta). - -* Step 1, using `mecat2pw` to detect overlapping candidates - -``` shell - -mecat2pw -j 0 -d MAP006-PCR-1_2D_pass.fasta -o candidatex.txt -w wrk_dir -t 16 -x 1 - -``` - -* Step 2, correct the noisy reads based on their pairwise overlapping candidates. - -```shell - -mecat2cns -i 0 -t 16 -x 1 candidates.txt MAP006-PCR-1_2D_pass.fasta corrected_ecoli.fasta - -``` - -* Step 3, extract the longest 25X corrected reads - -```shell - -extract_sequences corrected_ecoli.fasta corrected_ecoli_25x.fasta 4800000 25 - -``` - -* Step 4, assemble the longest 25X corrected reads using `mecat2cacu` - -```shell - -mecat2canu -trim-assemble -p ecoli -d ecoli genomeSize=4800000 ErrorRate=0.06 maxMemory=40 maxThreads=16 useGrid=0 Overlapper=mecat2asmpw -nanopore-corrected corrected_ecoli_25x.fasta - -``` - - -After step 4, the assembled genome is given in file `ecoli/ecoli.contigs.fasta`. Details of the contigs can be found in file `ecoli/ecoli.layout.tigInfo`. - - -# Input Format - -MECAT is capable of processing `FASTA`, `FASTQ`, and `H5` format files. However, the `H5` files must first be transfered to `FASTA` -format by running `DEXTRACTOR/dextract` before running MECAT. For example: -```shell -find pathto/raw_reads -name "*.bax.h5" -exec readlink -f {} \; > reads.fofn -while read line; do dextract -v $line >> reads.fasta ; done < reads.fofn -``` -the extracted result should be the reads.fasta file for mecat's input file. - -# Program Descriptions - - - -We describe in detail each module of MECAT, including their options and output formats. - - - -## `mecat2pw` - - - -### options - -The command for running `mecat2pw` is - -```shell - -mecat2pw -j [task] -d [fasta/fastq] -w [working folder] -t [# of threads] -o [output] -n [# of candidates] -a [overlap size] -k [# of kmers] -g [0/1] -x [0/1] - -``` - -The options are: - -* `-j [task]`, job name, 0 = detect overlapping candidates only, 1 = output overlaps in M4 format, default = 1. If we are to correct noisy reads, outputing overlapping candidates is enough. - -* `-d [fasta/fastq]`, reads file name in FASTA or FASTQ format. - -* `-w [working folder]`, a directory for storing temporary results, will be created if not exists. - -* `-t [# of threads]`, number of CPU threads used for overlapping, default=1. - -* `-o [output]`, output file name - -* `-n [# of candidates]`, number of candidates considered for gapped extension, default=100. Since each chunk is about 2GB size, number of candidates(NC) should be set by genome size (GS).For GS < 20M, NC should be set as 200; For GS>20M and GS<200M; NC should be set as 100; For GS>200M, NC should be set as 50. - -* `-a [overlap size]`, only output overlaps with length >= a. Default: 2000 if x is set to 0, 500 if x is set to 1. - -* `-k [# of kmers]`, two blocks between two reads having >= k kmer matches will be considered as a matched block pair. Default: 4 if x is set to 0, 2 if x is set to 1. - -* `-g [0/1]`, output the gapped extension start point (1) or not (0), default=0. - -* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. - - -### output format - - - -If the job is detecting overlapping candidates, the results are output in `can` format, each result of which occupies one line and 9 fields: - -```shell - -[A ID] [B ID] [A strand] [B strand] [A gapped start] [B gapped start] [voting score] [A length] [B length] - -``` - - - -`mecat2pw` outputs overlapping results in `M4` format, of which one result is given in one line. The fileds of `M4` format is given in order below: - -```shell - -[A ID] [B ID] [% identity] [voting score] [A strand] [A start] [A end] [A length] [B strand] [B start] [B end] [B length] - -``` - -If the `-g` option is set to `1`, two more fields indicating the extension starting points are given: - -```shell - -[A ID] [B ID] [% identity] [voting score] [A strand] [A start] [A end] [A length] [B strand] [B start] [B end] [B length] [A ext start] [B ext start] - -``` - -In the `strand` field, `0` stands for the forward strand and `1` stands for the reverse strand. All the positions are zero-based and are based on the forward strand, whatever which strand the sequence is mapped. Here are some examples: - -```shell - -44 500 83.6617 30 0 349 8616 24525 0 1 10081 21813 - -353 500 83.2585 28 0 10273 18410 22390 1 0 10025 21813 - -271 500 80.4192 13 0 14308 19585 22770 1 4547 10281 21813 - -327 501 89.8652 117 0 10002 22529 22529 1 9403 21810 21811 - -328 501 90.8777 93 0 0 10945 22521 1 0 10902 21811 - -``` - -In the examples above, read 500 overlaps with reads 44, 353, 271, 327 and 328. - - - -### memory consumption - - - -Before overlapping is conducted, the reads will be split into several chunks. Each chunk is about 2GB in size so that the overlapping can be run on a 8GB RAM computer. - - - -## `mecat2ref` - - - -### options - - - -`mecat2ref` is used for mapping SMRT reads to the reference genomes. The command is - -```shell - -mecat2ref -d [reads] -r [reference] -w [folder] -t [# of threads] -o [output] -b [# of results] -m [output format] -x [0/1] - -``` - -The meanings of each option are as follows: - -* `-d [reads]`, reads file name in FASTA/FASTQ format - -* `-r [reference]`, reference genome file name in FASTA format - -* `-w [folder]`, a directory for storing temporary results - -* `-t [# of threads]`, number of working CPU threads - -* `-o [output]`, output file name - -* `-b [# of result]`, output the best b alignments - -* `-m [output format]`, output format: 0 = ref, 1 = M4, 2 = SAM, default = 0 - -* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. - -### output format - - - -`mecat2ref` outputs results in one of the three formats: the `ref` format, the `M4` format, and the `SAM` format. - - - -For the `ref` format, each result occupies three lines in the form: - -```shell - -[read name] [ref name] [ref strand] [voting score] [read start] [read end] [read length] [ref start] [ref end] - -mapped read subsequence - -mapped reference subsequence - -``` - -The strands of the reads are always forward. In the `[ref strand]` field, `F` indicates forward strand while `R` indicates reverse strand. All the positions are zero-based and relative to the forward strand. Here is an example: - -```shell - -1 gi|556503834|ref|NC_000913.3| F 10 2 58 1988134 1988197 - -AAT-AGCGCCTGCCAGGCG-TCTTTT--CCGGCCATTGT-CGCAG--CACTGTAACGCGTAAAA - -AATTAGCGCCTGCCAGGCGGTCTTTTTTCCGGCCATTGTTCGCAGGG-ACTGTAACGCGTAAAA - -``` - -In this example, read 1 is mapped to the reference `gi|556503834|ref|NC_000913.3|`. - - - -### memory consumption - - - -* Index for the genome: genomeSize * 8 bytes - -* Compressed index for each CPU thread: genomeSize \* 0.1 * t bytes - -* Local alignment: 100M * t + 1G bytes - - - -## `mecat2cns` - - - -`mecat2cns` is an adaptive error correction tool for high-noise single-molecula sequencing reads. It is as accurate as [pbdagcon](https://github.com/PacificBiosciences/pbdagcon.git) and as fast as FalconSense. Inputs to `mecat2cns` can be either `can` format or `M4` format. The command for running `mecat2cns` is - -```shell - -mecat2cns [options] overlaps-file reads output - -``` - -The options are - -* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. - -* `-i [input type]`, input format, 0 = `can`, 1 = `M4 - -* `-t [# of threads]`, number of CPU threads for consensus - -* `-p [batch size]`, batch size the reads will be partitioned - -* `-r [ratio]`, minimum mapping ratio - -* `-a [overlap size]`, overlaps with length >= a will be used. - -* `-c [coverage]`, minimum coverage, default=6 - -* `-l [length]`, minimum length of the corrected sequence - -If `x` is `0`, then the default values for the other options are: -```shell --i 1 -t 1 -p 100000 -r 0.9 -a 2000 -c 6 -l 5000 -``` -If `x` is `1`, then the default values for the other options are: -```shell --i 1 -t 1 -p 100000 -r 0.4 -a 400 -c 6 -l 2000 -``` - - - -If the inputs are `M4` format, the overlap results in `[overlaps-file]` must contain the gapped extension start point, which means the option `-g` in `mecat2pw` must be set to 1, otherwise `mecat2cns` will fail to run. Also note that the memory requirement of `mecat2cns` is about 1/4 of the total size of the reads. For example, if the reads are of total size 1GB, then `mecat2cns` will occupy about 250MB memory. - - - -### output format - - - -The corrected sequences are given in FASTA format. The header of each corrected sequence consists of three components seperated by underlines: - -```shell - ->A_B_C_D - -``` - -where - -* `A` is the original read id - -* `B` is the left-most effective position - -* `C` is the right-most effective position - -* `D` is the length of the corrected sequence - - - -by effective position we mean the position in the original sequence that is covered by at least `c` (the argument to the option `-c`) reads. - -## `extract_sequences` - - - -`extract_sequences` was applied into extract 25X 0r 40X longest sequences from the corrected data. The command is - -``` shell - -extract_sequences [the input fasta file from mecat2cns] [the output filename] [genome size] [coverage] - -``` - - - -## `mecat2canu` - - - -`mecat2canu` is a modified and more efficient version of the [Canu pipeline](https://github.com/marbl/canu). `mecat2canu` accelerates `canu` by replacing its overlapper `mhap` by `mecat2asmpw`, which is a customized version of `mecat2pw`. The options of `mecat2canu` are the same as `canu` except its overlapper is replaced by `mecat2asmpw`. The command for assemblying corrected Pacbio reads is - -``` shell - -mecat2canu -d [working-folder] -p [file-prefix] -trim-assemble errorRate=[fraction error] \ - - -overlapper=mecat2asmpw genomeSize=[genome size] \ - - maxMemory=[host memory size] maxThreads=[# of CPU threads] usedGrid=0 \ - - -pacbio-corrected reads-name - -``` -The command for assemblying corrected Nanopore reads is -``` shell - -mecat2canu -d [working-folder] -p [file-prefix] -trim-assemble errorRate=[fraction error] \ - - -overlapper=mecat2asmpw genomeSize=[genome size] \ - - maxMemory=[host memory size] maxThreads=[# of CPU threads] usedGrid=0 \ - - -nanopore-corrected reads-name - -``` - - -After assembling, the results are given in the folder `working-folder`. The assembled genome is given in the file `working-folder/file-prefix.contigs.fasta` and the details of the contigs are given in the file `working-folder/file-prefix.layout.tigInfo`. - - - -# `Citation` - -Chuan-Le Xiao, Ying Chen, Shang-Qian Xie, Kai-Ning Chen, Yan Wang, Yue Han, Feng Luo, Zhi Xie. [MECAT: fast mapping, error correction, and de novo assembly for single-molecule sequencing reads](https://www.nature.com/articles/nmeth.4432). Nature Methods, 2017, 14: 1072-1074 - -# `Contact` - -* Chuan-Le Xiao, xiaochuanle@126.com - -* Ying Chen, chenying2016@gmail.com - -* Feng Luo, luofeng@clemson.edu - -# `Update Information` -Updates in MECAT V1.3 (2017.12.18): - -* Correct text error in HDF5 Installation. - -* Update the makefile in dextract . - -* Update citation. - -Updates in MECAT V1.2 (2017.5.22): - -* Add `trimming module` in `mecat2canu` to improve the integrality of the assembly. - -* Add supports for Nanopore data. - -* Improve the sensitivity of `mecat2ref`. - -MECAT v1.1 replaced the old MECAT,some debug were resolved and some new fuctions were added: -* 1. we added the extracted tools for the raw `H5` format files. -* 2. some debugs from running mecat2canu were solved - - - - - - - - - - - - - - - - - +**We have released a new version [MECAT2](https://github.com/xiaochuanle/MECAT2). Please go and download that new version. This version will not be updated any more.** + +# Contents + +* [Introduction](#S-introduction) + +* [Installation](#S-installation) + +* [Quick Start](#S-quick-start) + +* [Input Format](#S-input-format) + +* [Program Descriptions](#S-program-description) + +* [Citation](#S-citation) + +* [Contact](#S-contact) + +* [Update Information](#S-update) + +# Introdction + + + +MECAT is an ultra-fast Mapping, Error Correction and de novo Assembly Tools for single molecula sequencing (SMRT) reads. MECAT employs novel alignment and error correction algorithms that are much more efficient than the state of art of aligners and error correction tools. MECAT can be used for effectively de novo assemblying large genomes. For example, on a 32-thread computer with 2.0 GHz CPU , MECAT takes 9.5 days to assemble a human genome based on 54x SMRT data, which is 40 times faster than the current [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/). We also use MECAT to assemble a diploid human genome based on 102x SMRT data only in 25 days. The latter assembly leads a great improvement of quality to the previous genome assembled from the 54x haploid SMRT data. MECAT performance were compared with [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/), [FALCON](https://github.com/PacificBiosciences/falcon) and [Canu(v1.3)](http://canu.readthedocs.io/en/latest/) in five real datasets. The quality of assembled contigs produced by MECAT is the same or better than that of the [PBcR-Mhap pipeline](http://cbcb.umd.edu/software/pbcr/mhap/) and [FALCON](https://github.com/PacificBiosciences/falcon). Here are some comparisons on the 32-thread computer with 2.0 GHz CPU and 512 GB RAM memory: + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
GenomePipelineThread numberTotal running time (h)NG50 of genome
E.coliFALCON161.214,635,129
PBcR-MHAP161.294,652,272
Canu160.714,648,002
MECAT160.244,649,626
YeastFALCON162.16587,169
PBcR-MHAP164.2818,229
Canu165.11739,902
MECAT160.91929,350
A.thalianaFALCON16223.847,583,032
PBcR-MHAP16188.79,610,192
Canu16118.578,315,338
MECAT1610.7312600961
D.melanogasterFALCON16140.7215,664,372
PBcR-MHAP16101.2213,627,256
Canu1669.3414,179,324
MECAT169.5818,111,159
Human(54X)PBcR-MHAH(f)3257501,857,788
PBcR-MHAH(s)32130004,320,471
MECAT32230.544,878,957
+
+ + +MECAT consists of four modules: + +* `mecat2pw`, a fast and accurate pairwise mapping tool for SMRT reads + +* `mecat2ref`, a fast and accurate reference mapping tool for SMRT reads + +* `mecat2cns`, correct noisy reads based on their pairwise overlaps + +* `mecat2canu`, a modified and more efficient version of the [Canu pipeline](https://github.com/marbl/canu). [Canu](https://github.com/marbl/canu) is a customized version of the Celera Assembler that designed for high-noise single-molecule sequencing + + + +MEAP is written in C, C++, and perl. It is open source and distributed under the [GPLv3](http://www.gnu.org/licenses/gpl-3.0.html) license. + + + +# Installation + +The current directory is `/public/users/chenying/smrt_asm`. + +* Install `MECAT`: +``` shell +git clone https://github.com/xiaochuanle/MECAT.git +cd MECAT +make +cd .. +``` +After installation, all the executables are found in `MECAT/Linux-amd64/bin`. The folder name `Linux-amd64` will vary in operating systems. For example, in MAC, the executables are put in `MECAT/Darwin-amd64/bin`. + +* Install `HDF5`: +``` shell +wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.15-patch1/src/hdf5-1.8.15-patch1.tar.gz +tar xzvf hdf5-1.8.15-patch1.tar.gz +mkdir hdf5 +cd hdf5-1.8.15-patch1 +./configure --enable-cxx --prefix=/public/users/chenying/smrt_asm/hdf5 +make +make install +cd .. +``` +The header files of `HDF5` are in `hdf5/include`. The library files of `HDF5` are in `hdf5/lib` (in some systems, they are put in `hdf5/lib64`, check it!). + +* Install `dextract` +``` shell +git clone https://github.com/PacificBiosciences/DEXTRACTOR.git +cp MECAT/dextract_makefile DEXTRACTOR +cd DEXTRACTOR +export HDF5_INCLUDE=/public/users/chenying/smrt_asm/hdf5/include +export HDF5_LIB=/public/users/chenying/smrt_asm/hdf5/lib +make -f dextract_makefile +cd .. +``` + +* Add relative pathes +``` shell +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/public/users/chenying/smrt_asm/hdf5/lib +export PATH=/public/users/chenying/smrt_asm/MECAT/Linux-amd64/bin:$PATH +export PATH=/public/users/chenying/smrt_asm/DEXTRACTOR:$PATH +``` + + +# Quick Start + + + +Using MECAT to assemble a genome involves 4 steps. Here we take assemblying the genome of Ecoli as an example, to go through each step in order. Options of each command will be given in next section. + +## Assemblying Pacbio Data + +We download the reads [ecoli_filtered.fastq.gz](http://gembox.cbcb.umd.edu/mhap/raw/ecoli_filtered.fastq.gz) from the MHAP website. By decompressing it we obtain `ecoli_filtered.fastq`. + +* Step 1, using `mecat2pw` to detect overlapping candidates + +``` shell + +mecat2pw -j 0 -d ecoli_filtered.fastq -o ecoli_filtered.fastq.pm.can -w wrk_dir -t 16 + +``` + +* Step 2, correct the noisy reads based on their pairwise overlapping candidates. + +```shell + +mecat2cns -i 0 -t 16 ecoli_filtered.fastq.pm.can ecoli_filtered.fastq corrected_ecoli_filtered + +``` + +* Step 3, extract the longest 25X corrected reads + +```shell + +extract_sequences corrected_ecoli_filtered.fasta corrected_ecoli_25x.fasta 4800000 25 + +``` + +* Step 4, assemble the longest 25X corrected reads using `mecat2cacu` + +```shell + +mecat2canu -trim-assemble -p ecoli -d ecoli genomeSize=4800000 ErrorRate=0.02 maxMemory=40 maxThreads=16 useGrid=0 Overlapper=mecat2asmpw -pacbio-corrected corrected_ecoli_25x.fasta + +``` + +## Assemblying Nanopore Data + +Download [MAP006-PCR-1_2D_pass.fasta](http://nanopore.s3.climb.ac.uk/MAP006-PCR-1_2D_pass.fasta). + +* Step 1, using `mecat2pw` to detect overlapping candidates + +``` shell + +mecat2pw -j 0 -d MAP006-PCR-1_2D_pass.fasta -o candidatex.txt -w wrk_dir -t 16 -x 1 + +``` + +* Step 2, correct the noisy reads based on their pairwise overlapping candidates. + +```shell + +mecat2cns -i 0 -t 16 -x 1 candidates.txt MAP006-PCR-1_2D_pass.fasta corrected_ecoli.fasta + +``` + +* Step 3, extract the longest 25X corrected reads + +```shell + +extract_sequences corrected_ecoli.fasta corrected_ecoli_25x.fasta 4800000 25 + +``` + +* Step 4, assemble the longest 25X corrected reads using `mecat2cacu` + +```shell + +mecat2canu -trim-assemble -p ecoli -d ecoli genomeSize=4800000 ErrorRate=0.06 maxMemory=40 maxThreads=16 useGrid=0 Overlapper=mecat2asmpw -nanopore-corrected corrected_ecoli_25x.fasta + +``` + + +After step 4, the assembled genome is given in file `ecoli/ecoli.contigs.fasta`. Details of the contigs can be found in file `ecoli/ecoli.layout.tigInfo`. + + +# Input Format + +MECAT is capable of processing `FASTA`, `FASTQ`, and `H5` format files. However, the `H5` files must first be transfered to `FASTA` +format by running `DEXTRACTOR/dextract` before running MECAT. For example: +```shell +find pathto/raw_reads -name "*.bax.h5" -exec readlink -f {} \; > reads.fofn +while read line; do dextract -v $line >> reads.fasta ; done < reads.fofn +``` +the extracted result should be the reads.fasta file for mecat's input file. + +# Program Descriptions + + + +We describe in detail each module of MECAT, including their options and output formats. + + + +## `mecat2pw` + + + +### options + +The command for running `mecat2pw` is + +```shell + +mecat2pw -j [task] -d [fasta/fastq] -w [working folder] -t [# of threads] -o [output] -n [# of candidates] -a [overlap size] -k [# of kmers] -g [0/1] -x [0/1] + +``` + +The options are: + +* `-j [task]`, job name, 0 = detect overlapping candidates only, 1 = output overlaps in M4 format, default = 1. If we are to correct noisy reads, outputing overlapping candidates is enough. + +* `-d [fasta/fastq]`, reads file name in FASTA or FASTQ format. + +* `-w [working folder]`, a directory for storing temporary results, will be created if not exists. + +* `-t [# of threads]`, number of CPU threads used for overlapping, default=1. + +* `-o [output]`, output file name + +* `-n [# of candidates]`, number of candidates considered for gapped extension, default=100. Since each chunk is about 2GB size, number of candidates(NC) should be set by genome size (GS).For GS < 20M, NC should be set as 200; For GS>20M and GS<200M; NC should be set as 100; For GS>200M, NC should be set as 50. + +* `-a [overlap size]`, only output overlaps with length >= a. Default: 2000 if x is set to 0, 500 if x is set to 1. + +* `-k [# of kmers]`, two blocks between two reads having >= k kmer matches will be considered as a matched block pair. Default: 4 if x is set to 0, 2 if x is set to 1. + +* `-g [0/1]`, output the gapped extension start point (1) or not (0), default=0. + +* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. + + +### output format + + + +If the job is detecting overlapping candidates, the results are output in `can` format, each result of which occupies one line and 9 fields: + +```shell + +[A ID] [B ID] [A strand] [B strand] [A gapped start] [B gapped start] [voting score] [A length] [B length] + +``` + + + +`mecat2pw` outputs overlapping results in `M4` format, of which one result is given in one line. The fileds of `M4` format is given in order below: + +```shell + +[A ID] [B ID] [% identity] [voting score] [A strand] [A start] [A end] [A length] [B strand] [B start] [B end] [B length] + +``` + +If the `-g` option is set to `1`, two more fields indicating the extension starting points are given: + +```shell + +[A ID] [B ID] [% identity] [voting score] [A strand] [A start] [A end] [A length] [B strand] [B start] [B end] [B length] [A ext start] [B ext start] + +``` + +In the `strand` field, `0` stands for the forward strand and `1` stands for the reverse strand. All the positions are zero-based and are based on the forward strand, whatever which strand the sequence is mapped. Here are some examples: + +```shell + +44 500 83.6617 30 0 349 8616 24525 0 1 10081 21813 + +353 500 83.2585 28 0 10273 18410 22390 1 0 10025 21813 + +271 500 80.4192 13 0 14308 19585 22770 1 4547 10281 21813 + +327 501 89.8652 117 0 10002 22529 22529 1 9403 21810 21811 + +328 501 90.8777 93 0 0 10945 22521 1 0 10902 21811 + +``` + +In the examples above, read 500 overlaps with reads 44, 353, 271, 327 and 328. + + + +### memory consumption + + + +Before overlapping is conducted, the reads will be split into several chunks. Each chunk is about 2GB in size so that the overlapping can be run on a 8GB RAM computer. + + + +## `mecat2ref` + + + +### options + + + +`mecat2ref` is used for mapping SMRT reads to the reference genomes. The command is + +```shell + +mecat2ref -d [reads] -r [reference] -w [folder] -t [# of threads] -o [output] -b [# of results] -m [output format] -x [0/1] + +``` + +The meanings of each option are as follows: + +* `-d [reads]`, reads file name in FASTA/FASTQ format + +* `-r [reference]`, reference genome file name in FASTA format + +* `-w [folder]`, a directory for storing temporary results + +* `-t [# of threads]`, number of working CPU threads + +* `-o [output]`, output file name + +* `-b [# of result]`, output the best b alignments + +* `-m [output format]`, output format: 0 = ref, 1 = M4, 2 = SAM, default = 0 + +* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. + +### output format + + + +`mecat2ref` outputs results in one of the three formats: the `ref` format, the `M4` format, and the `SAM` format. + + + +For the `ref` format, each result occupies three lines in the form: + +```shell + +[read name] [ref name] [ref strand] [voting score] [read start] [read end] [read length] [ref start] [ref end] + +mapped read subsequence + +mapped reference subsequence + +``` + +The strands of the reads are always forward. In the `[ref strand]` field, `F` indicates forward strand while `R` indicates reverse strand. All the positions are zero-based and relative to the forward strand. Here is an example: + +```shell + +1 gi|556503834|ref|NC_000913.3| F 10 2 58 1988134 1988197 + +AAT-AGCGCCTGCCAGGCG-TCTTTT--CCGGCCATTGT-CGCAG--CACTGTAACGCGTAAAA + +AATTAGCGCCTGCCAGGCGGTCTTTTTTCCGGCCATTGTTCGCAGGG-ACTGTAACGCGTAAAA + +``` + +In this example, read 1 is mapped to the reference `gi|556503834|ref|NC_000913.3|`. + + + +### memory consumption + + + +* Index for the genome: genomeSize * 8 bytes + +* Compressed index for each CPU thread: genomeSize \* 0.1 * t bytes + +* Local alignment: 100M * t + 1G bytes + + + +## `mecat2cns` + + + +`mecat2cns` is an adaptive error correction tool for high-noise single-molecula sequencing reads. It is as accurate as [pbdagcon](https://github.com/PacificBiosciences/pbdagcon.git) and as fast as FalconSense. Inputs to `mecat2cns` can be either `can` format or `M4` format. The command for running `mecat2cns` is + +```shell + +mecat2cns [options] overlaps-file reads output + +``` + +The options are + +* `-x [0/1]`, sequencing platform: 0 = Pacbio, 1 = Nanopore. Default: 0. + +* `-i [input type]`, input format, 0 = `can`, 1 = `M4 + +* `-t [# of threads]`, number of CPU threads for consensus + +* `-p [batch size]`, batch size the reads will be partitioned + +* `-r [ratio]`, minimum mapping ratio + +* `-a [overlap size]`, overlaps with length >= a will be used. + +* `-c [coverage]`, minimum coverage, default=6 + +* `-l [length]`, minimum length of the corrected sequence + +If `x` is `0`, then the default values for the other options are: +```shell +-i 1 -t 1 -p 100000 -r 0.9 -a 2000 -c 6 -l 5000 +``` +If `x` is `1`, then the default values for the other options are: +```shell +-i 1 -t 1 -p 100000 -r 0.4 -a 400 -c 6 -l 2000 +``` + + + +If the inputs are `M4` format, the overlap results in `[overlaps-file]` must contain the gapped extension start point, which means the option `-g` in `mecat2pw` must be set to 1, otherwise `mecat2cns` will fail to run. Also note that the memory requirement of `mecat2cns` is about 1/4 of the total size of the reads. For example, if the reads are of total size 1GB, then `mecat2cns` will occupy about 250MB memory. + + + +### output format + + + +The corrected sequences are given in FASTA format. The header of each corrected sequence consists of three components seperated by underlines: + +```shell + +>A_B_C_D + +``` + +where + +* `A` is the original read id + +* `B` is the left-most effective position + +* `C` is the right-most effective position + +* `D` is the length of the corrected sequence + + + +by effective position we mean the position in the original sequence that is covered by at least `c` (the argument to the option `-c`) reads. + +## `extract_sequences` + + + +`extract_sequences` was applied into extract 25X 0r 40X longest sequences from the corrected data. The command is + +``` shell + +extract_sequences [the input fasta file from mecat2cns] [the output filename] [genome size] [coverage] + +``` + + + +## `mecat2canu` + + + +`mecat2canu` is a modified and more efficient version of the [Canu pipeline](https://github.com/marbl/canu). `mecat2canu` accelerates `canu` by replacing its overlapper `mhap` by `mecat2asmpw`, which is a customized version of `mecat2pw`. The options of `mecat2canu` are the same as `canu` except its overlapper is replaced by `mecat2asmpw`. The command for assemblying corrected Pacbio reads is + +``` shell + +mecat2canu -d [working-folder] -p [file-prefix] -trim-assemble errorRate=[fraction error] \ + + -overlapper=mecat2asmpw genomeSize=[genome size] \ + + maxMemory=[host memory size] maxThreads=[# of CPU threads] usedGrid=0 \ + + -pacbio-corrected reads-name + +``` +The command for assemblying corrected Nanopore reads is +``` shell + +mecat2canu -d [working-folder] -p [file-prefix] -trim-assemble errorRate=[fraction error] \ + + -overlapper=mecat2asmpw genomeSize=[genome size] \ + + maxMemory=[host memory size] maxThreads=[# of CPU threads] usedGrid=0 \ + + -nanopore-corrected reads-name + +``` + + +After assembling, the results are given in the folder `working-folder`. The assembled genome is given in the file `working-folder/file-prefix.contigs.fasta` and the details of the contigs are given in the file `working-folder/file-prefix.layout.tigInfo`. + + + +# `Citation` + +Chuan-Le Xiao, Ying Chen, Shang-Qian Xie, Kai-Ning Chen, Yan Wang, Yue Han, Feng Luo, Zhi Xie. [MECAT: fast mapping, error correction, and de novo assembly for single-molecule sequencing reads](https://www.nature.com/articles/nmeth.4432). Nature Methods, 2017, 14: 1072-1074 + +# `Contact` + +* Chuan-Le Xiao, xiaochuanle@126.com + +* Ying Chen, chenying2016@gmail.com + +* Feng Luo, luofeng@clemson.edu + +# `Update Information` +Updates in MECAT V1.3 (2017.12.18): + +* Correct text error in HDF5 Installation. + +* Update the makefile in dextract . + +* Update citation. + +Updates in MECAT V1.2 (2017.5.22): + +* Add `trimming module` in `mecat2canu` to improve the integrality of the assembly. + +* Add supports for Nanopore data. + +* Improve the sensitivity of `mecat2ref`. + +MECAT v1.1 replaced the old MECAT,some debug were resolved and some new fuctions were added: +* 1. we added the extracted tools for the raw `H5` format files. +* 2. some debugs from running mecat2canu were solved + + + + + + + + + + + + + + + + + diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 5164d01..323ce19 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -13,8 +13,8 @@ get_sw_parameters_small() swp.row_size = 4096; swp.column_size = 4096; swp.segment_aln_size = 4096; - swp.max_seq_size = 100000; - swp.max_aln_size = 100000; + swp.max_seq_size = MAX_SEQ_SIZE; + swp.max_aln_size = MAX_SEQ_SIZE; swp.d_path_size = 5000000; swp.aln_path_size = 5000000; @@ -29,8 +29,8 @@ get_sw_parameters_large() swp.row_size = 4096; swp.column_size = 4096; swp.segment_aln_size = 4096; - swp.max_seq_size = 100000; - swp.max_aln_size = 100000; + swp.max_seq_size = MAX_SEQ_SIZE; + swp.max_aln_size = MAX_SEQ_SIZE; swp.d_path_size = 5000000; swp.aln_path_size = 5000000; diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index b7bd2bf..6b4a066 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -308,7 +308,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t read_size(overlaps[sid].ssize); + const idx_t read_size(overlaps[read_id].ssize); std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); tstr.resize(read_size); @@ -355,7 +355,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData M5Record& m5 = pctd.m5; CnsAlns& cns_vec = pctd.cns_alns; std::vector& cns_results = pctd.cns_results; - const idx_t read_size = overlaps[sid].ssize; + const idx_t read_size = overlaps[read_id].ssize; std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; tstr.resize(read_size); @@ -422,13 +422,13 @@ check_cov_stats(u1_t* cov_stats, int soff, int send) } void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { - PackedDB& reads(ctd.reads); + const PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); DiffRunningData* const drd_s(pctd.drd_s); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t ssize(reads.read_size(sid)); + const idx_t ssize(reads.read_size(read_id)); std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); tstr.resize(ssize); @@ -443,7 +443,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD CnsTableItem* cns_table(pctd.cns_table); std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); cns_vec.clear(); - std::set used_ids; + std::set used_ids; u1_t* cov_stats(pctd.id_list); std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); for (idx_t i(sid); i < eid && num_added < max_added; ++i) { @@ -484,7 +484,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t ssize(reads.read_size(sid)); + const idx_t ssize(reads.read_size(read_id)); std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; tstr.resize(ssize); diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index 0f6aadd..b3729b2 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -9,10 +9,10 @@ static int input_type_pacbio = 1; static int num_threads_pacbio = 1; -static double mapping_ratio_pacbio = 0.9; -static int align_size_pacbio = 2000; -static int cov_pacbio = 6; -static int min_size_pacbio = 5000; +static double mapping_ratio_pacbio = 0.6; +static int align_size_pacbio = 1000; +static int cov_pacbio = 4; +static int min_size_pacbio = 2000; static bool print_usage_pacbio = false; static int tech_pacbio = TECH_PACBIO; @@ -31,7 +31,7 @@ static int full_reads = 0; static idx_t read_buffer_size = 0; static int preprocess_reads = 0; static int reorder_reads = 0; -static size_t batch_size = 8589934592; // 8 GB +static idx_t batch_size = 8589934592; // 8 GB static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -85,22 +85,22 @@ make_options(const ConsensusOptions& options) if (options.num_threads > -1) { cmd << " -" << num_threads_n << " " << options.num_threads; } - if (options.batch_size) { + if (options.batch_size && options.batch_size != batch_size) { cmd << " -" << batch_size_n << " " << options.batch_size; } - if (options.min_mapping_ratio >= 0) { + if (options.min_mapping_ratio >= 0 && ((options.tech == TECH_PACBIO && options.min_mapping_ratio != mapping_ratio_pacbio) || (options.tech != TECH_PACBIO && options.min_mapping_ratio != mapping_ratio_nanopore))) { cmd << " -" << mapping_ratio_n << " " << options.min_mapping_ratio; } - if (options.min_align_size >= 0) { + if (options.min_align_size >= 0 && ((options.tech == TECH_PACBIO && options.min_align_size != align_size_pacbio) || (options.tech != TECH_PACBIO && options.min_align_size != align_size_nanopore))) { cmd << " -" << align_size_n << " " << options.min_align_size; } - if (options.min_cov >= 0) { + if (options.min_cov >= 0 && ((options.tech == TECH_PACBIO && options.min_cov != cov_pacbio) || (options.tech != TECH_PACBIO && options.min_cov != cov_nanopore))) { cmd << " -" << cov_n << " " << options.min_cov; } - if (options.min_size >= 0) { + if (options.min_size >= 0 && ((options.tech == TECH_PACBIO && options.min_size != min_size_pacbio) || (options.tech != TECH_PACBIO && options.min_size != min_size_nanopore))) { cmd << " -" << min_size_n << " " << options.min_size; } - if (options.num_partition_files > 0) { + if (options.num_partition_files > 0 && options.num_partition_files != num_partition_files) { cmd << " -" << num_partition_files_n << " " << options.num_partition_files; } if (options.grid_options != NULL) { diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index dbed64b..e20532b 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -104,9 +104,12 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi if (stat(input.c_str(), &buf) == -1) { ERROR("Could not get file size: %s", input.c_str()); } - // each candidate line takes approximately 44 characters, but go with 32; + // each candidate line takes on average 44 characters, but go with 32; // each one produces two candidates (forward and reverse) const idx_t num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size); + // separate them by read id on this pass, as we don't know how + // many candidates each read is part of; we're assuming an even + // distribution on average const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); std::vector read_sizes; PackedDB::read_sizes(pac_prefix, read_sizes); @@ -142,10 +145,14 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi if (!read_sizes[ec.sid] || !read_sizes[ec.qid]) { continue; } + // make sure these match before tossing them r_assert(ec.ssize == read_sizes[ec.sid] && ec.qsize == read_sizes[ec.qid]); + // as variable sizes may be different, make sure the ones + // we read in can be safely stored r_assert(ec.sid <= ExtensionCandidateCompressed::max_value); r_assert(ec.qid <= ExtensionCandidateCompressed::max_value); r_assert(ec.sext <= ExtensionCandidateCompressed::max_value); + // qext is one bit smaller than the others r_assert(ec.qext <= ExtensionCandidateCompressed::max_qext); r_assert(ec.score <= ExtensionCandidateCompressed::max_value); if (L <= ec.sid && ec.sid < R) { diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index a65d57c..d4804d2 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -180,7 +180,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { ERROR("Could not seek to end of fasta db to get size\n"); } const idx_t file_size(pstream.tellg()); - max_db_size = std::min(file_size, size); + max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { safe_calloc(pac, u1_t, max_db_size); } diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index 9034052..493ad6e 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -28,7 +28,7 @@ static int maxc = 100; static int output_gapped_start_point = 1; -static int kmer_size = 13; +static int kmer_size = 15; static const double ddfs_cutoff_pacbio = 0.25; static const double ddfs_cutoff_nanopore = 0.25; static double ddfs_cutoff = ddfs_cutoff_pacbio; diff --git a/src/mecat2pw/pw_options.cpp b/src/mecat2pw/pw_options.cpp index ff5ee30..fa82cb7 100644 --- a/src/mecat2pw/pw_options.cpp +++ b/src/mecat2pw/pw_options.cpp @@ -9,9 +9,9 @@ static const int kDefaultNumThreads = 1; static const int kDefaultNumCandidates = 100; -static const int kDefaultAlignSizePacbio = 2000; +static const int kDefaultAlignSizePacbio = 1000; static const int kDefaultAlignSizeNanopore = 500; -static const int kDefaultKmerMatchPacbio = 4; +static const int kDefaultKmerMatchPacbio = 3; static const int kDefaultKmerMatchNanopore = 2; void From 3a71f316aa43d28337c7c09bac78244effd3797b Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 17 Apr 2019 11:49:42 -0500 Subject: [PATCH 18/56] random cleanup: stderr -> cerr, timer cleanup --- src/common/alignment.h | 3 +- src/common/defs.h | 49 +++++++++++++------------- src/common/lookup_table.cpp | 8 ++--- src/common/split_database.cpp | 2 +- src/mecat2cns/options.cpp | 19 +++++----- src/mecat2cns/overlaps_partition.cpp | 20 +++++------ src/mecat2cns/overlaps_partition.h | 4 +-- src/mecat2cns/overlaps_store.h | 4 +-- src/mecat2cns/reads_correction_aux.cpp | 4 +-- 9 files changed, 55 insertions(+), 58 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index e4ae6c1..d10e6b1 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -13,8 +13,9 @@ struct ExtensionCandidate int score; }; -// candidates only use a few of these values, so make a smaller structure for them +// candidates only use a few of these values, so make a smaller structure for them; // sid and qid are used a lot, and we sort on score, so stash qdir inside msb of qext + struct ExtensionCandidateCompressed { uint32_t sid, qid, sext, qext_, score; int qdir() const { diff --git a/src/common/defs.h b/src/common/defs.h index 4235cc4..1b19651 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -4,7 +4,7 @@ #include #include #include - +#include // fixed, setprecision() #include #include @@ -163,32 +163,33 @@ do { \ #include -struct Timer -{ - struct timeval start; - struct timeval end; - - void go() { gettimeofday(&start, NULL); } - void stop() { gettimeofday(&end, NULL); } - double elapsed() { return end.tv_sec - start.tv_sec + 1.0 * (end.tv_usec - start.tv_usec) / 1000000; } +struct Timer { + struct timeval start; + struct timeval end; + void go() { + gettimeofday(&start, NULL); + } + void stop() { + gettimeofday(&end, NULL); + } + double elapsed() const { + return end.tv_sec - start.tv_sec + double(end.tv_usec - start.tv_usec) / 1000000; + } }; -struct DynamicTimer -{ - DynamicTimer(const char* func) : m_func(func) - { - if (m_func) fprintf(stderr, "[%s] begins.\n", m_func); - timer.go(); +class DynamicTimer { + public: + explicit DynamicTimer(const std::string& func) : m_func(func) { + std::cerr << "[" << m_func << "] begins.\n"; + m_timer.go(); } - ~DynamicTimer() - { - timer.stop(); - fprintf(stderr, "[%s] takes %.2f secs.\n", m_func, timer.elapsed()); + ~DynamicTimer() { + m_timer.stop(); + std::cerr << "[" << m_func << "] takes " << std::fixed << std::setprecision(2) << m_timer.elapsed() << " secs.\n"; } - -private: - const char* m_func; - Timer timer; + private: + const std::string& m_func; + Timer m_timer; }; const u1_t* get_dna_encode_table(); @@ -200,7 +201,7 @@ const u1_t* get_dna_complement_table(); #define FWD 0 #define REV 1 #define REVERSE_STRAND(s) (1-(s)) -#define MAX_SEQ_SIZE 500000 +#define MAX_SEQ_SIZE 5000000 #define MAX_INVALID_END_SIZE 200 #define MIN_EXTEND_SIZE 500 #define MIN_OVERLAP_SIZE 1000 diff --git a/src/common/lookup_table.cpp b/src/common/lookup_table.cpp index 253efcb..3b0f464 100644 --- a/src/common/lookup_table.cpp +++ b/src/common/lookup_table.cpp @@ -97,13 +97,13 @@ create_ref_index(volume_t* v, int kmer_size, int num_threads) if (index->kmer_counts[i] > 128) index->kmer_counts[i] = 0; num_kmers += index->kmer_counts[i]; } - printf("number of kmers: %d\n", num_kmers); + std::cout << "number of kmers: " << num_kmers << "\n"; safe_malloc(index->kmer_offsets, int, num_kmers); safe_malloc(index->kmer_starts, int*, index_count); if (v->curr < 10 * 1000000) num_threads = 1; int kmers_per_thread = (num_kmers + num_threads - 1) / num_threads; - fprintf(stderr, "%d threads are used for filling offset lists.\n", num_threads); + std::cerr << num_threads << " threads are used for filling offset lists.\n"; uint32_t hash_boundaries[2 * num_threads]; uint32_t L = 0; num_kmers = 0; @@ -120,7 +120,7 @@ create_ref_index(volume_t* v, int kmer_size, int num_threads) if (kmer_cnt >= kmers_per_thread) { - printf("thread %d: %d\t%d\n", tid, L, i); + std::cout << "thread " << tid << ": " << L << "\t" << i << "\n"; hash_boundaries[2 * tid] = L; hash_boundaries[2 * tid + 1] = i; ++tid; @@ -135,7 +135,7 @@ create_ref_index(volume_t* v, int kmer_size, int num_threads) } if (kmer_cnt) { - printf("thread %d: %d\t%d\n", tid, L, index_count - 1); + std::cout << "thread " << tid << ": " << L << "\t" << (index_count - 1) << "\n"; hash_boundaries[2 * tid] = L; hash_boundaries[2 * tid + 1] = index_count - 1; } diff --git a/src/common/split_database.cpp b/src/common/split_database.cpp index ffaf86e..bca6567 100644 --- a/src/common/split_database.cpp +++ b/src/common/split_database.cpp @@ -524,6 +524,6 @@ print_volume_names(volume_names_t* vn) for (i = 0; i < num_vols; ++i) { const char* name = get_vol_name(vn, i); - fprintf(stderr, "%s\n", name); + std::cerr << name << "\n"; } } diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index b3729b2..f7251bb 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -149,7 +149,7 @@ void print_usage(const char* prog) { << "-" << align_size_n << " \tminimum overlap size\n" << "-" << cov_n << " \tminimum coverage under consideration\n" << "-" << min_size_n << " \tminimum length of corrected sequence\n" - << "-" << num_partition_files_n << " \tnumber of partition files when partitioning overlap results (if 0, then use system limit)\n" + << "-" << num_partition_files_n << " \tnumber of partition files per pass when partitioning overlap results (if 0, then use system limit)\n" << "-" << grid_options_n << " \toptions for grid submission\n" << "-" << grid_options_split_n << " \toptions for split grid submission\n" << "-" << reads_to_correct_n << " \tnumber of reads to correct [all]\n" @@ -211,7 +211,7 @@ int detect_tech(int argc, char* argv[]) { for (int i = 0; i < argc; ++i) { if (strcmp(tech_nstr, argv[i]) == 0) { if (i + 1 == argc) { - fprintf(stderr, "argument to option '%c' is missing.\n", tech_n); + std::cerr << "argument to option '" << tech_n << "' is missing.\n"; t = -1; } std::cout << tech_nstr << "\n"; @@ -220,7 +220,7 @@ int detect_tech(int argc, char* argv[]) { } else if (argv[i + 1][0] == '1') { t = TECH_NANOPORE; } else { - fprintf(stderr, "invalid argument to option '%c': %s\n", tech_n, argv[i + 1]); + std::cerr << "invalid argument to option '" << tech_n << "': " << argv[i + 1] << "\n"; t = -1; } break; @@ -230,14 +230,13 @@ int detect_tech(int argc, char* argv[]) { } int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { - bool parse_success = true; - int tech = detect_tech(argc, argv); + bool parse_success(true); + const int tech(detect_tech(argc, argv)); if (tech == -1) { return 1; } t = init_consensus_options(tech); int opt_char; - char err_char; opterr = 0; while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:PO")) != -1) { switch (opt_char) { @@ -247,7 +246,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { } else if (optarg[0] == '1') { t.input_type = INPUT_TYPE_M4; } else { - fprintf(stderr, "invalid argument to option '%c': %s\n", input_type_n, optarg); + std::cerr << "invalid argument to option '" << input_type_n << "': " << optarg << "\n"; return 1; } break; @@ -306,13 +305,11 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { t.reorder_reads = 1; break; case '?': - err_char = (char)optopt; - fprintf(stderr, "unrecognised option '%c'\n", err_char); + std::cerr << "unrecognised option '" << char(optopt) << "'\n"; return 1; break; case ':': - err_char = (char)optopt; - fprintf(stderr, "argument to option '%c' is missing.\n", err_char); + std::cerr << "argument to option '" << char(optopt) << "' is missing.\n"; return 1; break; } diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index e20532b..b020fce 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -85,17 +85,15 @@ static void get_repeat_reads(const char* const m4_file_name, const double min_co } #endif -void generate_partition_index_file_name(const char* const m4_file_name, std::string& ret) { - ret = m4_file_name; +void generate_partition_index_file_name(const std::string& input_file_name, std::string& ret) { + ret = input_file_name; ret += ".partition_files"; } -void generate_partition_file_name(const char* const m4_file_name, const idx_t part, std::string& ret) { - ret = m4_file_name; - ret += ".part"; +void generate_partition_file_name(const std::string& input_file_name, const idx_t part, std::string& ret) { std::ostringstream os; os << part; - ret += os.str(); + ret = input_file_name + ".part" + os.str(); } void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t batch_size, const int num_files, const idx_t num_reads) { @@ -118,7 +116,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi off_t input_pos; int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); std::string idx_file_name; - generate_partition_index_file_name(input.c_str(), idx_file_name); + generate_partition_index_file_name(input, idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec; @@ -169,7 +167,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi } } for (int k(0); k < nf; ++k) { - fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); + std::cerr << prw.file_names[k] << " contains " << prw.counts[k] << " overlaps\n"; if (prw.counts[k] != 0) { idx_file << prw.file_names[k] << "\n"; } @@ -223,7 +221,7 @@ void partition_candidates_reorder(const std::string& input, const size_t batch_s std::vector read_to_file(num_reads, -1); allocate_reads_to_files(num_batches, align_counts, read_to_file); std::string idx_file_name; - generate_partition_index_file_name(input.c_str(), idx_file_name); + generate_partition_index_file_name(input, idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec; @@ -271,7 +269,7 @@ void partition_candidates_reorder(const std::string& input, const size_t batch_s } const int nf(efid - sfid); for (int k(0); k < nf; ++k) { - fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); + std::cerr << prw.file_names[k] << " contains " << prw.counts[k] << " overlaps\n"; if (prw.counts[k] != 0) { idx_file << prw.file_names[k] << "\n"; } @@ -329,7 +327,7 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra } } for (int k(0); k < nf; ++k) { - fprintf(stderr, "%s contains %ld overlaps\n", prw.file_names[k].c_str(), prw.counts[k]); + std::cerr << prw.file_names[k] << " contains " << prw.counts[k] << " overlaps\n"; if (prw.counts[k] != 0) { idx_file << prw.file_names[k] << "\n"; } diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 3ee276d..595868d 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -7,9 +7,9 @@ #include "../common/alignment.h" -void generate_partition_index_file_name(const char* m4_file_name, std::string& ret); +void generate_partition_index_file_name(const std::string& input_file_name, std::string& ret); -void generate_partition_file_name(const char* m4_file_name, const idx_t part, std::string& ret); +void generate_partition_file_name(const std::string& input_file_name, const idx_t part, std::string& ret); void partition_m4records(const char* m4_file_name, double min_cov_ratio, size_t batch_size, int min_read_size, int num_files); diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index d954581..3638042 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -13,7 +13,7 @@ template class PartitionResultsWriter { public: - typedef void (*file_name_generator)(const char* prefix, idx_t id, std::string& name); + typedef void (*file_name_generator)(const std::string& prefix, idx_t id, std::string& name); public: const int kNumFiles; // effective open file limit int kStoreSize; @@ -170,7 +170,7 @@ template class PartitionResultsWriter { counts.assign(num_open_files, 0); files = new std::ofstream[num_open_files]; for (int i(0); i < num_open_files; ++i) { - fng(prefix.c_str(), i + batch_start_, file_names[i]); + fng(prefix, i + batch_start_, file_names[i]); const std::string tmp_file(file_names[i] + ".tmp"); if (is_restart) { if (access(file_names[i].c_str(), F_OK) == 0) { diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 20d66cf..ab9aac6 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -96,7 +96,7 @@ void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, } void allocate_ecs(ConsensusThreadData& data, ExtensionCandidateCompressed* const ec_list, const idx_t nec) { - const int n(data.rco.num_threads); + const idx_t n(data.rco.num_threads); // split by number of ec's, rather than reads, since reads ids // are not contiguous and we could get empty lists for (idx_t i(0), k(0); k != n; ++k) { @@ -104,7 +104,7 @@ void allocate_ecs(ConsensusThreadData& data, ExtensionCandidateCompressed* const // drop fractions here, as we'll likely add a few more ec's below i += (nec - i) / (n - k); if (i != nec) { // include all ec's for the last read - const int final_sid(ec_list[i].sid); + const uint32_t final_sid(ec_list[i].sid); for (++i; i != nec && ec_list[i].sid == final_sid; ++i) { } } data.data[k].num_candidates = i - start; From dfac5c67725846eeb2e0ba4a7313219023e703aa Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 17 Apr 2019 12:20:30 -0500 Subject: [PATCH 19/56] trashed reorder attempt in the end, it would just take too much memory to hold the read-read pairings, which doesn't work well when the whole point is to limit memory usage --- src/mecat2cns/main.cpp | 21 +-- src/mecat2cns/options.cpp | 23 +-- src/mecat2cns/options.h | 2 - src/mecat2cns/overlaps_partition.cpp | 248 +------------------------ src/mecat2cns/overlaps_partition.h | 4 - src/mecat2cns/packed_db.cpp | 80 -------- src/mecat2cns/packed_db.h | 1 - src/mecat2cns/reads_correction_can.cpp | 4 +- 8 files changed, 13 insertions(+), 370 deletions(-) diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index fd73351..90d883c 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -107,27 +107,14 @@ int main(int argc, char** argv) { wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { + // this speeds up candidate starts const idx_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); if (rco.reads_to_correct <= 0 || n_reads < rco.reads_to_correct) { rco.reads_to_correct = n_reads; } - if (rco.reorder_reads) { - std::vector read_order; // [old_rid] = new_rid - // [read id] = (offset, size) - std::vector > read_index; - std::vector align_counts; // [new_rid] = aligns - // XXX - all these need to be rewritten - account for - // preprocessing (do size checks up front), don't use - // excessive memory, write to disk as you go to allow - // checkpointing - // - // XXX - does not set aligns_count on restart - make_read_sort_order(rco.m4, "fasta.db", "reads.order", "fasta_ordered.db", rco.reads_to_correct, rco.min_cov, read_order, read_index, align_counts); - PackedDB::convert_db_to_ordered_db("fasta.db", "fasta_ordered.db", read_index, read_order); - partition_candidates_reorder(rco.m4, rco.batch_size, rco.num_partition_files, rco.reads_to_correct, read_order, align_counts); - } else { - partition_candidates(rco.m4, "fasta.db", rco.batch_size, rco.num_partition_files, rco.reads_to_correct); - } + // can't reorder reads for memory efficiency - takes + // too much memory to hold read-read pairings + partition_candidates(rco.m4, "fasta.db", rco.batch_size, rco.num_partition_files, rco.reads_to_correct); } else { partition_m4records(rco.m4, rco.min_mapping_ratio - 0.02, rco.batch_size, rco.min_size, rco.num_partition_files); } diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index f7251bb..5c570ef 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -29,8 +29,6 @@ static int default_tech = TECH_PACBIO; static int num_partition_files = 0; static int full_reads = 0; static idx_t read_buffer_size = 0; -static int preprocess_reads = 0; -static int reorder_reads = 0; static idx_t batch_size = 8589934592; // 8 GB static const char input_type_n = 'i'; @@ -50,8 +48,6 @@ static const char reads_to_correct_n = 'R'; static const char grid_start_delay_n = 'D'; static const char full_reads_n = 'F'; static const char read_buffer_size_n = 'b'; -static const char preprocess_reads_n = 'P'; -static const char reorder_reads_n = 'O'; void print_pacbio_default_options() @@ -124,12 +120,6 @@ make_options(const ConsensusOptions& options) if (options.read_buffer_size) { cmd << " -" << read_buffer_size_n << " " << options.read_buffer_size; } - if (options.preprocess_reads) { - cmd << " -" << preprocess_reads_n; - } - if (options.reorder_reads) { - cmd << " -" << reorder_reads_n; - } cmd << " " << options.m4; cmd << " " << options.reads; cmd << " " << options.corrected_reads; @@ -156,8 +146,6 @@ void print_usage(const char* prog) { << "-" << grid_start_delay_n << " \tseconds to delay between starting grid jobs\n" << "-" << full_reads_n << "\t\toutput full reads, not just the corrected parts\n" << "-" << read_buffer_size_n << " \tbytes of memory to buffer reads [no buffer]\n" - << "-" << preprocess_reads_n << "\t\tconvert reads from fasta to fasta db before processing (implied by -" << read_buffer_size_n << " and -" << reorder_reads_n << ")\n" - << "-" << reorder_reads_n << "\t\treorder reads before processing to improve memory efficiency\n" << "-" << usage_n << "\t\tprint usage info\n" << "\n" << "If 'x' is set to be '0' (pacbio), then the other options have the following default values: \n"; @@ -180,8 +168,6 @@ ConsensusOptions init_consensus_options(const int tech) { t.grid_start_delay = 0; t.full_reads = full_reads; t.read_buffer_size = read_buffer_size; - t.preprocess_reads = preprocess_reads; - t.reorder_reads = reorder_reads; t.batch_size = batch_size; if (tech == TECH_PACBIO) { t.input_type = input_type_pacbio; @@ -238,7 +224,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { t = init_consensus_options(tech); int opt_char; opterr = 0; - while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:PO")) != -1) { + while ((opt_char = getopt(argc, argv, "i:t:p:r:a:c:l:x:hG:I:n:R:S:D:k:Fb:")) != -1) { switch (opt_char) { case input_type_n: if (optarg[0] == '0') { @@ -296,13 +282,6 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { break; case read_buffer_size_n: t.read_buffer_size = atoll(optarg); - // implies preprocess_reads, so fall through here - case preprocess_reads_n: - t.preprocess_reads = 1; - break; - case reorder_reads_n: - t.preprocess_reads = 1; - t.reorder_reads = 1; break; case '?': std::cerr << "unrecognised option '" << char(optopt) << "'\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index 6711fa4..f17a3df 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -29,8 +29,6 @@ struct ConsensusOptions int grid_start_delay; int full_reads; idx_t read_buffer_size; - int preprocess_reads; - int reorder_reads; }; void diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index b020fce..fb87fb3 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -6,6 +6,7 @@ #include // stat(), struct stat #include #include // make_pair(), pair<> +#include // ceil() #include "overlaps_store.h" #include "reads_correction_aux.h" @@ -86,8 +87,7 @@ static void get_repeat_reads(const char* const m4_file_name, const double min_co #endif void generate_partition_index_file_name(const std::string& input_file_name, std::string& ret) { - ret = input_file_name; - ret += ".partition_files"; + ret = input_file_name + ".partition_files"; } void generate_partition_file_name(const std::string& input_file_name, const idx_t part, std::string& ret) { @@ -104,7 +104,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi } // each candidate line takes on average 44 characters, but go with 32; // each one produces two candidates (forward and reverse) - const idx_t num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size); + const idx_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size)); // separate them by read id on this pass, as we don't know how // many candidates each read is part of; we're assuming an even // distribution on average @@ -140,10 +140,11 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi prw.OpenFiles(sfid, efid, input, generate_partition_file_name, "partition.done"); } while (in >> ec) { + // size has been set to zero for too-small reads if (!read_sizes[ec.sid] || !read_sizes[ec.qid]) { continue; } - // make sure these match before tossing them + // make sure values match before tossing the values r_assert(ec.ssize == read_sizes[ec.sid] && ec.qsize == read_sizes[ec.qid]); // as variable sizes may be different, make sure the ones // we read in can be safely stored @@ -178,108 +179,6 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi prw.finalize(); } -// assign reads to files in read order so that each file -// has about the same number of candidates - -static void allocate_reads_to_files(const idx_t num_batches, const std::vector& align_counts, std::vector& read_to_file) { - idx_t total_aligns(0); - std::vector::const_iterator a(align_counts.begin()); - const std::vector::const_iterator end_a(align_counts.end()); - for (; a != end_a; ++a) { - total_aligns += *a; - } - idx_t read_id(0); - for (int batch(0); batch != num_batches; ++batch) { - idx_t align_count(0); - // drop fractions here, as we'll overcount some below - const idx_t want_aligns(total_aligns / (num_batches - batch)); - for (; align_count < want_aligns; ++read_id) { - // skip over reads we're not correcting, as they might - // actually have alignments (which we want to ignore) - if (align_counts[read_id]) { - align_count += align_counts[read_id]; - read_to_file[read_id] = batch; - } - } - total_aligns -= align_count; - } -} - -void partition_candidates_reorder(const std::string& input, const size_t batch_size, const int num_files, const idx_t num_reads, const std::vector& read_order, const std::vector& align_counts) { - DynamicTimer dtimer(__func__); - struct stat buf; - if (stat(input.c_str(), &buf) == -1) { - ERROR("Could not get file size: %s", input.c_str()); - } - // each candidate line takes approximately 44 characters, but go with 32; - // each one produces two candidates (forward and reverse) - const int num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size); - PartitionResultsWriter prw(num_files); - idx_t i(0); - off_t input_pos; - int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); - std::vector read_to_file(num_reads, -1); - allocate_reads_to_files(num_batches, align_counts, read_to_file); - std::string idx_file_name; - generate_partition_index_file_name(input, idx_file_name); - std::ofstream idx_file; - open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); - ExtensionCandidate ec; - ExtensionCandidateCompressed nec; - // and here we go through the input file num_batches times, - // being limited by the number of open output files we can have - for (; i < num_batches; i += prw.kNumFiles) { - const int sfid(i); - const int efid(std::min(sfid + prw.kNumFiles, num_batches)); - std::ifstream in; - open_fstream(in, input.c_str(), std::ios::in); - if (is_restart) { - if (!in.seekg(input_pos)) { - ERROR("Input seek failed while restoring checkpoint: %s", input.c_str()); - } - is_restart = 0; - } else { - prw.OpenFiles(sfid, efid, input, generate_partition_file_name, "partition.done"); - } - while (in >> ec) { - ec.qid = read_order[ec.qid]; // convert to new read order - ec.sid = read_order[ec.sid]; - if (ec.qid == -1 || ec.sid == -1) { - continue; - } - r_assert(ec.sid <= ExtensionCandidateCompressed::max_value); - r_assert(ec.qid <= ExtensionCandidateCompressed::max_value); - r_assert(ec.sext <= ExtensionCandidateCompressed::max_value); - r_assert(ec.qext <= ExtensionCandidateCompressed::max_qext); - r_assert(ec.score <= ExtensionCandidateCompressed::max_value); - const int sfile(read_to_file[ec.sid]); - if (sfid <= sfile && sfile < efid) { - nec.set(ec); - if (prw.WriteOneResult(sfile, nec.sid, nec)) { - prw.checkpoint(in.tellg()); - } - } - const int qfile(read_to_file[ec.qid]); - if (sfid <= qfile && qfile < efid) { - nec.set_swap(ec); - if (prw.WriteOneResult(qfile, nec.sid, nec)) { - prw.checkpoint(in.tellg()); - } - } - } - const int nf(efid - sfid); - for (int k(0); k < nf; ++k) { - std::cerr << prw.file_names[k] << " contains " << prw.counts[k] << " overlaps\n"; - if (prw.counts[k] != 0) { - idx_file << prw.file_names[k] << "\n"; - } - } - prw.CloseFiles(); - } - close_fstream(idx_file); - prw.finalize(); -} - void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const size_t batch_size, const int min_read_size, const int num_files) { DynamicTimer dtimer(__func__); idx_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); @@ -289,7 +188,7 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra if (stat(m4_file_name, &buf) == -1) { ERROR("Could not get file size: %s", m4_file_name); } - const idx_t num_batches(buf.st_size / 32 * 2 * sizeof(ExtensionCandidate) / batch_size); + const idx_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidate) / batch_size)); const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); std::string idx_file_name; generate_partition_index_file_name(m4_file_name, idx_file_name); @@ -347,138 +246,3 @@ void load_partition_files_info(const char* const idx_file_name, std::vector& old_read_sizes, std::vector& new_order, std::vector& align_counts) { - // first read in candidates and find all read-read pairings - std::vector > aligns; - ExtensionCandidate ec; - std::ifstream in; - open_fstream(in, input.c_str(), std::ios::in); - while (in >> ec) { - // screen out small reads, aligns between reads we don't care about; - // use old_read_sizes as we've already screened out the small reads - if (old_read_sizes[ec.sid] && old_read_sizes[ec.qid] && (ec.qid < num_reads || ec.sid < num_reads)) { - r_assert(old_read_sizes[ec.sid] == ec.ssize && old_read_sizes[ec.qid] == ec.qsize); - if (ec.qid < num_reads) { - aligns.push_back(std::make_pair(ec.qid, ec.sid)); - } - if (ec.sid < num_reads) { - aligns.push_back(std::make_pair(ec.sid, ec.qid)); - } - } - } - close_fstream(in); - std::sort(aligns.begin(), aligns.end()); - // generate index into aligns - align_counts.assign(num_reads, 0); - std::vector aligns_index(num_reads, -1); - const idx_t end_i(aligns.size()); - for (idx_t i(0); i != end_i;) { - const idx_t start(i); - const idx_t read_id(aligns[i].first); - for (++i; i != end_i && aligns[i].first == read_id; ++i) { } - if (i - start >= min_cov) { - aligns_index[read_id] = start; - align_counts[read_id] = i - start; - } - } - // generate the new read order; - // make sure to make space for all reads that - // could be used, not just those we're correcting - std::vector used(old_read_sizes.size(), 0); - new_order.reserve(old_read_sizes.size()); - size_t next_search(0); - for (idx_t next_unused(0);;) { - // skip over used reads, reads with too few alignments - for (; next_unused != num_reads && (used[next_unused] || aligns_index[next_unused] == -1); ++next_unused) { } - if (next_unused == num_reads) { - break; - } - used[next_unused] = 1; - new_order.push_back(next_unused); - // add all reads aligned to, and aligned to those, and so on - for (; next_search != new_order.size(); ++next_search) { - const idx_t sid(new_order[next_search]); - // we only use aligns to first num_reads reads - if (sid < num_reads) { - idx_t i(aligns_index[sid]); - if (i != -1) { - for (; i != end_i && aligns[i].first == sid; ++i) { - const idx_t qid(aligns[i].second); - if (!used[qid]) { - used[qid] = 1; - new_order.push_back(qid); - } - } - } - } - } - } -} - -// create a new order for reads to speed up pulling in reads for alignment -// processing (for conditions where memory is not sufficient to hold all -// reads in memory); generates a file with the old -> new read ordering, -// also generates an index for a fasta db file with the reads in the new -// order (to allow easy conversion of the original fasta to a fasta db -// with the new read ordering) - -void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, const idx_t num_reads, const int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts) { - // if file already exists, just read it in - if (access(sort_file_name.c_str(), F_OK) == 0) { - struct stat buf; - if (stat(sort_file_name.c_str(), &buf) == -1) { - ERROR("Could not stat read reorder file: %s", sort_file_name.c_str()); - } - read_order.resize(buf.st_size / sizeof(idx_t)); - std::ifstream in; - open_fstream(in, sort_file_name.c_str(), std::ios::in); - if (!in.read((char *)(&read_order[0]), buf.st_size)) { - ERROR("Error reading read reorder file: %s", sort_file_name.c_str()); - } - close_fstream(in); - PackedDB::read_index(pac_prefix, read_index); - return; - } - DynamicTimer dtimer(__func__); - std::vector old_read_sizes; // pre-reorder sizes - PackedDB::read_sizes(old_pac_prefix, old_read_sizes); - std::vector new_order; // [new_read_id] = old_read_id - std::vector presort_align_counts; - generate_new_read_order(input, num_reads, min_cov, old_read_sizes, new_order, presort_align_counts); - // now reverse new_order into read_order; - // also, generate index for reordered read database - read_order.assign(old_read_sizes.size(), -1); - read_index.resize(new_order.size()); - align_counts.assign(new_order.size(), 0); - idx_t total_size(0); - const idx_t end_new_rid(new_order.size()); - for (idx_t new_rid(0); new_rid != end_new_rid; ++new_rid) { - const idx_t old_rid(new_order[new_rid]); - read_order[old_rid] = new_rid; - std::pair& b(read_index[new_rid]); - b.first = total_size; - b.second = old_read_sizes[old_rid]; - total_size += (b.second + 3) / 4; - if (old_rid < num_reads) { - align_counts[new_rid] = presort_align_counts[old_rid]; - } - } - PackedDB::create_index(pac_prefix, read_index); - const std::string sort_file_name_tmp(sort_file_name + ".tmp"); - std::ofstream out; - open_fstream(out, sort_file_name_tmp.c_str(), std::ios::out | std::ios::binary); - if (!out.write((char *)(&read_order[0]), sizeof(idx_t) * read_order.size())) { - ERROR("Error writing to read reorder file: %s", sort_file_name_tmp.c_str()); - } - close_fstream(out); - if (rename(sort_file_name_tmp.c_str(), sort_file_name.c_str()) == -1) { - ERROR("Could not rename read reorder file: %s", sort_file_name_tmp.c_str()); - } -} diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 595868d..b05ebc9 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -15,10 +15,6 @@ void partition_m4records(const char* m4_file_name, double min_cov_ratio, size_t void partition_candidates(const std::string& input, const std::string& pac_prefix, size_t batch_size, int num_files, idx_t num_reads); -void partition_candidates_reorder(const std::string& input, size_t batch_size, int num_files, idx_t num_reads, const std::vector& read_order, const std::vector& align_counts); - void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); -void make_read_sort_order(const std::string& input, const std::string& old_pac_prefix, const std::string& sort_file_name, const std::string& pac_prefix, idx_t num_reads, int min_cov, std::vector& read_order, std::vector >& read_index, std::vector& align_counts); - #endif // OVERLAPS_PARTITION_H diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index d4804d2..f481a185 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -370,83 +370,3 @@ void PackedDB::read_sizes(const std::string& output_prefix, std::vector& sizes.push_back(j); } } - -static int check_reorder_restart(const std::string& ckpt_file, size_t& rid) { - std::ifstream in(ckpt_file.c_str()); - if (!in) { - return 0; - } - in >> rid; - if (!in) { - ERROR("Read error while restoring checkpoint from %s", ckpt_file.c_str()); - } - return 1; -} - -static void checkpoint_reorder(const std::string& ckpt_file, const std::string& ckpt_file_tmp, const size_t rid) { - std::ofstream out(ckpt_file_tmp.c_str()); - if (!out) { - LOG(stderr, "Checkpoint failed: couldn't open %s", ckpt_file_tmp.c_str()); - return; - } - out << rid << "\n"; - if (!out) { - LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp.c_str()); - return; - } - out.close(); - if (rename(ckpt_file_tmp.c_str(), ckpt_file.c_str()) == -1) { - LOG(stderr, "Checkpoint failed: rename failed: %s", ckpt_file.c_str()); - } -} - -// we don't need to generate the index, and we'll be writing to the output -// file somewhat randomly - -void PackedDB::convert_db_to_ordered_db(const std::string& input_prefix , const std::string& output_prefix, const std::vector >& index, const std::vector& read_order) { - const std::string pac_name(output_prefix + ".pac"); - // see if we already did this - if (access(pac_name.c_str(), F_OK) == 0) { - return; - } - DynamicTimer dtimer(__func__); - const std::string pac_name_tmp(pac_name + ".tmp"); - const std::string ckpt_name(pac_name + ".ckpt"); - const std::string ckpt_name_tmp(ckpt_name + ".tmp"); - PackedDB old_reads; - old_reads.open_db(input_prefix, MAX_SEQ_SIZE); - std::ofstream pout; - size_t old_rid(0); - if (check_reorder_restart(ckpt_name, old_rid)) { - // don't truncate, but file does need to exist already - open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::in | std::ios::binary); - } else { - open_fstream(pout, pac_name_tmp.c_str(), std::ios::out | std::ios::binary); - } - time_t next_checkpoint_time(time(0) + 300); - for (; old_rid != read_order.size(); ++old_rid) { - const idx_t new_rid(read_order[old_rid]); - if (new_rid == -1) { // we don't use this read, so skip - continue; - } - const char* seq(old_reads.load_read(old_rid)); - const std::pair& info(index[new_rid]); - const idx_t rbytes((info.second + 3) / 4); - if (!pout.seekp(info.first)) { - ERROR("Failed to seek to position: %s", pac_name_tmp.c_str()); - } - if (!pout.write(seq, rbytes)) { - ERROR("Write error to file %s", pac_name_tmp.c_str()); - } - if (time(0) >= next_checkpoint_time) { - checkpoint_reorder(ckpt_name, ckpt_name_tmp, old_rid); - next_checkpoint_time = time(0) + 300; - } - } - checkpoint_reorder(ckpt_name, ckpt_name_tmp, old_rid); - close_fstream(pout); - if (rename(pac_name_tmp.c_str(), pac_name.c_str()) == -1) { - ERROR("Could not rename tmp database file"); - } - unlink(ckpt_name.c_str()); -} diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index f2ca674..c0d4138 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -32,7 +32,6 @@ class PackedDB { static void create_index(const std::string& output_prefix, const std::vector >& index); static void read_index(const std::string& output_prefix, std::vector >& index); static void read_sizes(const std::string& output_prefix, std::vector& sizes); - static void convert_db_to_ordered_db(const std::string& input_prefix, const std::string& output_prefix, const std::vector >& index, const std::vector& read_order); // opens data file, reads in index file void open_db(const std::string& filename, idx_t memory_footprint); const char* load_read(idx_t read_id); diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 125ac77..5c9a25a 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -134,7 +134,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySidAndScore()); // if we're memory limited and we didn't already reorder, do it here; // spend some cpu time to reduce number of passes - if (data.rco.read_buffer_size && !data.rco.reorder_reads) { + if (data.rco.read_buffer_size) { // don't die if we run out of memory, just do it the slow way try { ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); @@ -194,7 +194,7 @@ int reads_correction_can(ReadsCorrectionOptions& rco) { std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); PackedDB reads; - reads.open_db(rco.reorder_reads ? "fasta_ordered.db" : "fasta.db", rco.read_buffer_size); + reads.open_db("fasta.db", rco.read_buffer_size); if (rco.job_index != -1) { return reads_correction_can_p(rco, partition_file_vec, reads); } else { From 64b0266c64c5d64ce8694186c178765588345c98 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 17 Apr 2019 14:08:59 -0500 Subject: [PATCH 20/56] changed sorting ec_list to better order reads mainly to reduce memory usage (no need to copy the list when I can simply sort it instead) --- src/mecat2cns/reads_correction_can.cpp | 108 +++++++++++++++---------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 5c9a25a..86171cf 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -1,8 +1,9 @@ #include "reads_correction_can.h" #include -#include // string +#include // numeric_limits<> #include +#include // string #include "MECAT_AlnGraphBoost.H" #include "mecat_correction.h" @@ -47,6 +48,41 @@ static void* reads_correction_func_can(void* const arg) { return NULL; } +struct CmpExtensionCandidateCompressedBySid { + bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { + return a.sid < b.sid; + } +}; + +// do full ordering, but using new rid instead of old +class OrderCmp { + public: + explicit OrderCmp(const std::vector& new_order, const idx_t num_reads) : old_rid_to_new_(num_reads, std::numeric_limits::max()) { + for (size_t i(0); i != new_order.size(); ++i) { + old_rid_to_new_[new_order[i]] = i; + } + } + ~OrderCmp() { } + bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { + if (a.sid != b.sid) { // primary sort + // for splitting up in allocate_ecs() + return old_rid_to_new_[a.sid] < old_rid_to_new_[b.sid]; + } else if (a.score != b.score) { // secondary sort + return b.score < a.score; // process best ones first + } else if (a.qid != b.qid) { // group ties by id + return a.qid < b.qid; // (doesn't matter if new or old) + } else if (a.qext() != b.qext()) { + return a.qext() < b.qext(); + } else if (a.sext != b.sext) { // tertiary sort + return a.sext < b.sext; // make sorting consistent + } else { + return a.qdir() < b.qdir(); + } + } + private: + std::vector old_rid_to_new_; +}; + class EC_Index { // offset into ec_list (and number of ecs) for each read id public: idx_t offset, count; @@ -59,13 +95,18 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // ecs per pass, with limited read space; also filters list to exclude // candidates of reads with low coverage -static ExtensionCandidateCompressed* reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_t& nec, const idx_t num_reads, const int min_cov) { +static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_t& nec, const idx_t reads_to_correct, const idx_t num_reads, const int min_cov) { + // allow us to easily access a given sid's aligns + std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySid()); idx_t total_ec(0); - std::vector index(num_reads); + std::vector index(reads_to_correct); // index existing list by sid for (idx_t i(0); i != nec;) { - const idx_t start(i); const idx_t sid(ec_list[i].sid); + if (sid >= reads_to_correct) { + break; + } + const idx_t start(i); for (++i; i != nec && ec_list[i].sid == sid; ++i) { } // make sure we have enough coverage // (don't need to check size, that happened during partition) @@ -75,17 +116,16 @@ static ExtensionCandidateCompressed* reorder_candidates(ExtensionCandidateCompre total_ec += count; } } - ExtensionCandidateCompressed* new_list(new ExtensionCandidateCompressed[total_ec]); // generate the new read order std::vector used(num_reads, 0); - std::vector new_order; + std::vector new_order; // [new_sid] = old_sid new_order.reserve(num_reads); idx_t next_unused(0); size_t next_search(0); for (;;) { // skip over used reads, reads with no alignments - for (; next_unused != num_reads && (used[next_unused] || index[next_unused].count == 0); ++next_unused) { } - if (next_unused == num_reads) { + for (; next_unused != reads_to_correct && (used[next_unused] || index[next_unused].count == 0); ++next_unused) { } + if (next_unused == reads_to_correct) { break; } used[next_unused] = 1; @@ -93,52 +133,38 @@ static ExtensionCandidateCompressed* reorder_candidates(ExtensionCandidateCompre // add all reads aligned to, and aligned to those, and so on for (; next_search != new_order.size(); ++next_search) { const idx_t sid(new_order[next_search]); - const EC_Index& a(index[sid]); - // only include reads that align to this one if we plan to - // actually run those alignments (that is, if it has sufficient - // coverage) - if (a.count >= min_cov) { - idx_t i(a.offset); - const idx_t end_i(i + a.count); - for (; i != end_i; ++i) { - const idx_t qid(ec_list[i].qid); - if (!used[qid]) { - used[qid] = 1; - new_order.push_back(qid); + if (sid < reads_to_correct) { + const EC_Index& a(index[sid]); + if (a.count) { + idx_t i(a.offset); + const idx_t end_i(i + a.count); + for (; i != end_i; ++i) { + const idx_t qid(ec_list[i].qid); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); + } } } } } } - // copy over the ecs to the new list in the new order - idx_t pos(0); - std::vector::const_iterator a(new_order.begin()); - const std::vector::const_iterator end_a(new_order.end()); - for (; a != end_a; ++a) { - const EC_Index& b(index[*a]); - if (b.count) { - memcpy(new_list + pos, ec_list + b.offset, sizeof(ExtensionCandidateCompressed) * b.count); - pos += b.count; - } - } - delete[] ec_list; - // only do this at the end, once we know we'll successfully complete + // re-sort ec_list with new order (note that coverage-excluded + // ec's will sort last, so we also change nec to exclude them) + OrderCmp new_rid_order(new_order, num_reads); + std::sort(ec_list, ec_list + nec, new_rid_order); nec = total_ec; - return new_list; } // load and sort partition data, assign to threads, start threads static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { idx_t nec; ExtensionCandidateCompressed* ec_list(load_partition_data(m4_file_name, nec)); - std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySidAndScore()); - // if we're memory limited and we didn't already reorder, do it here; - // spend some cpu time to reduce number of passes + // if we're memory limited spend some cpu time to reduce number of passes if (data.rco.read_buffer_size) { - // don't die if we run out of memory, just do it the slow way - try { - ec_list = reorder_candidates(ec_list, nec, data.reads.num_reads(), data.rco.min_cov); - } catch (std::bad_alloc &except) { } + reorder_candidates(ec_list, nec, data.rco.reads_to_correct, data.reads.num_reads(), data.rco.min_cov); + } else { + std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySidAndScore()); } pthread_t thread_ids[data.rco.num_threads]; while (data.ec_offset != nec) { From dd464d08f788fdc32462a9313572288d11925afc Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 17 Apr 2019 14:25:39 -0500 Subject: [PATCH 21/56] cleaned up some comparison functions --- src/mecat2cns/reads_correction_aux.h | 9 +++++- src/mecat2cns/reads_correction_can.cpp | 39 +++++++++++++------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index e2b0ad7..be42b42 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -127,11 +127,18 @@ struct CmpExtensionCandidateBySidAndScore { return a.qext < b.qext; } else if (a.sext != b.sext) { // tertiary sort return a.sext < b.sext; // make sorting consistent + } else if (a.soff != b.soff) { + return a.soff < b.soff; + } else if (a.qoff != b.qoff) { + return a.qoff < b.qoff; + } else if (a.send != b.send) { + return a.send < b.send; + } else if (a.qend != b.qend) { + return a.qend < b.qend; } else { return a.qdir < b.qdir; } // sdir, qsize, ssize are all the same by this point - // qoff, soff, qend, send are all zero } }; diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 86171cf..be7cf77 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -55,21 +55,23 @@ struct CmpExtensionCandidateCompressedBySid { }; // do full ordering, but using new rid instead of old -class OrderCmp { +class NewOrderCmp { public: - explicit OrderCmp(const std::vector& new_order, const idx_t num_reads) : old_rid_to_new_(num_reads, std::numeric_limits::max()) { + // sort reads not used (no new_order entry) at end + explicit NewOrderCmp(const std::vector& new_order, const idx_t num_reads) : old_rid_to_new_(num_reads, std::numeric_limits::max()) { for (size_t i(0); i != new_order.size(); ++i) { old_rid_to_new_[new_order[i]] = i; } } - ~OrderCmp() { } + ~NewOrderCmp() { } + // same as CmpExtensionCandidateCompressedBySidAndScore, but with new order bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { if (a.sid != b.sid) { // primary sort // for splitting up in allocate_ecs() return old_rid_to_new_[a.sid] < old_rid_to_new_[b.sid]; } else if (a.score != b.score) { // secondary sort return b.score < a.score; // process best ones first - } else if (a.qid != b.qid) { // group ties by id + } else if (a.qid != b.qid) { // group ties by qid return a.qid < b.qid; // (doesn't matter if new or old) } else if (a.qext() != b.qext()) { return a.qext() < b.qext(); @@ -99,17 +101,15 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ // allow us to easily access a given sid's aligns std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySid()); idx_t total_ec(0); + // count will be zero for any read id beyond reads_to_correct std::vector index(reads_to_correct); // index existing list by sid for (idx_t i(0); i != nec;) { + // we are guaranteed sid is < reads_to_correct by partitioning const idx_t sid(ec_list[i].sid); - if (sid >= reads_to_correct) { - break; - } const idx_t start(i); for (++i; i != nec && ec_list[i].sid == sid; ++i) { } // make sure we have enough coverage - // (don't need to check size, that happened during partition) const idx_t count(i - start); if (count >= min_cov) { index[sid] = EC_Index(start, count); @@ -119,7 +119,7 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ // generate the new read order std::vector used(num_reads, 0); std::vector new_order; // [new_sid] = old_sid - new_order.reserve(num_reads); + new_order.reserve(num_reads); // possible overestimate, but whatever idx_t next_unused(0); size_t next_search(0); for (;;) { @@ -133,17 +133,16 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ // add all reads aligned to, and aligned to those, and so on for (; next_search != new_order.size(); ++next_search) { const idx_t sid(new_order[next_search]); + // make sure read has index entry if (sid < reads_to_correct) { const EC_Index& a(index[sid]); - if (a.count) { - idx_t i(a.offset); - const idx_t end_i(i + a.count); - for (; i != end_i; ++i) { - const idx_t qid(ec_list[i].qid); - if (!used[qid]) { - used[qid] = 1; - new_order.push_back(qid); - } + idx_t i(a.offset); + const idx_t end_i(i + a.count); + for (; i != end_i; ++i) { + const idx_t qid(ec_list[i].qid); + if (!used[qid]) { + used[qid] = 1; + new_order.push_back(qid); } } } @@ -151,8 +150,8 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ } // re-sort ec_list with new order (note that coverage-excluded // ec's will sort last, so we also change nec to exclude them) - OrderCmp new_rid_order(new_order, num_reads); - std::sort(ec_list, ec_list + nec, new_rid_order); + NewOrderCmp new_rid_order_cmp(new_order, num_reads); + std::sort(ec_list, ec_list + nec, new_rid_order_cmp); nec = total_ec; } From 8f773ad892ad2c464c007fcc72bf7ed74cca135b Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 17 Apr 2019 14:45:47 -0500 Subject: [PATCH 22/56] added k/m/g suffixes to integer options --- src/mecat2cns/options.cpp | 41 ++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index 5c570ef..acc4475 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -29,7 +29,7 @@ static int default_tech = TECH_PACBIO; static int num_partition_files = 0; static int full_reads = 0; static idx_t read_buffer_size = 0; -static idx_t batch_size = 8589934592; // 8 GB +static idx_t batch_size = idx_t(1) << 33; // 8 GB static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -49,6 +49,25 @@ static const char grid_start_delay_n = 'D'; static const char full_reads_n = 'F'; static const char read_buffer_size_n = 'b'; +// convert number with potential suffix (k, m, g) +static idx_t convert_integer(const std::string& s) { + std::istringstream x(s); + idx_t value; + x >> value; + const size_t i(s.find_first_not_of("0123456789")); + if (i != std::string::npos) { + switch (s[i]) { + case 'g': + value *= 1024; + case 'm': + value *= 1024; + case 'k': + value *= 1024; + } + } + return value; +} + void print_pacbio_default_options() { @@ -237,22 +256,22 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { } break; case num_threads_n: - t.num_threads = atoi(optarg); + t.num_threads = convert_integer(optarg); break; case batch_size_n: - t.batch_size = atoll(optarg); + t.batch_size = convert_integer(optarg); break; case mapping_ratio_n: t.min_mapping_ratio = atof(optarg); break; case align_size_n: - t.min_align_size = atoi(optarg); + t.min_align_size = convert_integer(optarg); break; case cov_n: - t.min_cov = atoi(optarg); + t.min_cov = convert_integer(optarg); break; case min_size_n: - t.min_size = atoll(optarg); + t.min_size = convert_integer(optarg); break; case usage_n: t.print_usage_info = true; @@ -264,24 +283,24 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { t.grid_options_split = optarg; break; case job_index_n: - t.job_index = atoi(optarg); + t.job_index = convert_integer(optarg); break; case reads_to_correct_n: - t.reads_to_correct = atoi(optarg); + t.reads_to_correct = convert_integer(optarg); break; case grid_start_delay_n: - t.grid_start_delay = atoi(optarg); + t.grid_start_delay = convert_integer(optarg); break; case tech_n: break; case num_partition_files_n: - t.num_partition_files = atoi(optarg); + t.num_partition_files = convert_integer(optarg); break; case full_reads_n: t.full_reads = 1; break; case read_buffer_size_n: - t.read_buffer_size = atoll(optarg); + t.read_buffer_size = convert_integer(optarg); break; case '?': std::cerr << "unrecognised option '" << char(optopt) << "'\n"; From cf26d54ae14b9f443a4f454e888fecb98e7a0637 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 18 Apr 2019 11:53:27 -0500 Subject: [PATCH 23/56] changed some fprintf to use cerr instead also in the middle of some memory testing --- src/common/defs.h | 4 +- src/mecat2cns/overlaps_partition.cpp | 2 +- src/mecat2cns/overlaps_store.h | 1 + src/mecat2cns/packed_db.cpp | 62 ++------------------------ src/mecat2cns/packed_db.h | 6 +-- src/mecat2cns/reads_correction_aux.h | 11 ++--- src/mecat2cns/reads_correction_can.cpp | 36 ++++++++------- 7 files changed, 35 insertions(+), 87 deletions(-) diff --git a/src/common/defs.h b/src/common/defs.h index 1b19651..2433aaa 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -179,7 +179,7 @@ struct Timer { class DynamicTimer { public: - explicit DynamicTimer(const std::string& func) : m_func(func) { + explicit DynamicTimer(const char* const func) : m_func(func) { std::cerr << "[" << m_func << "] begins.\n"; m_timer.go(); } @@ -188,7 +188,7 @@ class DynamicTimer { std::cerr << "[" << m_func << "] takes " << std::fixed << std::setprecision(2) << m_timer.elapsed() << " secs.\n"; } private: - const std::string& m_func; + const std::string m_func; Timer m_timer; }; diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index fb87fb3..b644f92 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -128,7 +128,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); const idx_t L(sfid * reads_per_batch); - const idx_t R(efid < num_batches ? efid * reads_per_batch : num_reads); + const idx_t R(std::min(efid * reads_per_batch, num_reads)); std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); if (is_restart) { diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 3638042..db4cc35 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -210,6 +210,7 @@ template T* load_partition_data(const char* const path, idx_t& num_res in.seekg(0, std::ios::beg); num_results = fs / sizeof(T); T* const arr(new T[num_results]); +std::cerr << "load_partition_data: allocating for " << num_results << " objects of size " << sizeof(T) << "\n"; in.read((char*)arr, fs); // can't use static_cast<> if (!in) { ERROR("Error reading partition data: %s", path); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index f481a185..83ac77f 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -214,21 +214,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { } close_fstream(index); } -} - -const char* PackedDB::load_read(const idx_t read_id) { - const SeqIndex& si(seq_idx[read_id]); - if (!pstream.is_open()) { // all in memory already - return (char*)pac + si.memory_offset / 4; - } - if (!pstream.seekg(si.file_offset)) { - ERROR("Error seeking on fasta db"); - } - const idx_t bytes((si.size + 3) / 4); - if (!pstream.read((char*)pac, bytes)) { - ERROR("Error reading fasta db"); - } - return (char*)pac; +std::cerr << "open_db: read count " << read_count << ", index size " << seq_idx.size() << ", max db size " << max_db_size << "\n"; } idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, const idx_t nec) { @@ -278,6 +264,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co if (max_db_size == 0) { max_db_size = total_size; safe_calloc(pac, u1_t, max_db_size); +std::cerr << "load_reads: allocating " << max_db_size << "\n"; } // now read in the reads std::set::const_iterator a(read_ids.begin()); @@ -304,49 +291,6 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co return i; } -void PackedDB::create_index(const std::string& output_prefix, const std::vector >& index) { - const std::string index_name(output_prefix + ".idx"); - const std::string index_name_tmp(index_name + ".tmp"); - std::ofstream out; - open_fstream(out, index_name_tmp.c_str(), std::ios::out); - std::vector >::const_iterator a(index.begin()); - const std::vector >::const_iterator end_a(index.end()); - for (; a != end_a; ++a) { - out << a->first << "\t" << a->second << "\n"; - if (!out) { - ERROR("Error writing to pac index file: %s", index_name_tmp.c_str()); - } - } - close_fstream(out); - if (rename(index_name_tmp.c_str(), index_name.c_str()) == -1) { - ERROR("Error renaming pac index file: %s", index_name_tmp.c_str()); - } -} - -void PackedDB::read_index(const std::string& output_prefix, std::vector >& index) { - const std::string pac_name(output_prefix + ".pac"); - std::ifstream in(pac_name.c_str()); - // pre-allocate index if possible - if (in.is_open()) { - // get number of reads from end of database - if (!in.seekg(-sizeof(size_t), std::ios_base::end)) { - ERROR("Could not seek to end of fasta db to get size\n"); - } - size_t read_count; - if (!in.read((char*)&read_count, sizeof(size_t))) { - ERROR("Could not read fasta db to get size\n"); - } - in.close(); - index.reserve(read_count); - } - const std::string index_name(output_prefix + ".idx"); - open_fstream(in, index_name.c_str(), std::ios::in); - idx_t i, j; - while (in >> i >> j) { - index.push_back(std::make_pair(i, j)); - } -} - void PackedDB::read_sizes(const std::string& output_prefix, std::vector& sizes) { const std::string pac_name(output_prefix + ".pac"); std::ifstream in(pac_name.c_str()); @@ -362,6 +306,7 @@ void PackedDB::read_sizes(const std::string& output_prefix, std::vector& } in.close(); sizes.reserve(read_count); +std::cerr << "read_sizes: read_count " << read_count << "\n"; } const std::string index_name(output_prefix + ".idx"); open_fstream(in, index_name.c_str(), std::ios::in); @@ -369,4 +314,5 @@ void PackedDB::read_sizes(const std::string& output_prefix, std::vector& while (in >> i >> j) { sizes.push_back(j); } +std::cerr << "read_sizes: sizes size " << sizes.size() << "\n"; } diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index c0d4138..2829310 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -28,18 +28,14 @@ class PackedDB { void load_fasta_db(const char* fasta); // returns number of reads static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); - // create a prospective index file for pac file to be written in random order - static void create_index(const std::string& output_prefix, const std::vector >& index); - static void read_index(const std::string& output_prefix, std::vector >& index); static void read_sizes(const std::string& output_prefix, std::vector& sizes); // opens data file, reads in index file void open_db(const std::string& filename, idx_t memory_footprint); - const char* load_read(idx_t read_id); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { const SeqIndex &si(seq_idx[id]); - r_assert(size == si.size); + //r_assert(size == si.size); // mostly obsolete now if (forward) { const idx_t offset(si.memory_offset); for (idx_t i(0); i < si.size; ++i) { diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index be42b42..bd732ed 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -118,7 +118,7 @@ class CnsAlns { struct CmpExtensionCandidateBySidAndScore { bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) { if (a.sid != b.sid) { // primary sort - return a.sid < b.sid; // for splitting up in allocate_ecs() + return a.sid < b.sid; } else if (a.score != b.score) { // secondary sort return b.score < a.score; // process best ones first } else if (a.qid != b.qid) { @@ -182,10 +182,11 @@ class ConsensusPerThreadData { public: ConsensusPerThreadData() : drd_s(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), drd_l(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_large())), m5(MAX_SEQ_SIZE) { cns_results.reserve(MAX_CNS_RESULTS); - query.reserve(MAX_SEQ_SIZE); - target.reserve(MAX_SEQ_SIZE); - qaln.reserve(MAX_SEQ_SIZE); - saln.reserve(MAX_SEQ_SIZE); + // MAX_SEQ_SIZE is very large now, so let's not pre-allocate + //query.reserve(MAX_SEQ_SIZE); + //target.reserve(MAX_SEQ_SIZE); + //qaln.reserve(MAX_SEQ_SIZE); + //saln.reserve(MAX_SEQ_SIZE); } ~ConsensusPerThreadData() { delete drd_s; diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index be7cf77..a2de09a 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -55,20 +55,17 @@ struct CmpExtensionCandidateCompressedBySid { }; // do full ordering, but using new rid instead of old -class NewOrderCmp { +// (we cannot store anything in this class, as it gets copied a lot) + +class OrderCmp { public: - // sort reads not used (no new_order entry) at end - explicit NewOrderCmp(const std::vector& new_order, const idx_t num_reads) : old_rid_to_new_(num_reads, std::numeric_limits::max()) { - for (size_t i(0); i != new_order.size(); ++i) { - old_rid_to_new_[new_order[i]] = i; - } - } - ~NewOrderCmp() { } + explicit OrderCmp(const std::vector& order) : order_(order) { } + ~OrderCmp() { } // same as CmpExtensionCandidateCompressedBySidAndScore, but with new order bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { if (a.sid != b.sid) { // primary sort // for splitting up in allocate_ecs() - return old_rid_to_new_[a.sid] < old_rid_to_new_[b.sid]; + return order_[a.sid] < order_[b.sid]; } else if (a.score != b.score) { // secondary sort return b.score < a.score; // process best ones first } else if (a.qid != b.qid) { // group ties by qid @@ -82,7 +79,7 @@ class NewOrderCmp { } } private: - std::vector old_rid_to_new_; + const std::vector& order_; // [read_id] = order }; class EC_Index { // offset into ec_list (and number of ecs) for each read id @@ -95,7 +92,8 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // reorder list so reads are more concentrated and we can process more // ecs per pass, with limited read space; also filters list to exclude -// candidates of reads with low coverage +// candidates of reads with low coverage; we filter out alignments of +// low-coverage reads, so nec may be reduced static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_t& nec, const idx_t reads_to_correct, const idx_t num_reads, const int min_cov) { // allow us to easily access a given sid's aligns @@ -118,7 +116,7 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ } // generate the new read order std::vector used(num_reads, 0); - std::vector new_order; // [new_sid] = old_sid + std::vector new_order; new_order.reserve(num_reads); // possible overestimate, but whatever idx_t next_unused(0); size_t next_search(0); @@ -149,8 +147,13 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ } } // re-sort ec_list with new order (note that coverage-excluded - // ec's will sort last, so we also change nec to exclude them) - NewOrderCmp new_rid_order_cmp(new_order, num_reads); + // ec's will sort last, so we also change nec to exclude them); + // sort reads not used (no new_order entry) at end + std::vector rid_to_order(num_reads, std::numeric_limits::max()); + for (size_t i(0); i != new_order.size(); ++i) { + rid_to_order[new_order[i]] = i; + } + OrderCmp new_rid_order_cmp(rid_to_order); std::sort(ec_list, ec_list + nec, new_rid_order_cmp); nec = total_ec; } @@ -159,9 +162,10 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { idx_t nec; ExtensionCandidateCompressed* ec_list(load_partition_data(m4_file_name, nec)); - // if we're memory limited spend some cpu time to reduce number of passes + // if we're memory limited spend some cpu time to speed up passes + // (~16s for ~300s speedup in test case) if (data.rco.read_buffer_size) { - reorder_candidates(ec_list, nec, data.rco.reads_to_correct, data.reads.num_reads(), data.rco.min_cov); + reorder_candidates(ec_list, nec, data.rco.reads_to_correct ? data.rco.reads_to_correct : data.reads.num_reads(), data.reads.num_reads(), data.rco.min_cov); } else { std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySidAndScore()); } From dc89bb8677d4546ec8212a8300fb11d0ddf5d055 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 18 Apr 2019 15:20:33 -0500 Subject: [PATCH 24/56] convert id_list to vector<> --- src/common/alignment.h | 34 +++++------ src/common/defs.h | 5 +- src/mecat2cns/mecat_correction.cpp | 88 +++++++++++++++------------- src/mecat2cns/packed_db.h | 16 ++++- src/mecat2cns/reads_correction_aux.h | 19 ++---- 5 files changed, 87 insertions(+), 75 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index d10e6b1..ce6e65a 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -259,37 +259,37 @@ class M5Record { idx_t& m5qid() { return qid; } - const idx_t& m5qid() const { + const idx_t m5qid() const { return qid; } idx_t& m5qsize() { return qsize; } - const idx_t& m5qsize() const { + const idx_t m5qsize() const { return qsize; } idx_t& m5qoff() { return qstart; } - const idx_t& m5qoff() const { + const idx_t m5qoff() const { return qstart; } idx_t& m5qend() { return qend; } - const idx_t& m5qend() const { + const idx_t m5qend() const { return qend; } int& m5qdir() { return qdir; } - const int& m5qdir() const { + const int m5qdir() const { return qdir; } idx_t& m5sid() { return sid; } - const idx_t& m5sid() const { + const idx_t m5sid() const { return sid; } idx_t& m5ssize() { @@ -307,49 +307,49 @@ class M5Record { idx_t& m5send() { return send; } - const idx_t& m5send() const { + const idx_t m5send() const { return send; } int& m5sdir() { return sdir; } - const int& m5sdir() const { + const int m5sdir() const { return sdir; } int& m5score() { return score; } - const int& m5score() const { + const int m5score() const { return score; } int& m5mat() { return mat; } - const int& m5mat() const { + const int m5mat() const { return mat; } int& m5mis() { return mis; } - const int& m5mis() const { + const int m5mis() const { return mis; } int& m5ins() { return ins; } - const int& m5ins() const { + const int m5ins() const { return ins; } int& m5dels() { return dels; } - const int& m5dels() const { + const int m5dels() const { return dels; } int& m5mapq() { return mapq; } - const int& m5mapq() const { + const int m5mapq() const { return mapq; } char*& m5qaln() { @@ -373,19 +373,19 @@ class M5Record { double& m5ident() { return ident; } - const double& m5ident() const { + const double m5ident() const { return ident; } idx_t& m5qext() { return qext; } - const idx_t& m5qext() const { + const idx_t m5qext() const { return qext; } idx_t& m5sext() { return sext; } - const idx_t& m5sext() const { + const idx_t m5sext() const { return sext; } }; diff --git a/src/common/defs.h b/src/common/defs.h index 2433aaa..042d47e 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -88,6 +88,7 @@ do { \ #define safe_malloc(arr, type, count) \ do { \ size_t __sm__sz__ = sizeof(type) * (count); \ +std::cerr << __func__ << ": safe_malloc: allocating " << __sm__sz__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ (arr) = (type *)malloc(__sm__sz__); \ if (!(arr)) ERROR("malloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sm__sz__); \ } while(0) @@ -95,6 +96,7 @@ do { \ #define safe_calloc(arr, type, count) \ do { \ size_t __sc__sz__ = sizeof(type) * (count); \ +std::cerr << __func__ << " safe_calloc: allocating " << __sc__sz__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ (arr) = (type *)calloc(1, __sc__sz__); \ if (!(arr)) ERROR("calloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sc__sz__); \ } while(0) @@ -102,6 +104,7 @@ do { \ #define safe_realloc(arr, type, count) \ do { \ size_t __sr__size__ = sizeof(type) * count; \ +std::cerr << __func__ << " safe_realloc: allocating " << __sr__size__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ arr = (type *)realloc(arr, __sr__size__); \ if (!arr) \ { \ @@ -201,7 +204,7 @@ const u1_t* get_dna_complement_table(); #define FWD 0 #define REV 1 #define REVERSE_STRAND(s) (1-(s)) -#define MAX_SEQ_SIZE 5000000 +#define MAX_SEQ_SIZE 500000 #define MAX_INVALID_END_SIZE 200 #define MIN_EXTEND_SIZE 500 #define MIN_OVERLAP_SIZE 1000 diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 6b4a066..3c10777 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -13,7 +13,7 @@ namespace ns_meap_cns { // returns type of coverage present -inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item, const int min_cov) { +static inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item, const int min_cov) { const int cov(ceil((cns_item.mat_cnt + cns_item.ins_cnt) * 0.8)); uint1 ident; if (cns_item.mat_cnt >= cov) { // coverage is 80% or more matches @@ -39,7 +39,7 @@ struct CompareOverlapByOverlapSize } }; -void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* const cns_table, const char* const org_seq) { +static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* const cns_table, const char* const org_seq) { r_assert(qaln.size() == saln.size()); const idx_t aln_size(qaln.size()); for (idx_t i(0); i < aln_size; ) { @@ -64,7 +64,7 @@ void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t st } } -void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int min_cov, std::string& aux_qstr, std::string& aux_tstr, std::string& cns) { +static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int min_cov, std::string& aux_qstr, std::string& aux_tstr, std::string& cns) { AlnGraphBoost ag(se - sb + 1); int sb_out; for (CnsAln* a(cns_vec.begin()); a != cns_vec.end(); ++a) { @@ -76,7 +76,10 @@ void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int ag.consensus(min_cov * 0.4, cns); } -void meap_consensus_one_segment(const CnsTableItem* const cns_list, const int cns_list_size, uint1* const cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { +static void meap_consensus_one_segment(const CnsTableItem* const cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { + if (static_cast(cns_id_vec.size()) < cns_list_size) { + cns_id_vec.resize(cns_list_size); + } // get types of coverage for (int i(0); i < cns_list_size; ++i) { cns_id_vec[i] = identify_one_consensus_item(cns_list[i], min_cov); @@ -120,7 +123,7 @@ struct CmpMappingRangeBySoff { } }; -void get_effective_ranges(std::vector& mranges, std::vector& eranges, const int read_size, const int min_size) { +static void get_effective_ranges(std::vector& mranges, std::vector& eranges, const int read_size, const int min_size) { eranges.clear(); if (mranges.size() == 0) { return; @@ -166,7 +169,7 @@ void get_effective_ranges(std::vector& mranges, std::vector& cns_results, CnsResult& cr, const idx_t beg, @@ -201,7 +204,7 @@ output_cns_result(std::vector& cns_results, } } -inline bool +static inline bool check_ovlp_mapping_range(const int qb, const int qe, const int qs, const int sb, const int se, const int ss, double ratio) @@ -216,7 +219,7 @@ check_ovlp_mapping_range(const int qb, const int qe, const int qs, // look for areas of high coverage of about min_size or more, // improve them and stick on the results pile -void consensus_worker(const CnsTableItem* const cns_table, uint1* const id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { +static void consensus_worker(const CnsTableItem* const cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(ceil(0.95 * min_size)); @@ -251,7 +254,7 @@ static void decode_and_append_sequence(std::string& s, const char* const seq, id // same as consensus_worker, but produces entire read as one entry; // uncorrected sections are just copied as is; -void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { +static void consensus_worker_one_read(const CnsTableItem* const cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); @@ -303,8 +306,7 @@ void consensus_worker_one_read(const CnsTableItem* const cns_table, uint1* const void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidate* overlaps((ExtensionCandidate*)pctd.candidates); - DiffRunningData* const drd_s(pctd.drd_s); - DiffRunningData* drd(NULL); + DiffRunningData* const drd(&pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -331,7 +333,6 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); const idx_t sext(ovlp.sext); - drd = drd_s; const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size)); if (r) { normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); @@ -350,8 +351,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData { PackedDB& reads = ctd.reads; ExtensionCandidate* overlaps = (ExtensionCandidate*)pctd.candidates; - DiffRunningData* drd_s = pctd.drd_s; - DiffRunningData* drd = NULL; + DiffRunningData* drd = &pctd.drd; M5Record& m5 = pctd.m5; CnsAlns& cns_vec = pctd.cns_alns; std::vector& cns_results = pctd.cns_results; @@ -389,7 +389,6 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData idx_t qext = ovlp.qext; idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; - drd = drd_s; bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ovlp.qsize, m5soff(m5), m5send(m5), ovlp.ssize, min_mapping_ratio)) { @@ -405,26 +404,33 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } -inline bool -check_cov_stats(u1_t* cov_stats, int soff, int send) -{ - const int max_cov = 20; - int n = 0; - for (int i = soff; i < send; ++i) - if (cov_stats[i] >= max_cov) +// limit coverage - if there's not yet sufficient coverage, add coverage and return true; +// once coverage gets high enough, stop adding coverage and return false + +static inline int check_cov_stats(std::vector& cov_stats, const int soff, const int send) { + if (static_cast(cov_stats.size()) < send) { + cov_stats.resize(send, 0); + } + const int max_cov(20); + int n(0); + for (int i(soff); i < send; ++i) { + if (cov_stats[i] >= max_cov) { ++n; - if (send - soff >= n + 200) - { - for (int i = soff; i < send; ++i) ++cov_stats[i]; - return true; + } + } + if (send - soff >= n + 200) { + for (int i = soff; i < send; ++i) { + ++cov_stats[i]; + } + return 1; } - return false; + return 0; } void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { const PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); - DiffRunningData* const drd_s(pctd.drd_s); + DiffRunningData* const drd(&pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -432,7 +438,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); tstr.resize(ssize); - reads.GetSequence(read_id, true, tstr.data(), ssize); + reads.GetSequence(read_id, true, tstr.data()); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -444,8 +450,8 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; - u1_t* cov_stats(pctd.id_list); - std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); + std::vector& id_list(pctd.id_list); // used to be called cov_stats + id_list.clear(); for (idx_t i(sid); i < eid && num_added < max_added; ++i) { const ExtensionCandidateCompressed& ec(candidates[i]); if (used_ids.find(ec.qid) != used_ids.end()) { @@ -453,11 +459,11 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD } const idx_t qsize(reads.read_size(ec.qid)); qstr.resize(qsize); - reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data(), qsize); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data()); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr.data(), qext, qsize, tstr.data(), ec.sext, tstr.size(), drd_s, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr.data(), qext, qsize, tstr.data(), ec.sext, tstr.size(), drd, m5, 0.15, min_align_size)); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { - if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { + if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); @@ -479,8 +485,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); - DiffRunningData* const drd_s(pctd.drd_s); - DiffRunningData* drd(NULL); + DiffRunningData* const drd(&pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -488,7 +493,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; tstr.resize(ssize); - reads.GetSequence(read_id, true, tstr.data(), ssize); + reads.GetSequence(read_id, true, tstr.data()); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -500,8 +505,8 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); cns_vec.clear(); std::set used_ids; - u1_t* const cov_stats(pctd.id_list); - std::fill(cov_stats, cov_stats + MAX_SEQ_SIZE, 0); + std::vector& id_list(pctd.id_list); // used to be called cov_stats + id_list.clear(); for (idx_t i(sid); i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { ++num_ext; const ExtensionCandidateCompressed& ec(candidates[i]); @@ -510,13 +515,12 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea } const idx_t qsize(reads.read_size(ec.qid)); qstr.resize(qsize); - reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data(), qsize); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data()); const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - drd = drd_s; const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size)); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { - if (check_cov_stats(cov_stats, m5soff(m5), m5send(m5))) { + if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 2829310..54ed019 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -33,9 +33,23 @@ class PackedDB { void open_db(const std::string& filename, idx_t memory_footprint); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); + void GetSequence(const idx_t id, const bool forward, char* const seq) const { + const SeqIndex &si(seq_idx[id]); + if (forward) { + const idx_t offset(si.memory_offset); + for (idx_t i(0); i < si.size; ++i) { + seq[i] = get_char(offset + i); + } + } else { + const idx_t offset(si.memory_offset + si.size - 1); + for (idx_t i(0); i < si.size; ++i) { + seq[i] = 3 - get_char(offset - i); + } + } + } void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { const SeqIndex &si(seq_idx[id]); - //r_assert(size == si.size); // mostly obsolete now + r_assert(size == si.size); if (forward) { const idx_t offset(si.memory_offset); for (idx_t i(0); i < si.size; ++i) { diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index bd732ed..12dc8d6 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -168,10 +168,9 @@ class ConsensusPerThreadData { // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed // for candidate runs (to reduce memory usage) void* candidates; - ns_banded_sw::DiffRunningData* drd_s; - ns_banded_sw::DiffRunningData* drd_l; + ns_banded_sw::DiffRunningData drd; CnsTableItem cns_table[MAX_SEQ_SIZE]; - uint1 id_list[MAX_SEQ_SIZE]; + std::vector id_list; M5Record m5; CnsAlns cns_alns; std::vector cns_results; @@ -180,18 +179,10 @@ class ConsensusPerThreadData { std::string qaln; std::string saln; public: - ConsensusPerThreadData() : drd_s(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), drd_l(new ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_large())), m5(MAX_SEQ_SIZE) { + ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), m5(MAX_SEQ_SIZE) { cns_results.reserve(MAX_CNS_RESULTS); - // MAX_SEQ_SIZE is very large now, so let's not pre-allocate - //query.reserve(MAX_SEQ_SIZE); - //target.reserve(MAX_SEQ_SIZE); - //qaln.reserve(MAX_SEQ_SIZE); - //saln.reserve(MAX_SEQ_SIZE); - } - ~ConsensusPerThreadData() { - delete drd_s; - delete drd_l; } + ~ConsensusPerThreadData() { } }; class ConsensusThreadData { @@ -202,7 +193,7 @@ class ConsensusThreadData { pthread_mutex_t out_lock; idx_t ec_offset; // this doesn't work as a vector - all the pointers end up pointing - // to the same values, and eventually it seg faults (probably a + // to the same values, and eventually it seg faults (possibly a // compiler optimization bug) ConsensusPerThreadData* data; public: From 5e38cb840d8f3683bf2c092d6be42813d65f45dc Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 18 Apr 2019 15:49:06 -0500 Subject: [PATCH 25/56] changed cns_table to vector --- src/mecat2cns/dw.cpp | 30 ++++++++-------- src/mecat2cns/dw.h | 2 +- src/mecat2cns/mecat_correction.cpp | 52 +++++++++++++++------------- src/mecat2cns/reads_correction_aux.h | 20 +++-------- 4 files changed, 48 insertions(+), 56 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 323ce19..d65f186 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -459,8 +459,8 @@ int dw(const char* query, const int query_size, const int query_start, const cha return 1; } -bool GetAlignment(const char* const query, const int query_start, const int query_size, const char* const target, const int target_start, const int target_size, DiffRunningData* const drd, M5Record& m5, const double error_rate, const int min_aln_size) { - if (!dw(query, query_size, query_start, target, target_size, target_start, drd->DynQ, drd->DynT, drd->align, drd->d_path, drd->aln_path, drd->result, &drd->swp, error_rate, min_aln_size)) { +bool GetAlignment(const char* const query, const int query_start, const int query_size, const char* const target, const int target_start, const int target_size, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { + if (!dw(query, query_size, query_start, target, target_size, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, &drd.swp, error_rate, min_aln_size)) { return 0; } const int consecutive_match_region_size(4); @@ -469,9 +469,9 @@ bool GetAlignment(const char* const query, const int query_start, const int quer int trb(0); // t starting pads int eit(0); // matching run length int k; - for (k = 0; k < drd->result->out_store_size; ++k) { - const char qc(drd->result->out_store1[k]); - const char tc(drd->result->out_store2[k]); + for (k = 0; k < drd.result->out_store_size; ++k) { + const char qc(drd.result->out_store1[k]); + const char tc(drd.result->out_store2[k]); if (qc != '-') { ++qrb; } @@ -494,9 +494,9 @@ bool GetAlignment(const char* const query, const int query_start, const int quer // trim trailing end of alignment int qre(0); // q ending pads int tre(0); // t ending pads - for (k = drd->result->out_store_size - 1, eit = 0; start_aln_id < k; --k) { - const char qc(drd->result->out_store1[k]); - const char tc(drd->result->out_store2[k]); + for (k = drd.result->out_store_size - 1, eit = 0; start_aln_id < k; --k) { + const char qc(drd.result->out_store1[k]); + const char tc(drd.result->out_store2[k]); if (qc != '-') { ++qre; } @@ -514,17 +514,17 @@ bool GetAlignment(const char* const query, const int query_start, const int quer tre -= consecutive_match_region_size; const int end_aln_id(k + consecutive_match_region_size + 1); m5qsize(m5) = query_size; - m5qoff(m5) = drd->result->query_start + qrb; - m5qend(m5) = drd->result->query_end - qre; + m5qoff(m5) = drd.result->query_start + qrb; + m5qend(m5) = drd.result->query_end - qre; m5qdir(m5) = FWD; m5ssize(m5) = target_size; - m5soff(m5) = drd->result->target_start + trb; - m5send(m5) = drd->result->target_end - tre; + m5soff(m5) = drd.result->target_start + trb; + m5send(m5) = drd.result->target_end - tre; m5sdir(m5) = FWD; const int aln_size(end_aln_id - start_aln_id); - memcpy(m5qaln(m5), drd->result->out_store1 + start_aln_id, aln_size); - memcpy(m5saln(m5), drd->result->out_store2 + start_aln_id, aln_size); - memcpy(m5pat(m5), drd->result->out_match_pattern + start_aln_id, aln_size); + memcpy(m5qaln(m5), drd.result->out_store1 + start_aln_id, aln_size); + memcpy(m5saln(m5), drd.result->out_store2 + start_aln_id, aln_size); + memcpy(m5pat(m5), drd.result->out_match_pattern + start_aln_id, aln_size); m5qaln(m5)[aln_size] = '\0'; m5saln(m5)[aln_size] = '\0'; m5pat(m5)[aln_size] = '\0'; diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 927dc9b..6828fea 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -177,7 +177,7 @@ int dw(const char* query, const int query_size, const int query_start, PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, double error_rate, const int min_aln_size); -bool GetAlignment(const char* query, int query_start, int query_size, const char* target, int target_start, int target_size, DiffRunningData* drd, M5Record& m5, double error_rate, int min_aln_size); +bool GetAlignment(const char* query, int query_start, int query_size, const char* target, int target_start, int target_size, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); } // end of namespace ns_banded_sw diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 3c10777..d5e8502 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -2,6 +2,8 @@ #include "MECAT_AlnGraphBoost.H" +#include // numeric_limits::max() + using namespace ns_banded_sw; namespace ns_meap_cns { @@ -39,10 +41,10 @@ struct CompareOverlapByOverlapSize } }; -static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, CnsTableItem* const cns_table, const char* const org_seq) { +static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, std::vector& cns_table, const char* const org_seq) { r_assert(qaln.size() == saln.size()); const idx_t aln_size(qaln.size()); - for (idx_t i(0); i < aln_size; ) { + for (idx_t i(0); i < aln_size;) { const char q(qaln[i]); const char s(saln[i]); if (q == '-' && s == '-') { // skip @@ -76,13 +78,13 @@ static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, con ag.consensus(min_cov * 0.4, cns); } -static void meap_consensus_one_segment(const CnsTableItem* const cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { +static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { if (static_cast(cns_id_vec.size()) < cns_list_size) { cns_id_vec.resize(cns_list_size); } // get types of coverage for (int i(0); i < cns_list_size; ++i) { - cns_id_vec[i] = identify_one_consensus_item(cns_list[i], min_cov); + cns_id_vec[i] = identify_one_consensus_item(cns_list[start_soff + i], min_cov); } std::string cns; target.clear(); @@ -91,7 +93,7 @@ static void meap_consensus_one_segment(const CnsTableItem* const cns_list, const // advance to matching coverage for (; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } while (i < cns_list_size) { - target.push_back(cns_list[i].base); + target.push_back(cns_list[start_soff + i].base); const int start(i); // advance to next matching coverage for (++i; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } @@ -104,7 +106,7 @@ static void meap_consensus_one_segment(const CnsTableItem* const cns_list, const } } if (need_refinement) { - meap_cns_one_indel(start + start_soff, i + start_soff, cns_vec, cns_list[start].mat_cnt + cns_list[start].ins_cnt, aux_qstr, aux_tstr, cns); + meap_cns_one_indel(start_soff + start, start_soff + i, cns_vec, cns_list[start_soff + start].mat_cnt + cns_list[start_soff + start].ins_cnt, aux_qstr, aux_tstr, cns); // trim first and last as they have good coverage if (cns.size() > 2) { target.append(cns.data() + 1, cns.size() - 2); @@ -219,7 +221,7 @@ check_ovlp_mapping_range(const int qb, const int qe, const int qs, // look for areas of high coverage of about min_size or more, // improve them and stick on the results pile -static void consensus_worker(const CnsTableItem* const cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { +static void consensus_worker(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(ceil(0.95 * min_size)); @@ -254,7 +256,7 @@ static void decode_and_append_sequence(std::string& s, const char* const seq, id // same as consensus_worker, but produces entire read as one entry; // uncorrected sections are just copied as is; -static void consensus_worker_one_read(const CnsTableItem* const cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { +static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); @@ -306,7 +308,7 @@ static void consensus_worker_one_read(const CnsTableItem* const cns_table, std:: void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidate* overlaps((ExtensionCandidate*)pctd.candidates); - DiffRunningData* const drd(&pctd.drd); + DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -319,8 +321,8 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); const int max_added(60); - CnsTableItem* cns_table(pctd.cns_table); - std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); + std::vector& cns_table(pctd.cns_table); + cns_table.assign(read_size); // reset table cns_vec.clear(); const idx_t L(sid); const idx_t R(eid - sid <= max_added ? eid : L + max_added); @@ -351,7 +353,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData { PackedDB& reads = ctd.reads; ExtensionCandidate* overlaps = (ExtensionCandidate*)pctd.candidates; - DiffRunningData* drd = &pctd.drd; + DiffRunningData& drd = pctd.drd; M5Record& m5 = pctd.m5; CnsAlns& cns_vec = pctd.cns_alns; std::vector& cns_results = pctd.cns_results; @@ -378,8 +380,8 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); } - CnsTableItem* cns_table = pctd.cns_table; - std::for_each(cns_table, cns_table + read_size, CnsTableItemCleaner()); + std::vector& cns_table = pctd.cns_table; + cns_table.assign(read_size); // reset table cns_vec.clear(); for (idx_t i = L; i < R; ++i) { @@ -411,16 +413,18 @@ static inline int check_cov_stats(std::vector& cov_stats, const int soff, if (static_cast(cov_stats.size()) < send) { cov_stats.resize(send, 0); } - const int max_cov(20); int n(0); for (int i(soff); i < send; ++i) { - if (cov_stats[i] >= max_cov) { + if (cov_stats[i] >= 20) { // max coverage ++n; } } if (send - soff >= n + 200) { - for (int i = soff; i < send; ++i) { - ++cov_stats[i]; + for (int i(soff); i < send; ++i) { + // don't let small redundant region cause overflow + if (cov_stats[i] != std::numeric_limits::max()) { + ++cov_stats[i]; + } } return 1; } @@ -430,7 +434,7 @@ static inline int check_cov_stats(std::vector& cov_stats, const int soff, void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { const PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); - DiffRunningData* const drd(&pctd.drd); + DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -446,8 +450,8 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD int num_added(0); const int max_added(60); eid = std::min(eid, sid + 200); // max of 200 extents - CnsTableItem* cns_table(pctd.cns_table); - std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); + std::vector& cns_table(pctd.cns_table); + cns_table.assign(ssize); // reset table cns_vec.clear(); std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats @@ -485,7 +489,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); - DiffRunningData* const drd(&pctd.drd); + DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); @@ -501,8 +505,8 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea int num_added(0); int num_ext(0); const int max_ext(200); - CnsTableItem* cns_table(pctd.cns_table); - std::for_each(cns_table, cns_table + ssize, CnsTableItemCleaner()); + std::vector& cns_table(pctd.cns_table); + cns_table.assign(ssize); // reset table cns_vec.clear(); std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 12dc8d6..95eaa82 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -9,25 +9,12 @@ #include "packed_db.h" #include "options.h" -struct CnsTableItem -{ +struct CnsTableItem { char base; uint1 mat_cnt; uint1 ins_cnt; uint1 del_cnt; - - CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) {} -}; - -struct CnsTableItemCleaner -{ - void operator()(CnsTableItem& item) - { - item.base = 'N'; - item.mat_cnt = 0; - item.ins_cnt = 0; - item.del_cnt = 0; - } + CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } }; #define MAX_CNS_OVLPS 100 @@ -169,7 +156,7 @@ class ConsensusPerThreadData { // for candidate runs (to reduce memory usage) void* candidates; ns_banded_sw::DiffRunningData drd; - CnsTableItem cns_table[MAX_SEQ_SIZE]; + std::vector cns_table; std::vector id_list; M5Record m5; CnsAlns cns_alns; @@ -180,6 +167,7 @@ class ConsensusPerThreadData { std::string saln; public: ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), m5(MAX_SEQ_SIZE) { + // we'll definitely be seeing this much use, so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); } ~ConsensusPerThreadData() { } From cf9127a2621b5ec0296222adf83452cd254083ec Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 18 Apr 2019 15:52:35 -0500 Subject: [PATCH 26/56] whoops, forgot to check for errors before commiting --- src/mecat2cns/mecat_correction.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index d5e8502..56642a6 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -237,7 +237,7 @@ static void consensus_worker(const std::vector& cns_table, std::ve // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start >= min_size_95) { - meap_consensus_one_segment(cns_table + start, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); if (cns_seq.size() >= static_cast(min_size)) { output_cns_result(cns_results, cns_result, start, i, cns_seq); } @@ -285,7 +285,7 @@ static void consensus_worker_one_read(const std::vector& cns_table // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start > min_size_95) { - meap_consensus_one_segment(cns_table + start, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); if (cns_seq.size() >= static_cast(min_size)) { // add corrected sequence cns_result.seq += cns_seq; @@ -322,7 +322,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa const int min_align_size(ctd.rco.min_align_size); const int max_added(60); std::vector& cns_table(pctd.cns_table); - cns_table.assign(read_size); // reset table + cns_table.assign(read_size, CnsTableItem()); // reset table cns_vec.clear(); const idx_t L(sid); const idx_t R(eid - sid <= max_added ? eid : L + max_added); @@ -381,7 +381,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData } std::vector& cns_table = pctd.cns_table; - cns_table.assign(read_size); // reset table + cns_table.assign(read_size, CnsTableItem()); // reset table cns_vec.clear(); for (idx_t i = L; i < R; ++i) { @@ -451,7 +451,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const int max_added(60); eid = std::min(eid, sid + 200); // max of 200 extents std::vector& cns_table(pctd.cns_table); - cns_table.assign(ssize); // reset table + cns_table.assign(ssize, CnsTableItem()); // reset table cns_vec.clear(); std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats @@ -506,7 +506,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea int num_ext(0); const int max_ext(200); std::vector& cns_table(pctd.cns_table); - cns_table.assign(ssize); // reset table + cns_table.assign(ssize, CnsTableItem()); // reset table cns_vec.clear(); std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats From d924f37e6d67aa7ca990452580d614718a81b891 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 18 Apr 2019 16:30:28 -0500 Subject: [PATCH 27/56] changed CnsAln and CnsAlns to vectors --- src/mecat2cns/mecat_correction.cpp | 2 +- src/mecat2cns/reads_correction_aux.h | 92 ++++++++++++++-------------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 56642a6..6b8d901 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -69,7 +69,7 @@ static void meap_add_one_aln(const std::string& qaln, const std::string& saln, i static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int min_cov, std::string& aux_qstr, std::string& aux_tstr, std::string& cns) { AlnGraphBoost ag(se - sb + 1); int sb_out; - for (CnsAln* a(cns_vec.begin()); a != cns_vec.end(); ++a) { + for (std::vector::iterator a(cns_vec.begin()); a != cns_vec.end(); ++a) { if (a->retrieve_aln_subseqs(sb, se, aux_qstr, aux_tstr, sb_out)) { ag.addAln(aux_qstr, aux_tstr, sb_out - sb + 1); } diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 95eaa82..deeb2ae 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -19,83 +19,80 @@ struct CnsTableItem { #define MAX_CNS_OVLPS 100 -struct MappingRange -{ +class MappingRange { + public: int start, end; - - MappingRange(int s, int e) : start(s), end(e) {} + explicit MappingRange() : start(0), end(0) { } + explicit MappingRange(const int s, const int e) : start(s), end(e) { } + ~MappingRange() { } }; -struct CnsAln -{ - int soff, send, aln_idx, aln_size; - char qaln[MAX_SEQ_SIZE]; - char saln[MAX_SEQ_SIZE]; - - bool retrieve_aln_subseqs(int sb, int se, std::string& qstr, std::string& tstr, int& sb_out) - { - if (se <= soff || sb >= send || aln_idx >= aln_size - 1) return false; - sb_out = std::max(soff, sb); +class CnsAln : public MappingRange { + public: + explicit CnsAln() : aln_idx(0) { } + explicit CnsAln(const int i, const int j, const int k, const std::string& q, const std::string& s) : MappingRange(i, j), aln_idx(k), qaln(s), saln(q) { } + ~CnsAln() { } + // don't know why this skips the first basepair + int retrieve_aln_subseqs(const int sb, const int se, std::string& qstr, std::string& tstr, int& sb_out) { + const int aln_size(static_cast(saln.size()) - 1); + if (se <= start || sb >= end || aln_idx >= aln_size) { + return 0; + } + sb_out = std::max(start, sb); qstr.clear(); tstr.clear(); - while(soff < sb && aln_idx < aln_size - 1) - { + while (start < sb && aln_idx < aln_size) { ++aln_idx; - if (saln[aln_idx] != GAP) ++soff; + if (saln[aln_idx] != GAP) { + ++start; + } } qstr += qaln[aln_idx]; tstr += saln[aln_idx]; - while (soff < se && aln_idx < aln_size - 1) - { + while (start < se && aln_idx < aln_size) { ++aln_idx; - if (saln[aln_idx] != GAP) ++soff; + if (saln[aln_idx] != GAP) { + ++start; + } qstr += qaln[aln_idx]; tstr += saln[aln_idx]; } - return true; + return 1; } + private: + int aln_idx; + std::string qaln, saln; }; class CnsAlns { public: - CnsAlns() : num_alns_(0) { - safe_malloc(cns_alns_, CnsAln, MAX_CNS_OVLPS); - } - ~CnsAlns() { - safe_free(cns_alns_); - } + CnsAlns() { } + ~CnsAlns() { } void clear() { - num_alns_ = 0; + cns_alns_.clear(); } - int num_alns() const { - return num_alns_; + size_t num_alns() const { + return cns_alns_.size(); } - CnsAln* begin() { - return cns_alns_; + std::vector::iterator begin() { + return cns_alns_.begin(); } - CnsAln* end() { - return cns_alns_ + num_alns_; + std::vector::const_iterator end() const { + return cns_alns_.end(); } void add_aln(const int soff, const int send, const std::string& qstr, const std::string& tstr) { r_assert(qstr.size() == tstr.size()); - CnsAln& a(cns_alns_[num_alns_++]); - a.soff = soff; - a.send = send; - a.aln_idx = 0; - a.aln_size = qstr.size(); - memcpy(a.qaln, qstr.c_str(), a.aln_size + 1); - memcpy(a.saln, tstr.c_str(), a.aln_size + 1); + cns_alns_.push_back(CnsAln(soff, send, 0, qstr, tstr)); } void get_mapping_ranges(std::vector& ranges) const { ranges.clear(); - ranges.reserve(num_alns_); - for (int i(0); i < num_alns_; ++i) { - ranges.push_back(MappingRange(cns_alns_[i].soff, cns_alns_[i].send)); + ranges.reserve(cns_alns_.size()); + for (size_t i(0); i < cns_alns_.size(); ++i) { + ranges.push_back(cns_alns_[i]); } } private: - int num_alns_; - CnsAln* cns_alns_; + std::vector cns_alns_; }; // 1k seems to work a bit better than 10k - perhaps less time waiting for @@ -167,7 +164,8 @@ class ConsensusPerThreadData { std::string saln; public: ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), m5(MAX_SEQ_SIZE) { - // we'll definitely be seeing this much use, so might as well preallocate + // we'll definitely be seeing at least this much use, + // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); } ~ConsensusPerThreadData() { } From d872961ef5ad1a1c31883934108fd458d2a5a6bc Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 19 Apr 2019 12:34:11 -0500 Subject: [PATCH 28/56] removed some duplicate and obsolete code --- src/common/alignment.cpp | 72 ---------------------------- src/common/alignment.h | 4 -- src/common/defs.h | 3 -- src/mecat2cns/mecat_correction.cpp | 26 ++++------ src/mecat2cns/packed_db.cpp | 26 +++++----- src/mecat2cns/packed_db.h | 26 +++------- src/mecat2cns/reads_correction_aux.h | 4 +- 7 files changed, 28 insertions(+), 133 deletions(-) diff --git a/src/common/alignment.cpp b/src/common/alignment.cpp index aeea467..f7ee36b 100644 --- a/src/common/alignment.cpp +++ b/src/common/alignment.cpp @@ -77,77 +77,5 @@ std::ostream& operator<<(std::ostream& out, const M4Record& m4) return out; } -void PrintM5Record(std::ostream& out, const M5Record& m5, const int printAln) -{ - out << "(" << m5qid(m5) << ", " << m5qsize(m5) << ", " << m5qoff(m5) << ", " << m5qend(m5) << ", " << m5qdir(m5) << ")" - << " x " - << "(" << m5sid(m5) << ", " << m5ssize(m5) << ", " << m5soff(m5) << ", " << m5send(m5) << ", " << m5sdir(m5) << ")" - << ", score = " << m5score(m5) - << ", mapq = " << m5mapq(m5) - << ", (" << m5mat(m5) << ", " << m5mis(m5) << ", " << m5ins(m5) << ", " << m5dels(m5) << ")\n"; - - if (printAln) - { - out << m5qaln(m5) << "\n"; - out << m5pat(m5) << "\n"; - out << m5saln(m5) << "\n"; - } -} - -void InitM5Record(M5Record& m5) -{ - m5qid(m5) = m5qsize(m5) = m5qoff(m5) = m5qend(m5) = 0; - m5sid(m5) = m5ssize(m5) = m5soff(m5) = m5send(m5) = 0; - m5qdir(m5) = m5sdir(m5) = 2; - m5score(m5) = m5mat(m5) = m5mis(m5) = m5ins(m5) = m5dels(m5) = -1; - m5mapq(m5) = 0; - m5qaln(m5) = m5pat(m5) = m5saln(m5) = NULL; -} - -void DestroyM5Record(M5Record& m5) -{ - if (m5qaln(m5)) delete[] m5qaln(m5); - InitM5Record(m5); -} - -/* -std::istream& operator>>(std::istream& in, ReferenceMapping& rm) -{ - if (!in) return in; - if (!(in >> rmqid(rm))) return in; - in >> rmsid(rm) - >> rmqdir(rm) - >> rmqoff(rm) - >> rmqend(rm) - >> rmqext(rm) - >> rmqsize(rm) - >> rmsdir(rm) - >> rmsoff(rm) - >> rmsend(rm) - >> rmsext(rm) - >> rmssize(rm); - return in; -} - -std::ostream& operator<<(std::ostream& out, const ReferenceMapping& rm) -{ - constexpr char delim = '\t'; - out << rmqid(rm) << delim - << rmsid(rm) << delim - << rmqdir(rm) << delim - << rmqoff(rm) << delim - << rmqend(rm) << delim - << rmqext(rm) << delim - << rmqsize(rm) << delim - << rmsdir(rm) << delim - << rmsoff(rm) << delim - << rmsend(rm) << delim - << rmsext(rm) << delim - << rmssize(rm) - << "\n"; - return out; -} -*/ - const int64_t ExtensionCandidateCompressed::max_value = std::numeric_limits::max(); const int64_t ExtensionCandidateCompressed::max_qext = std::numeric_limits::max() >> 1; diff --git a/src/common/alignment.h b/src/common/alignment.h index ce6e65a..9fdde9d 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -413,10 +413,6 @@ class M5Record { #define m5qext(m) ((m).qext) #define m5sext(m) ((m).sext) -void PrintM5Record(std::ostream& out, const M5Record& m5, const int printAln); -void InitM5Record(M5Record& m5); -void DestroyM5Record(M5Record& m5); - inline M5Record* NewM5Record(const idx_t maxAlnSize) { return new M5Record(maxAlnSize); } diff --git a/src/common/defs.h b/src/common/defs.h index 042d47e..3bd5c19 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -88,7 +88,6 @@ do { \ #define safe_malloc(arr, type, count) \ do { \ size_t __sm__sz__ = sizeof(type) * (count); \ -std::cerr << __func__ << ": safe_malloc: allocating " << __sm__sz__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ (arr) = (type *)malloc(__sm__sz__); \ if (!(arr)) ERROR("malloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sm__sz__); \ } while(0) @@ -96,7 +95,6 @@ std::cerr << __func__ << ": safe_malloc: allocating " << __sm__sz__ << " bytes, #define safe_calloc(arr, type, count) \ do { \ size_t __sc__sz__ = sizeof(type) * (count); \ -std::cerr << __func__ << " safe_calloc: allocating " << __sc__sz__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ (arr) = (type *)calloc(1, __sc__sz__); \ if (!(arr)) ERROR("calloc fail: %lu * %lu = %lu", sizeof(type), size_t(count), __sc__sz__); \ } while(0) @@ -104,7 +102,6 @@ std::cerr << __func__ << " safe_calloc: allocating " << __sc__sz__ << " bytes, " #define safe_realloc(arr, type, count) \ do { \ size_t __sr__size__ = sizeof(type) * count; \ -std::cerr << __func__ << " safe_realloc: allocating " << __sr__size__ << " bytes, " << count << " of size " << sizeof(type) << "\n"; \ arr = (type *)realloc(arr, __sr__size__); \ if (!arr) \ { \ diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 6b8d901..9763e41 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -315,8 +315,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa const idx_t read_size(overlaps[read_id].ssize); std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); - tstr.resize(read_size); - reads.GetSequence(read_id, true, tstr.data(), read_size); + reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -331,8 +330,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa } for (idx_t i(L); i < R; ++i) { Overlap& ovlp(overlaps[i]); - qstr.resize(ovlp.qsize); - reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); + reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); const idx_t sext(ovlp.sext); const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size)); @@ -360,8 +358,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData const idx_t read_size = overlaps[read_id].ssize; std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; - tstr.resize(read_size); - reads.GetSequence(read_id, true, tstr.data(), read_size); + reads.GetSequence(read_id, 1, tstr); std::string& nqstr = pctd.qaln; std::string& ntstr = pctd.saln; const int min_align_size = ctd.rco.min_align_size; @@ -386,8 +383,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData for (idx_t i = L; i < R; ++i) { Overlap& ovlp = overlaps[i]; - qstr.resize(ovlp.qsize); - reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr.data(), ovlp.qsize); + reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); idx_t qext = ovlp.qext; idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; @@ -441,8 +437,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t ssize(reads.read_size(read_id)); std::vector& qstr(pctd.query); std::vector& tstr(pctd.target); - tstr.resize(ssize); - reads.GetSequence(read_id, true, tstr.data()); + reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -462,10 +457,9 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD continue; } const idx_t qsize(reads.read_size(ec.qid)); - qstr.resize(qsize); - reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data()); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr.data(), qext, qsize, tstr.data(), ec.sext, tstr.size(), drd, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), ec.sext, tstr.size(), drd, m5, 0.15, min_align_size)); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { ++num_added; @@ -496,8 +490,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea const idx_t ssize(reads.read_size(read_id)); std::vector& qstr = pctd.query; std::vector& tstr = pctd.target; - tstr.resize(ssize); - reads.GetSequence(read_id, true, tstr.data()); + reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); @@ -518,8 +511,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea continue; } const idx_t qsize(reads.read_size(ec.qid)); - qstr.resize(qsize); - reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr.data()); + reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size)); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 83ac77f..984b0cd 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -22,18 +22,18 @@ void PackedDB::add_one_seq(const Sequence& seq) { if (max_db_size < needed_size) { idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } - u1_t* new_pac(NULL); - safe_calloc(new_pac, u1_t, (new_size + 3) / 4); + uint1* new_pac(NULL); + safe_calloc(new_pac, uint1, (new_size + 3) / 4); memcpy(new_pac, pac, (db_size + 3) / 4); safe_free(pac); pac = new_pac; max_db_size = new_size; } const Sequence::str_t& org_seq(seq.sequence()); - const u1_t* const table(get_dna_encode_table()); + const uint1* const table(get_dna_encode_table()); unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion for (idx_t i(0); i < si.size; ++i, ++db_size) { - const u1_t c(table[static_cast(org_seq[i])]); + const uint1 c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); } } @@ -95,8 +95,8 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string return read_count; } DynamicTimer dtimer(__func__); - u1_t buffer[MAX_SEQ_SIZE]; - const u1_t* const et(get_dna_encode_table()); + std::vector buffer; + const uint1* const et(get_dna_encode_table()); FastaReader fr(fasta.c_str()); const std::string pac_name_tmp(pac_name + ".tmp"); const std::string index_name_tmp(index_name + ".tmp"); @@ -137,12 +137,12 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string Sequence::str_t& s(read.sequence()); const idx_t rbytes((rsize + 3) / 4); // set_char uses | to set bits, so clear first - bzero(buffer, rbytes); + buffer.assign(rbytes, 0); for (idx_t i(0); i < rsize; ++i) { - const u1_t c(et[static_cast(s[i])]); + const uint1 c(et[static_cast(s[i])]); set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } - if (!pout.write((char*)buffer, rbytes)) { + if (!pout.write((char*)&buffer[0], rbytes)) { ERROR("Write error to file %s", pac_name_tmp.c_str()); } iout << pac_offset << "\t" << rsize << "\n"; @@ -182,7 +182,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { const idx_t file_size(pstream.tellg()); max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { - safe_calloc(pac, u1_t, max_db_size); + safe_calloc(pac, uint1, max_db_size); } size_t read_count; if (!pstream.read((char*)&read_count, sizeof(size_t))) { @@ -214,7 +214,6 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { } close_fstream(index); } -std::cerr << "open_db: read count " << read_count << ", index size " << seq_idx.size() << ", max db size " << max_db_size << "\n"; } idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, const idx_t nec) { @@ -263,8 +262,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co LOG(stderr, "using %ld bytes for %lu reads, %ld aligns (out of %ld)", total_size, read_ids.size(), i, nec); if (max_db_size == 0) { max_db_size = total_size; - safe_calloc(pac, u1_t, max_db_size); -std::cerr << "load_reads: allocating " << max_db_size << "\n"; + safe_calloc(pac, uint1, max_db_size); } // now read in the reads std::set::const_iterator a(read_ids.begin()); @@ -306,7 +304,6 @@ void PackedDB::read_sizes(const std::string& output_prefix, std::vector& } in.close(); sizes.reserve(read_count); -std::cerr << "read_sizes: read_count " << read_count << "\n"; } const std::string index_name(output_prefix + ".idx"); open_fstream(in, index_name.c_str(), std::ios::in); @@ -314,5 +311,4 @@ std::cerr << "read_sizes: read_count " << read_count << "\n"; while (in >> i >> j) { sizes.push_back(j); } -std::cerr << "read_sizes: sizes size " << sizes.size() << "\n"; } diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 54ed019..0abea70 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -18,38 +18,24 @@ class PackedDB { idx_t memory_offset, size; }; public: - PackedDB() : pac(NULL), db_size(0), max_db_size(0) { } + explicit PackedDB() : pac(0), db_size(0), max_db_size(0) { } ~PackedDB() { if (pac) { safe_free(pac); } } - // only call one of load_fasta_db and open_db exactly once - void load_fasta_db(const char* fasta); // returns number of reads static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); static void read_sizes(const std::string& output_prefix, std::vector& sizes); + // only call one of load_fasta_db and open_db exactly once + void load_fasta_db(const char* fasta); // opens data file, reads in index file void open_db(const std::string& filename, idx_t memory_footprint); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); - void GetSequence(const idx_t id, const bool forward, char* const seq) const { - const SeqIndex &si(seq_idx[id]); - if (forward) { - const idx_t offset(si.memory_offset); - for (idx_t i(0); i < si.size; ++i) { - seq[i] = get_char(offset + i); - } - } else { - const idx_t offset(si.memory_offset + si.size - 1); - for (idx_t i(0); i < si.size; ++i) { - seq[i] = 3 - get_char(offset - i); - } - } - } - void GetSequence(const idx_t id, const bool forward, char* const seq, const idx_t size) const { + void GetSequence(const idx_t id, const bool forward, std::vector& seq) const { const SeqIndex &si(seq_idx[id]); - r_assert(size == si.size); + seq.resize(si.size); if (forward) { const idx_t offset(si.memory_offset); for (idx_t i(0); i < si.size; ++i) { @@ -69,7 +55,7 @@ class PackedDB { return seq_idx[read_id].size; } private: - static void set_char(u1_t* const p, const idx_t idx, const u1_t c) { + static void set_char(std::vector& p, const idx_t idx, const u1_t c) { p[idx >> 2] |= c << ((~idx & 3) << 1); } void set_char(const idx_t idx, const u1_t c) { diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index deeb2ae..2df99e6 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -152,10 +152,10 @@ class ConsensusPerThreadData { // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed // for candidate runs (to reduce memory usage) void* candidates; - ns_banded_sw::DiffRunningData drd; + ns_banded_sw::DiffRunningData drd; // XXX - reduce memory footprint std::vector cns_table; std::vector id_list; - M5Record m5; + M5Record m5; // XXX - reduce memory footprint CnsAlns cns_alns; std::vector cns_results; std::vector query; From 83c542504c7da6497efbb383d22b1a892773399e Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 19 Apr 2019 12:53:25 -0500 Subject: [PATCH 29/56] converted M5Record to use vector instead of malloc --- src/common/alignment.h | 44 ++++++++------------------ src/mecat2cns/dw.cpp | 9 ++---- src/mecat2cns/mecat_correction.cpp | 8 ++--- src/mecat2cns/reads_correction_aux.cpp | 7 ++-- src/mecat2cns/reads_correction_aux.h | 6 ++-- 5 files changed, 28 insertions(+), 46 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index 9fdde9d..2af3585 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -242,20 +242,15 @@ class M5Record { int ins; // 14) insertion int dels; // 15) deletion int mapq; // 16) mapQ - char* pm_q; // 17) aligned query - char* pm_p; // 18) aligned pattern - char* pm_s; // 19) aligned subject + std::string pm_q; // 17) aligned query + std::string pm_p; // 18) aligned pattern + std::string pm_s; // 19) aligned subject double ident; // 20) identity percentage idx_t qext; idx_t sext; public: - M5Record() : pm_q(0), pm_p(0), pm_s(0) { } - explicit M5Record(const idx_t n) : pm_q(new char[n]), pm_p(new char[n]), pm_s(new char[n]) { } - ~M5Record() { - delete[] pm_q; - delete[] pm_p; - delete[] pm_s; - } + explicit M5Record() { } + ~M5Record() { } idx_t& m5qid() { return qid; } @@ -352,22 +347,22 @@ class M5Record { const int m5mapq() const { return mapq; } - char*& m5qaln() { + std::string& m5qaln() { return pm_q; } - const char* m5qaln() const { + const std::string& m5qaln() const { return pm_q; } - char*& m5pat() { + std::string& m5pat() { return pm_p; } - const char* m5pat() const { + const std::string& m5pat() const { return pm_p; } - char*& m5saln() { + std::string& m5saln() { return pm_s; } - const char* m5saln() const { + const std::string& m5saln() const { return pm_s; } double& m5ident() { @@ -406,24 +401,13 @@ class M5Record { #define m5ins(m) ((m).ins) #define m5dels(m) ((m).dels) #define m5mapq(m) ((m).mapq) -#define m5qaln(m) ((m).pm_q) -#define m5pat(m) ((m).pm_p) -#define m5saln(m) ((m).pm_s) +//#define m5qaln(m) ((m).m5qaln()) +//#define m5pat(m) ((m).m5pat()) +//#define m5saln(m) ((m).m5saln()) #define m5ident(m) ((m).ident) #define m5qext(m) ((m).qext) #define m5sext(m) ((m).sext) -inline M5Record* NewM5Record(const idx_t maxAlnSize) { - return new M5Record(maxAlnSize); -} - -inline M5Record* DeleteM5Record(M5Record* const m5) { - if (m5) { - delete m5; - } - return NULL; -} - inline int M5RecordOvlpSize(const M5Record& m) { const int oq(m5qend(m) - m5qoff(m)); const int os(m5send(m) - m5soff(m)); diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index d65f186..ee79ee1 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -522,12 +522,9 @@ bool GetAlignment(const char* const query, const int query_start, const int quer m5send(m5) = drd.result->target_end - tre; m5sdir(m5) = FWD; const int aln_size(end_aln_id - start_aln_id); - memcpy(m5qaln(m5), drd.result->out_store1 + start_aln_id, aln_size); - memcpy(m5saln(m5), drd.result->out_store2 + start_aln_id, aln_size); - memcpy(m5pat(m5), drd.result->out_match_pattern + start_aln_id, aln_size); - m5qaln(m5)[aln_size] = '\0'; - m5saln(m5)[aln_size] = '\0'; - m5pat(m5)[aln_size] = '\0'; + m5.m5qaln().assign(drd.result->out_store1 + start_aln_id, aln_size); + m5.m5saln().assign(drd.result->out_store2 + start_aln_id, aln_size); + m5.m5pat().assign(drd.result->out_match_pattern + start_aln_id, aln_size); return 1; } diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 9763e41..5ee36ab 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -335,7 +335,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa const idx_t sext(ovlp.sext); const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size)); if (r) { - normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } @@ -390,7 +390,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ovlp.qsize, m5soff(m5), m5send(m5), ovlp.ssize, min_mapping_ratio)) { - normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } @@ -464,7 +464,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } @@ -519,7 +519,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5qaln(m5), m5saln(m5), strlen(m5qaln(m5)), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); } diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index ab9aac6..16c5706 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -1,6 +1,7 @@ #include "reads_correction_aux.h" +#include // string -void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, std::string& qnorm, std::string& tnorm, const bool push) +void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, const bool push) { qnorm.clear(); tnorm.clear(); @@ -8,7 +9,7 @@ void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, st #ifndef NDEBUG int qcnt = 0, tcnt = 0; - for (idx_t i = 0; i < aln_size; ++i) + for (size_t i = 0; i < qstr.size(); ++i) { const char qc = qstr[i]; const char tc = tstr[i]; @@ -18,7 +19,7 @@ void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, st #endif // convert mismatches to indels - for (idx_t i = 0; i < aln_size; ++i) + for (size_t i = 0; i < qstr.size(); ++i) { const char qc = qstr[i]; const char tc = tstr[i]; diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 2df99e6..0d44fca 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -155,7 +155,7 @@ class ConsensusPerThreadData { ns_banded_sw::DiffRunningData drd; // XXX - reduce memory footprint std::vector cns_table; std::vector id_list; - M5Record m5; // XXX - reduce memory footprint + M5Record m5; CnsAlns cns_alns; std::vector cns_results; std::vector query; @@ -163,7 +163,7 @@ class ConsensusPerThreadData { std::string qaln; std::string saln; public: - ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())), m5(MAX_SEQ_SIZE) { + ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())) { // we'll definitely be seeing at least this much use, // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); @@ -282,7 +282,7 @@ class ConsensusThreadData { std::string done_file_, ckpt_file_, ckpt_file_tmp_; }; -void normalize_gaps(const char* qstr, const char* tstr, const idx_t aln_size, std::string& qnorm, std::string& tnorm, bool push); +void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, bool push); void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, idx_t nec); void allocate_ecs(ConsensusThreadData &data, ExtensionCandidateCompressed* ec_list, idx_t nec); From 3b43bc2db44b231770b825db3f39c7db22b20c7a Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 19 Apr 2019 16:29:05 -0500 Subject: [PATCH 30/56] changed DiffRunningData from malloc to vectors mostly - two of the subclasses still have small mallocs --- src/mecat2cns/dw.cpp | 677 ++++++++++--------------- src/mecat2cns/dw.h | 239 ++++----- src/mecat2cns/mecat_correction.cpp | 8 +- src/mecat2cns/mecat_correction.h | 4 - src/mecat2cns/reads_correction_aux.h | 4 +- src/mecat2cns/reads_correction_can.cpp | 4 +- src/mecat2cns/reads_correction_m4.cpp | 4 +- 7 files changed, 359 insertions(+), 581 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index ee79ee1..e901562 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,466 +1,307 @@ #include "dw.h" #include // memcpy(), memset() - -namespace ns_banded_sw { +#include // vector<> #define GAP_ALN 4 -SW_Parameters -get_sw_parameters_small() -{ - SW_Parameters swp; - swp.segment_size = 500; - swp.row_size = 4096; - swp.column_size = 4096; - swp.segment_aln_size = 4096; - swp.max_seq_size = MAX_SEQ_SIZE; - swp.max_aln_size = MAX_SEQ_SIZE; - swp.d_path_size = 5000000; - swp.aln_path_size = 5000000; - - return swp; -} - -SW_Parameters -get_sw_parameters_large() -{ - SW_Parameters swp; - swp.segment_size = 1000; - swp.row_size = 4096; - swp.column_size = 4096; - swp.segment_aln_size = 4096; - swp.max_seq_size = MAX_SEQ_SIZE; - swp.max_aln_size = MAX_SEQ_SIZE; - swp.d_path_size = 5000000; - swp.aln_path_size = 5000000; - - return swp; +static int CompareDPathData2(const void* const a, const void* const b) { + const DPathData2* const d1((const DPathData2*)a); + const DPathData2* const d2((const DPathData2*)b); + return (d1->d != d2->d) ? (d1->d - d2->d) : (d1->k - d2->k); } -DiffRunningData::DiffRunningData(const SW_Parameters& swp_in) -{ - swp = swp_in; - safe_malloc(query, char, swp.max_seq_size); - safe_malloc(target, char, swp.max_seq_size); - safe_malloc(DynQ, int, swp.row_size); - safe_malloc(DynT, int, swp.column_size); - align = new Alignment(swp.segment_aln_size); - result = new OutputStore(swp.max_aln_size); - safe_malloc(d_path, DPathData2, swp.d_path_size); - safe_malloc(aln_path, PathPoint, swp.aln_path_size); -} - -DiffRunningData::~DiffRunningData() -{ - safe_free(query); - safe_free(target); - safe_free(DynQ); - safe_free(DynT); - delete align; - delete result; - safe_free(d_path); - safe_free(aln_path); -} - -void fill_m4record_from_output_store(const OutputStore& result, - const idx_t qid, - const idx_t sid, - const char qstrand, - const char sstrand, - const idx_t qsize, - const idx_t ssize, - const idx_t q_off_in_aln, - const idx_t s_off_in_aln, - const idx_t q_ext, - const idx_t s_ext, - M4Record& m4) -{ - m4qid(m4) = qid; - m4sid(m4) = sid; - m4ident(m4) = result.ident; - m4vscore(m4) = 100; - m4qdir(m4) = 1 - (qstrand == 'F'); - m4qoff(m4) = q_off_in_aln + result.query_start; - m4qend(m4) = q_off_in_aln + result.query_end; - m4qsize(m4) = qsize; - m4sdir(m4) = 1 - (sstrand == 'F'); - m4soff(m4) = s_off_in_aln + result.target_start; - m4send(m4) = s_off_in_aln + result.target_end; - m4ssize(m4) = ssize; - m4qext(m4) = q_ext; - m4sext(m4) = s_ext; - - if (m4qdir(m4) == 1) - { - m4qoff(m4) = qsize - (q_off_in_aln + result.query_end); - m4qend(m4) = qsize - (q_off_in_aln + result.query_start); - m4qext(m4) = qsize - 1 - q_ext; - } - if (m4sdir(m4) == 1) - { - m4soff(m4) = ssize - (s_off_in_aln + result.target_end); - m4send(m4) = ssize - (s_off_in_aln + result.target_start); - m4sext(m4) = ssize - 1 - s_ext; +static void fill_align(const char* const query, const char* const target, const int get_aln_str, Alignment& align, std::vector& d_path, std::vector& aln_path, const int right_extend, const size_t aln_path_max) { + align.init(); + align.aln_q_e = d_path.back().x2; + align.aln_t_e = d_path.back().y2; + align.dist = d_path.back().d; + align.aln_str_size = (align.aln_q_e + align.aln_t_e + align.dist) / 2; + if (get_aln_str) { + aln_path.clear(); + DPathData2 seek(align.dist, d_path.back().k); + for (; seek.d >= 0 && aln_path.size() < aln_path_max; --seek.d) { + // there may be a better approach here than bsearch() + const DPathData2* const d_path_aux((const DPathData2*)bsearch(&seek, &d_path[0], d_path.size(), sizeof(DPathData2), CompareDPathData2)); + aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); + aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); + seek.k = d_path_aux->pre_k; + } + std::vector::const_reverse_iterator a(aln_path.rbegin()); + const std::vector::const_reverse_iterator end_a(aln_path.rend()); + int current_x(a->x); + int current_y(a->y); + align.aln_q_s = current_x; + align.aln_t_s = current_y; + int aln_pos(0); + // starting increment is safe as we're guaranteed two entries at least + for (++a; a != end_a; ++a) { + const int new_x(a->x); + const int new_y(a->y); + const int dx(new_x - current_x); + const int dy(new_y - current_y); + if (dx == 0 && dy == 0) { + continue; + } else if (dx == 0 && dy != 0) { + if (right_extend) { + for (int i(0); i < dy; ++i) { + align.q_aln_str[aln_pos + i] = GAP_ALN; + align.t_aln_str[aln_pos + i] = target[current_y + i]; + } + } else { + for (int i(0); i < dy; ++i) { + align.q_aln_str[aln_pos + i] = GAP_ALN; + align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + } + } + aln_pos += dy; + } else if (dx != 0 && dy == 0) { + if (right_extend) { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[current_x + i]; + align.t_aln_str[aln_pos + i] = GAP_ALN; + } + } else { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; + align.t_aln_str[aln_pos + i] = GAP_ALN; + } + } + aln_pos += dx; + } else { + if (right_extend) { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[current_x + i]; + } + for (int i(0); i < dy; ++i) { + align.t_aln_str[aln_pos + i] = target[current_y + i]; + } + } else { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; + } + for (int i(0); i < dy; ++i) { + align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + } + } + aln_pos += dy; + } + current_x = new_x; + current_y = new_y; + } + align.aln_str_size = aln_pos; + } else { + align.aln_q_s = 0; + align.aln_t_s = 0; } } -void print_candidate(const CandidateStartPosition& csp) -{ - std::cout << "qoff = " << csp.qoff - << ", toff = " << csp.toff - << ", tstart = " << csp.tstart - << ", tsize = " << csp.tsize - << ", tid = " << csp.tid - << ", num1 = " << csp.num1 - << ", num2 = " << csp.num2 - << ", score = " << csp.score - << ", chain = " << csp.chain - << ", l1 = " << csp.left_q - << ", r1 = " << csp.right_q - << ", l2 = " << csp.left_t - << ", r2 = " << csp.right_t - << "\n"; -} - -int CompareDPathData2(const void* a, const void* b) -{ - const DPathData2* d1 = (const DPathData2*)a; - const DPathData2* d2 = (const DPathData2*)b; - return (d1->d == d2->d) ? (d1->k - d2->k) : (d1->d - d2->d); -} - -struct SCompareDPathData2 -{ - bool operator()(const DPathData2& a, const DPathData2& b) - { return (a.d == b.d) ? (a.k < b.k) : (a.d < b.d); } -}; - -DPathData2* GetDPathIdx(const int d, const int k, const unsigned int max_idx, DPathData2* base) -{ - DPathData2 target; - target.d = d; - target.k = k; - DPathData2* ret = (DPathData2*)bsearch(&target, base, max_idx, sizeof(DPathData2), CompareDPathData2); - return ret; -} - -int Align(const char* query, const int q_len, const char* target, const int t_len, - const int band_tolerance, const int get_aln_str, Alignment* align, - int* V, int* U, DPathData2* d_path, PathPoint* aln_path, - const int right_extend, double error_rate) -{ - int k_offset; - int d; - int k, k2; - int best_m; - int min_k, new_min_k, max_k, new_max_k, pre_k; - int x, y; - int ck, cd, cx, cy, nx, ny; - int max_d, band_size; - unsigned long d_path_idx = 0, max_idx = 0; - int aln_path_idx, aln_pos, i, aligned = 0; - DPathData2* d_path_aux; - - max_d = (int)(2.0 * error_rate * (q_len + t_len)); - k_offset = max_d; - band_size = band_tolerance * 2; - align->init(); - best_m = -1; - min_k = 0; - max_k = 0; - d_path_idx = 0; - max_idx = 0; - - for (d = 0; d < max_d; ++d) - { - if (max_k - min_k > band_size) break; - - for (k = min_k; k <= max_k; k += 2) - { - if( k == min_k || (k != max_k && V[k - 1 + k_offset] < V[k + 1 + k_offset]) ) - { pre_k = k + 1; x = V[k + 1 + k_offset]; } - else - { pre_k = k - 1; x = V[k - 1 + k_offset] + 1; } - y = x - k; - d_path[d_path_idx].d = d; - d_path[d_path_idx].k = k; - d_path[d_path_idx].x1 = x; - d_path[d_path_idx].y1 = y; - - if (right_extend) - while( x < q_len && y < t_len && query[x] == target[y]) { ++x; ++y; } - else - while( x < q_len && y < t_len && query[-x] == target[-y]) { ++x; ++y; } - - d_path[d_path_idx].x2 = x; - d_path[d_path_idx].y2 = y; - d_path[d_path_idx].pre_k = pre_k; - ++d_path_idx; - - V[k + k_offset] = x; - U[k + k_offset] = x + y; - best_m = std::max(best_m, x + y); - if (x >= q_len || y >= t_len) - { aligned = 1; max_idx = d_path_idx; break; } - } - - // for banding - new_min_k = max_k; - new_max_k = min_k; - for (k2 = min_k; k2 <= max_k; k2 += 2) - if (U[k2 + k_offset] >= best_m - band_tolerance) - { new_min_k = std::min(new_min_k, k2); new_max_k = std::max(new_max_k, k2); } - max_k = new_max_k + 1; - min_k = new_min_k - 1; - - if (aligned) - { - align->aln_q_e = x; - align->aln_t_e = y; - align->dist = d; - align->aln_str_size = (x + y + d) / 2; - align->aln_q_s = 0; - align->aln_t_s = 0; - - if (get_aln_str) - { - cd = d; - ck = k; - aln_path_idx = 0; - while (cd >= 0 && aln_path_idx < q_len + t_len + 1) - { - d_path_aux = GetDPathIdx(cd, ck, max_idx, d_path); - aln_path[aln_path_idx].x = d_path_aux->x2; - aln_path[aln_path_idx].y = d_path_aux->y2; - ++aln_path_idx; - aln_path[aln_path_idx].x = d_path_aux->x1; - aln_path[aln_path_idx].y = d_path_aux->y1; - ++aln_path_idx; - ck = d_path_aux->pre_k; - cd -= 1; - } - --aln_path_idx; - cx = aln_path[aln_path_idx].x; - cy = aln_path[aln_path_idx].y; - align->aln_q_s = cx; - align->aln_t_s = cy; - aln_pos = 0; - while (aln_path_idx > 0) - { - --aln_path_idx; - nx = aln_path[aln_path_idx].x; - ny = aln_path[aln_path_idx].y; - if (cx == nx && cy == ny) continue; - if (cx == nx && cy != ny) - { - if (right_extend) - { - for (i = 0; i < ny - cy; ++i) align->q_aln_str[aln_pos + i] = GAP_ALN; - for (i = 0; i < ny - cy; ++i) align->t_aln_str[aln_pos + i] = target[cy + i]; - } - else - { - for (i = 0; i < ny - cy; ++i) align->q_aln_str[aln_pos + i] = GAP_ALN; - for (i = 0; i < ny - cy; ++i) align->t_aln_str[aln_pos + i] = target[-(cy + i)]; - } - aln_pos += ny - cy; - } - else if (cx != nx && cy == ny) - { - if (right_extend) - { - for (i = 0; i < nx - cx; ++i) align->q_aln_str[aln_pos + i] = query[cx + i]; - for (i = 0; i < nx - cx; ++i) align->t_aln_str[aln_pos + i] = GAP_ALN; - } - else - { - for (i = 0; i < nx - cx; ++i) align->q_aln_str[aln_pos + i] = query[-(cx + i)]; - for (i = 0; i < nx - cx; ++i) align->t_aln_str[aln_pos + i] = GAP_ALN; - } - aln_pos += nx - cx; - } - else - { - if (right_extend) - { - for (i = 0; i < nx - cx; ++i) align->q_aln_str[aln_pos + i] = query[cx + i]; - for (i = 0; i < ny - cy; ++i) align->t_aln_str[aln_pos + i] = target[cy + i]; - } - else - { - for (i = 0; i < nx - cx; ++i) align->q_aln_str[aln_pos + i] = query[-(cx + i)]; - for (i = 0; i < ny - cy; ++i) align->t_aln_str[aln_pos + i] = target[-(cy + i)]; - } - aln_pos += ny - cy; - } - cx = nx; - cy = ny; - } - align->aln_str_size = aln_pos; - } - break; - } - } - if (align->aln_q_e == q_len || align->aln_t_e == t_len) return 1; - else return 0; +static int Align(const char* query, const int q_len, const char* target, const int t_len, const int band_tolerance, const int get_aln_str, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int right_extend, const double error_rate) { + const int k_offset(2 * error_rate * (q_len + t_len)); + const int band_size(band_tolerance * 2); + d_path.clear(); + int best_m(-1), min_k(0), max_k(0); + for (int d(0); d < k_offset && max_k - min_k <= band_size; ++d) { + for (int k(min_k); k <= max_k; k += 2) { + int x, pre_k; + if (k == min_k || (k != max_k && V[k - 1 + k_offset] < V[k + 1 + k_offset])) { + pre_k = k + 1; + x = V[k + 1 + k_offset]; + } else { + pre_k = k - 1; + x = V[k - 1 + k_offset] + 1; + } + int y(x - k); + const int x1(x), y1(y); + if (right_extend) { + while (x < q_len && y < t_len && query[x] == target[y]) { + ++x; + ++y; + } + } else { + while (x < q_len && y < t_len && query[-x] == target[-y]) { + ++x; + ++y; + } + } + d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); + if (x >= q_len || y >= t_len) { + fill_align(query, target, get_aln_str, align, d_path, aln_path, right_extend, q_len + t_len + 1); + return 1; + } + V[k + k_offset] = x; + U[k + k_offset] = x + y; + best_m = std::max(best_m, x + y); + } + // for banding + int new_min_k(max_k); + int new_max_k(min_k); + for (int k2(min_k); k2 <= max_k; k2 += 2) { + if (U[k2 + k_offset] >= best_m - band_tolerance) { + new_min_k = std::min(new_min_k, k2); + new_max_k = std::max(new_max_k, k2); + } + } + max_k = new_max_k + 1; + min_k = new_min_k - 1; + } + return 0; } -void dw_in_one_direction(const char* query, const int query_size, const char* target, const int target_size, - int* U, int* V, Alignment* align, DPathData2* d_path, PathPoint* aln_path, - SW_Parameters* swp, OutputStore* result, const int right_extend, double error_rate) -{ - const idx_t ALN_SIZE = swp->segment_size; - const idx_t U_SIZE = swp->row_size; - const idx_t V_SIZE = swp->column_size; - int extend1 = 0, extend2 = 0; - const char* seq1 = query; - const char* seq2 = target; - int extend_size = std::min(query_size, target_size); - int seg_size; - int flag_end = 1; - int align_flag; - int i, j, k, num_matches; - while (flag_end) - { - if (extend_size > (ALN_SIZE + 100)) - { seg_size = ALN_SIZE; } - else - { seg_size = extend_size; flag_end = 0; } - memset(U, 0, sizeof(int) * U_SIZE); - memset(V, 0, sizeof(int) * V_SIZE); - if (right_extend) { seq1 = query + extend1; seq2 = target + extend2; } - else { seq1 = query - extend1; seq2 = target - extend2; } - align_flag = Align(seq1, seg_size, seq2, seg_size, 0.3 * seg_size, 400, align, U, V, d_path, aln_path, right_extend, error_rate); - if (align_flag) - { - for (k = align->aln_str_size - 1, i = 0, j = 0, num_matches = 0; k > -1 && num_matches < 4; --k) - { - if (align->q_aln_str[k] != GAP_ALN) ++i; - if (align->t_aln_str[k] != GAP_ALN) ++j; - if (align->q_aln_str[k] == align->t_aln_str[k]) ++num_matches; - else num_matches = 0; - } - if (flag_end) - { - i = ALN_SIZE - align->aln_q_e + i; - j = ALN_SIZE - align->aln_t_e + j; - if (i == ALN_SIZE) align_flag = 0; - extend1 = extend1 + ALN_SIZE - i; extend2 = extend2 + ALN_SIZE - j; - } - else - { - i = extend_size - align->aln_q_e; - j = extend_size - align->aln_t_e; - if (i == extend_size) align_flag = 0; - extend1 += (extend_size - i); extend2 += (extend_size - j); - k = align->aln_str_size - 1; - } - if (align_flag) - { - if (right_extend) - { - memcpy(result->right_store1 + result->right_store_size, align->q_aln_str, k + 1); - memcpy(result->right_store2 + result->right_store_size, align->t_aln_str, k + 1); - result->right_store_size += (k + 1); - } - else - { - memcpy(result->left_store1 + result->left_store_size, align->q_aln_str, k + 1); - memcpy(result->left_store2 + result->left_store_size, align->t_aln_str, k + 1); - result->left_store_size += (k + 1); - } - extend_size = std::min(query_size - extend1, target_size - extend2); - } - } - if (!align_flag) break; - } +static void dw_in_one_direction(const char* query, const int query_size, const char* target, const int target_size, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, SW_Parameters& swp, OutputStore& result, const int right_extend, double error_rate) { + const idx_t ALN_SIZE(swp.segment_size); + const idx_t U_SIZE(swp.row_size); + const idx_t V_SIZE(swp.column_size); + int extend_size(std::min(query_size, target_size)); // size left to extend + int seg_size(ALN_SIZE); + int extend1(0), extend2(0); + for (int not_at_end(1); not_at_end;) { + if (extend_size <= ALN_SIZE + 100) { + seg_size = extend_size; + not_at_end = 0; + } + const char* seq1; + const char* seq2; + if (right_extend) { + seq1 = query + extend1; + seq2 = target + extend2; + } else { + seq1 = query - extend1; + seq2 = target - extend2; + } + U.assign(U_SIZE, 0); + V.assign(V_SIZE, 0); + if (!Align(seq1, seg_size, seq2, seg_size, seg_size * 0.3, 400, align, U, V, d_path, aln_path, right_extend, error_rate)) { + break; + } + int i(0), j(0), k, num_matches(0); + for (k = align.aln_str_size - 1; k > -1 && num_matches < 4; --k) { + if (align.q_aln_str[k] != GAP_ALN) { + ++i; + } + if (align.t_aln_str[k] != GAP_ALN) { + ++j; + } + if (align.q_aln_str[k] == align.t_aln_str[k]) { + ++num_matches; + } else { + num_matches = 0; + } + } + if (not_at_end) { + ++k; + i += ALN_SIZE - align.aln_q_e; + if (i == ALN_SIZE) { + break; + } + j += ALN_SIZE - align.aln_t_e; + extend1 += ALN_SIZE - i; + extend2 += ALN_SIZE - j; + } else { + if (align.aln_q_e == 0) { + break; + } + extend1 += align.aln_q_e; + extend2 += align.aln_t_e; + k = align.aln_str_size; + } + if (right_extend) { + memcpy(result.right_store1 + result.right_store_size, align.q_aln_str, k); + memcpy(result.right_store2 + result.right_store_size, align.t_aln_str, k); + result.right_store_size += k; + } else { + memcpy(result.left_store1 + result.left_store_size, align.q_aln_str, k); + memcpy(result.left_store2 + result.left_store_size, align.t_aln_str, k); + result.left_store_size += k; + } + extend_size = std::min(query_size - extend1, target_size - extend2); + } } -int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, int* U, int* V, Alignment* align, DPathData2* d_path, PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, double error_rate, const int min_aln_size) { - result->init(); - align->init(); +static int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, SW_Parameters& swp, double error_rate, const int min_aln_size) { + result.init(); + align.init(); // left extend dw_in_one_direction(query + query_start - 1, query_start, target + target_start - 1, target_start, U, V, align, d_path, aln_path, swp, result, 0, error_rate); - align->init(); + align.init(); // right extend dw_in_one_direction(query + query_start, query_size - query_start, target + target_start, target_size - target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); // merge the results int i, j, k, idx = 0; const char* encode2char("ACGT-"); - for (k = result->left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { - unsigned char ch(result->left_store1[k]); + for (k = result.left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { + unsigned char ch(result.left_store1[k]); r_assert(ch <= 4); ch = encode2char[ch]; - result->out_store1[idx] = ch; + result.out_store1[idx] = ch; if (ch != '-') { ++i; } - ch = result->left_store2[k]; + ch = result.left_store2[k]; r_assert(ch <= 4); ch = encode2char[ch]; - result->out_store2[idx] = ch; + result.out_store2[idx] = ch; if (ch != '-') { ++j; } } - result->query_start = query_start - i; - if (result->query_start < 0) { + result.query_start = query_start - i; + if (result.query_start < 0) { std::cerr << "query_start = " << query_start << ", i = " << i << "\n"; - r_assert(result->query_start >= 0); + r_assert(result.query_start >= 0); } - result->target_start = target_start - j; - r_assert(result->target_start >= 0); - for (k = 0, i = 0, j = 0; k < result->right_store_size; ++k, ++idx) { - unsigned char ch(result->right_store1[k]); + result.target_start = target_start - j; + r_assert(result.target_start >= 0); + for (k = 0, i = 0, j = 0; k < result.right_store_size; ++k, ++idx) { + unsigned char ch(result.right_store1[k]); r_assert(ch <= 4); ch = encode2char[ch]; - result->out_store1[idx] = ch; + result.out_store1[idx] = ch; if (ch != '-') { ++i; } - ch = result->right_store2[k]; + ch = result.right_store2[k]; r_assert(ch <= 4); ch = encode2char[ch]; - result->out_store2[idx] = ch; + result.out_store2[idx] = ch; if (ch != '-') { ++j; } } - result->out_store_size = idx; - result->query_end = query_start + i; - result->target_end = target_start + j; - if (result->out_store_size < min_aln_size) { + result.out_store_size = idx; + result.query_end = query_start + i; + result.target_end = target_start + j; + if (result.out_store_size < min_aln_size) { return 0; } int mat(0), mis(0), ins(0), del(0); - for (j = 0; j < result->out_store_size; ++j) { - if (result->out_store1[j] == result->out_store2[j]) { + for (j = 0; j < result.out_store_size; ++j) { + if (result.out_store1[j] == result.out_store2[j]) { ++mat; - result->out_match_pattern[j] = '|'; - } else if (result->out_store1[j] == '-') { + result.out_match_pattern[j] = '|'; + } else if (result.out_store1[j] == '-') { ++ins; - result->out_match_pattern[j] = '*'; - } else if (result->out_store2[j] == '-') { + result.out_match_pattern[j] = '*'; + } else if (result.out_store2[j] == '-') { ++del; - result->out_match_pattern[j] = '*'; + result.out_match_pattern[j] = '*'; } else { ++mis; - result->out_match_pattern[j] = '*'; + result.out_match_pattern[j] = '*'; } } - result->out_store1[result->out_store_size] = 0; - result->out_store2[result->out_store_size] = 0; - result->out_match_pattern[result->out_store_size] = 0; - result->mat = mat; - result->mis = mis; - result->ins = ins; - result->del = del; - result->ident = double(100) * mat / result->out_store_size; + result.out_store1[result.out_store_size] = 0; + result.out_store2[result.out_store_size] = 0; + result.out_match_pattern[result.out_store_size] = 0; + result.mat = mat; + result.mis = mis; + result.ins = ins; + result.del = del; + result.ident = double(100) * mat / result.out_store_size; return 1; } bool GetAlignment(const char* const query, const int query_start, const int query_size, const char* const target, const int target_start, const int target_size, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { - if (!dw(query, query_size, query_start, target, target_size, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, &drd.swp, error_rate, min_aln_size)) { + if (!dw(query, query_size, query_start, target, target_size, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.swp, error_rate, min_aln_size)) { return 0; } const int consecutive_match_region_size(4); @@ -469,9 +310,9 @@ bool GetAlignment(const char* const query, const int query_start, const int quer int trb(0); // t starting pads int eit(0); // matching run length int k; - for (k = 0; k < drd.result->out_store_size; ++k) { - const char qc(drd.result->out_store1[k]); - const char tc(drd.result->out_store2[k]); + for (k = 0; k < drd.result.out_store_size; ++k) { + const char qc(drd.result.out_store1[k]); + const char tc(drd.result.out_store2[k]); if (qc != '-') { ++qrb; } @@ -494,9 +335,9 @@ bool GetAlignment(const char* const query, const int query_start, const int quer // trim trailing end of alignment int qre(0); // q ending pads int tre(0); // t ending pads - for (k = drd.result->out_store_size - 1, eit = 0; start_aln_id < k; --k) { - const char qc(drd.result->out_store1[k]); - const char tc(drd.result->out_store2[k]); + for (k = drd.result.out_store_size - 1, eit = 0; start_aln_id < k; --k) { + const char qc(drd.result.out_store1[k]); + const char tc(drd.result.out_store2[k]); if (qc != '-') { ++qre; } @@ -514,18 +355,16 @@ bool GetAlignment(const char* const query, const int query_start, const int quer tre -= consecutive_match_region_size; const int end_aln_id(k + consecutive_match_region_size + 1); m5qsize(m5) = query_size; - m5qoff(m5) = drd.result->query_start + qrb; - m5qend(m5) = drd.result->query_end - qre; + m5qoff(m5) = drd.result.query_start + qrb; + m5qend(m5) = drd.result.query_end - qre; m5qdir(m5) = FWD; m5ssize(m5) = target_size; - m5soff(m5) = drd.result->target_start + trb; - m5send(m5) = drd.result->target_end - tre; + m5soff(m5) = drd.result.target_start + trb; + m5send(m5) = drd.result.target_end - tre; m5sdir(m5) = FWD; const int aln_size(end_aln_id - start_aln_id); - m5.m5qaln().assign(drd.result->out_store1 + start_aln_id, aln_size); - m5.m5saln().assign(drd.result->out_store2 + start_aln_id, aln_size); - m5.m5pat().assign(drd.result->out_match_pattern + start_aln_id, aln_size); + m5.m5qaln().assign(drd.result.out_store1 + start_aln_id, aln_size); + m5.m5saln().assign(drd.result.out_store2 + start_aln_id, aln_size); + m5.m5pat().assign(drd.result.out_match_pattern + start_aln_id, aln_size); return 1; } - -} // end namespace ns_banded_sw diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 6828fea..4856696 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -2,152 +2,119 @@ #define DW_H #include +#include // string +#include // vector<> #include "../common/alignment.h" #include "../common/defs.h" -namespace ns_banded_sw { - -struct SW_Parameters -{ - idx_t segment_size; - idx_t row_size; - idx_t column_size; - idx_t segment_aln_size; - idx_t max_seq_size; - idx_t max_aln_size; - idx_t d_path_size; - idx_t aln_path_size; +struct SW_Parameters { + idx_t segment_size; + idx_t row_size; + idx_t column_size; + idx_t segment_aln_size; + SW_Parameters(const idx_t i, const idx_t j, const idx_t k, const idx_t l) : segment_size(i), row_size(j), column_size(k), segment_aln_size(l) { } }; -SW_Parameters -get_sw_parameters_small(); - -SW_Parameters -get_sw_parameters_large(); - -struct Alignment -{ - int aln_str_size; - int dist; - int aln_q_s; - int aln_q_e; - int aln_t_s; - int aln_t_e; - char* q_aln_str; - char* t_aln_str; - - void init() - { - aln_str_size = 0; - aln_q_s = aln_q_e = 0; - aln_t_s = aln_t_e = 0; - } - - Alignment(const idx_t max_aln_size) - { - safe_malloc(q_aln_str, char, max_aln_size); - safe_malloc(t_aln_str, char, max_aln_size); - } - ~Alignment() - { - safe_free(q_aln_str); - safe_free(t_aln_str); - } +inline SW_Parameters get_sw_parameters_small() { + // 1000 instead of 500 for large + return SW_Parameters(500, 4096, 4096, 4096); +} + +struct Alignment { + int aln_str_size; + int dist; + int aln_q_s; + int aln_q_e; + int aln_t_s; + int aln_t_e; + char* q_aln_str; + char* t_aln_str; + void init() { + aln_str_size = 0; + aln_q_s = aln_q_e = 0; + aln_t_s = aln_t_e = 0; + } + Alignment(const idx_t max_aln_size) { + safe_malloc(q_aln_str, char, max_aln_size); + safe_malloc(t_aln_str, char, max_aln_size); + } + ~Alignment() { + safe_free(q_aln_str); + safe_free(t_aln_str); + } }; -struct OutputStore -{ - char* left_store1; - char* left_store2; - char* right_store1; - char* right_store2; - char* out_store1; - char* out_store2; - char* out_match_pattern; - - int left_store_size; - int right_store_size; - int out_store_size; - int query_start, query_end; - int target_start, target_end; - int mat, mis, ins, del; +struct OutputStore { + char* left_store1; + char* left_store2; + char* right_store1; + char* right_store2; + char* out_store1; + char* out_store2; + char* out_match_pattern; + int left_store_size; + int right_store_size; + int out_store_size; + int query_start, query_end; + int target_start, target_end; + int mat, mis, ins, del; double ident; - - OutputStore(const idx_t max_aln_size) - { - safe_malloc(left_store1, char, max_aln_size); - safe_malloc(left_store2, char, max_aln_size); - safe_malloc(right_store1, char, max_aln_size); - safe_malloc(right_store2, char, max_aln_size); - safe_malloc(out_store1, char, max_aln_size); - safe_malloc(out_store2, char, max_aln_size); - safe_malloc(out_match_pattern, char, max_aln_size); - } - - ~OutputStore() - { - safe_free(left_store1); - safe_free(left_store2); - safe_free(right_store1); - safe_free(right_store2); - safe_free(out_store1); - safe_free(out_store2); - safe_free(out_match_pattern); - } - - void init() - { - left_store_size = right_store_size = out_store_size = 0; - } + OutputStore(const idx_t max_aln_size) { + safe_malloc(left_store1, char, max_aln_size); + safe_malloc(left_store2, char, max_aln_size); + safe_malloc(right_store1, char, max_aln_size); + safe_malloc(right_store2, char, max_aln_size); + safe_malloc(out_store1, char, max_aln_size); + safe_malloc(out_store2, char, max_aln_size); + safe_malloc(out_match_pattern, char, max_aln_size); + } + ~OutputStore() { + safe_free(left_store1); + safe_free(left_store2); + safe_free(right_store1); + safe_free(right_store2); + safe_free(out_store1); + safe_free(out_store2); + safe_free(out_match_pattern); + } + void init() { + left_store_size = right_store_size = out_store_size = 0; + } }; -struct DPathData -{ - int pre_k, x1, y1, x2, y2; +struct DPathData { + int x1, y1, x2, y2, pre_k; + explicit DPathData() { } + explicit DPathData(const int i, const int j, const int k, const int l, const int m) : x1(i), y1(j), x2(k), y2(l), pre_k(m) { } }; -struct DPathData2 -{ - int d, k, pre_k, x1, y1, x2, y2; +struct DPathData2 : public DPathData { + int d, k; + explicit DPathData2(const int i, const int j) : d(i), k(j) { } + explicit DPathData2(const int i, const int j, const int k, const int l, const int m, const int n, const int p) : DPathData(k, l, m, n, p), d(i), k(j) { } }; -struct PathPoint -{ - int x, y; +struct PathPoint { + int x, y; + explicit PathPoint(const int i, const int j) : x(i), y(j) { } }; -struct DiffRunningData -{ - SW_Parameters swp; - char* query; - char* target; - int* DynQ; - int* DynT; - Alignment* align; - OutputStore* result; - DPathData2* d_path; - PathPoint* aln_path; - - DiffRunningData(const SW_Parameters& swp_in); - ~DiffRunningData(); +class DiffRunningData { + public: + SW_Parameters swp; + Alignment align; + OutputStore result; + std::string query, target; + std::vector DynQ, DynT; + std::vector d_path; + std::vector aln_path; + public: + explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in), align(swp_in.segment_aln_size), result(swp_in.segment_aln_size) { } + ~DiffRunningData() { } }; -void fill_m4record_from_output_store(const OutputStore& result, - const idx_t qid, - const idx_t sid, - const char qstrand, - const char sstrand, - const idx_t qsize, - const idx_t ssize, - const idx_t q_off_in_aln, - const idx_t s_off_in_aln, - const idx_t q_ext, - const idx_t s_ext, - M4Record& m4); - -struct CandidateStartPosition -{ +struct CandidateStartPosition { idx_t qoff; idx_t toff; idx_t tstart; @@ -161,24 +128,6 @@ struct CandidateStartPosition char chain; }; -void print_candidate(const CandidateStartPosition& csp); - -int Align(const char* query, const int q_len, const char* target, const int t_len, - const int band_tolerance, const int get_aln_str, Alignment* align, - int* V, int* U, DPathData2* d_path, PathPoint* aln_path, const int right_extend); - -void dw_in_one_direction(const char* query, const int query_size, const char* target, const int target_size, - int* U, int* V, Alignment* align, DPathData2* d_path, PathPoint* aln_path, - SW_Parameters* swp, OutputStore* result, const int right_extend); - -int dw(const char* query, const int query_size, const int query_start, - const char* target, const int target_size, const int target_start, - int* U, int* V, Alignment* align, DPathData2* d_path, - PathPoint* aln_path, OutputStore* result, SW_Parameters* swp, - double error_rate, const int min_aln_size); - bool GetAlignment(const char* query, int query_start, int query_size, const char* target, int target_start, int target_size, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); -} // end of namespace ns_banded_sw - #endif // DW_H diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 5ee36ab..fb8ed54 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -4,10 +4,6 @@ #include // numeric_limits::max() -using namespace ns_banded_sw; - -namespace ns_meap_cns { - #define FMAT 1 #define FDEL 2 #define FINS 4 @@ -67,7 +63,7 @@ static void meap_add_one_aln(const std::string& qaln, const std::string& saln, i } static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, const int min_cov, std::string& aux_qstr, std::string& aux_tstr, std::string& cns) { - AlnGraphBoost ag(se - sb + 1); + ns_meap_cns::AlnGraphBoost ag(se - sb + 1); int sb_out; for (std::vector::iterator a(cns_vec.begin()); a != cns_vec.end(); ++a) { if (a->retrieve_aln_subseqs(sb, se, aux_qstr, aux_tstr, sb_out)) { @@ -529,5 +525,3 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea eranges.push_back(MappingRange(0, ssize)); consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } - -} // namespace ns_meap_cns { diff --git a/src/mecat2cns/mecat_correction.h b/src/mecat2cns/mecat_correction.h index e2204ad..09bb8c5 100644 --- a/src/mecat2cns/mecat_correction.h +++ b/src/mecat2cns/mecat_correction.h @@ -3,8 +3,6 @@ #include "reads_correction_aux.h" -namespace ns_meap_cns { - void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); @@ -17,6 +15,4 @@ consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData & void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); -} // namespace ns_meap_cns - #endif // MEAP_CORRECTION_H diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 0d44fca..752021e 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -152,7 +152,7 @@ class ConsensusPerThreadData { // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed // for candidate runs (to reduce memory usage) void* candidates; - ns_banded_sw::DiffRunningData drd; // XXX - reduce memory footprint + DiffRunningData drd; // XXX - reduce memory footprint std::vector cns_table; std::vector id_list; M5Record m5; @@ -163,7 +163,7 @@ class ConsensusPerThreadData { std::string qaln; std::string saln; public: - ConsensusPerThreadData() : drd(ns_banded_sw::DiffRunningData(ns_banded_sw::get_sw_parameters_small())) { + ConsensusPerThreadData() : drd(DiffRunningData(get_sw_parameters_small())) { // we'll definitely be seeing at least this much use, // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index a2de09a..7cc15e4 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -25,7 +25,7 @@ static void* reads_correction_func_can(void* const arg) { if (i - start < data.rco.min_cov) { continue; } - ns_meap_cns::consensus_one_read_can_pacbio(data, pdata, sid, start, i); + consensus_one_read_can_pacbio(data, pdata, sid, start, i); if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { data.write_buffer(tid, i); } @@ -38,7 +38,7 @@ static void* reads_correction_func_can(void* const arg) { if (i - start < data.rco.min_cov) { continue; } - ns_meap_cns::consensus_one_read_can_nanopore(data, pdata, sid, start, i); + consensus_one_read_can_nanopore(data, pdata, sid, start, i); if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { data.write_buffer(tid, i); } diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index a7a0a97..0617493 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -30,9 +30,9 @@ void* reads_correction_func_m4(void* arg) { continue; } if (data.rco.tech == TECH_PACBIO) { - ns_meap_cns::consensus_one_read_m4_pacbio(data, pdata, sid, i, j); + consensus_one_read_m4_pacbio(data, pdata, sid, i, j); } else { - ns_meap_cns::consensus_one_read_m4_nanopore(data, pdata, sid, i, j); + consensus_one_read_m4_nanopore(data, pdata, sid, i, j); } if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { pthread_mutex_lock(&data.out_lock); From 21349b5ca48febd59e058db4303a087bbcb28f2a Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 22 Apr 2019 12:33:36 -0500 Subject: [PATCH 31/56] added max_aln_size back in after accidentally removing it --- src/mecat2cns/dw.cpp | 188 ++++++++++++++++++++++--------------------- src/mecat2cns/dw.h | 9 ++- 2 files changed, 101 insertions(+), 96 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index e901562..50b27d6 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -10,92 +10,88 @@ static int CompareDPathData2(const void* const a, const void* const b) { return (d1->d != d2->d) ? (d1->d - d2->d) : (d1->k - d2->k); } -static void fill_align(const char* const query, const char* const target, const int get_aln_str, Alignment& align, std::vector& d_path, std::vector& aln_path, const int right_extend, const size_t aln_path_max) { +static void fill_align(const char* const query, const char* const target, Alignment& align, std::vector& d_path, std::vector& aln_path, const int right_extend, const size_t aln_path_max) { align.init(); align.aln_q_e = d_path.back().x2; align.aln_t_e = d_path.back().y2; align.dist = d_path.back().d; align.aln_str_size = (align.aln_q_e + align.aln_t_e + align.dist) / 2; - if (get_aln_str) { - aln_path.clear(); - DPathData2 seek(align.dist, d_path.back().k); - for (; seek.d >= 0 && aln_path.size() < aln_path_max; --seek.d) { - // there may be a better approach here than bsearch() - const DPathData2* const d_path_aux((const DPathData2*)bsearch(&seek, &d_path[0], d_path.size(), sizeof(DPathData2), CompareDPathData2)); - aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); - aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); - seek.k = d_path_aux->pre_k; - } - std::vector::const_reverse_iterator a(aln_path.rbegin()); - const std::vector::const_reverse_iterator end_a(aln_path.rend()); - int current_x(a->x); - int current_y(a->y); - align.aln_q_s = current_x; - align.aln_t_s = current_y; - int aln_pos(0); - // starting increment is safe as we're guaranteed two entries at least - for (++a; a != end_a; ++a) { - const int new_x(a->x); - const int new_y(a->y); - const int dx(new_x - current_x); - const int dy(new_y - current_y); - if (dx == 0 && dy == 0) { - continue; - } else if (dx == 0 && dy != 0) { - if (right_extend) { - for (int i(0); i < dy; ++i) { - align.q_aln_str[aln_pos + i] = GAP_ALN; - align.t_aln_str[aln_pos + i] = target[current_y + i]; - } - } else { - for (int i(0); i < dy; ++i) { - align.q_aln_str[aln_pos + i] = GAP_ALN; - align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; - } + // get align path + aln_path.clear(); + DPathData2 seek(align.dist, d_path.back().k); + for (; seek.d >= 0 && aln_path.size() < aln_path_max; --seek.d) { + // there may be a better approach here than bsearch() + const DPathData2* const d_path_aux((const DPathData2*)bsearch(&seek, &d_path[0], d_path.size(), sizeof(DPathData2), CompareDPathData2)); + aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); + aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); + seek.k = d_path_aux->pre_k; + } + std::vector::const_reverse_iterator a(aln_path.rbegin()); + const std::vector::const_reverse_iterator end_a(aln_path.rend()); + int current_x(a->x); + int current_y(a->y); + align.aln_q_s = current_x; + align.aln_t_s = current_y; + int aln_pos(0); + // starting increment is safe as we're guaranteed two entries at least + for (++a; a != end_a; ++a) { + const int new_x(a->x); + const int new_y(a->y); + const int dx(new_x - current_x); + const int dy(new_y - current_y); + if (dx == 0 && dy == 0) { + continue; + } else if (dx == 0 && dy != 0) { + if (right_extend) { + for (int i(0); i < dy; ++i) { + align.q_aln_str[aln_pos + i] = GAP_ALN; + align.t_aln_str[aln_pos + i] = target[current_y + i]; } - aln_pos += dy; - } else if (dx != 0 && dy == 0) { - if (right_extend) { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[current_x + i]; - align.t_aln_str[aln_pos + i] = GAP_ALN; - } - } else { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; - align.t_aln_str[aln_pos + i] = GAP_ALN; - } + } else { + for (int i(0); i < dy; ++i) { + align.q_aln_str[aln_pos + i] = GAP_ALN; + align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + } + } + aln_pos += dy; + } else if (dx != 0 && dy == 0) { + if (right_extend) { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[current_x + i]; + align.t_aln_str[aln_pos + i] = GAP_ALN; } - aln_pos += dx; } else { - if (right_extend) { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[current_x + i]; - } - for (int i(0); i < dy; ++i) { - align.t_aln_str[aln_pos + i] = target[current_y + i]; - } - } else { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; - } - for (int i(0); i < dy; ++i) { - align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; - } + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; + align.t_aln_str[aln_pos + i] = GAP_ALN; } - aln_pos += dy; } - current_x = new_x; - current_y = new_y; + aln_pos += dx; + } else { + if (right_extend) { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[current_x + i]; + } + for (int i(0); i < dy; ++i) { + align.t_aln_str[aln_pos + i] = target[current_y + i]; + } + } else { + for (int i(0); i < dx; ++i) { + align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; + } + for (int i(0); i < dy; ++i) { + align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + } + } + aln_pos += dy; } - align.aln_str_size = aln_pos; - } else { - align.aln_q_s = 0; - align.aln_t_s = 0; + current_x = new_x; + current_y = new_y; } + align.aln_str_size = aln_pos; } -static int Align(const char* query, const int q_len, const char* target, const int t_len, const int band_tolerance, const int get_aln_str, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int right_extend, const double error_rate) { +static int Align(const char* const query, const int q_len, const char* const target, const int t_len, const int band_tolerance, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int right_extend, const double error_rate) { const int k_offset(2 * error_rate * (q_len + t_len)); const int band_size(band_tolerance * 2); d_path.clear(); @@ -125,7 +121,7 @@ static int Align(const char* query, const int q_len, const char* target, const i } d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); if (x >= q_len || y >= t_len) { - fill_align(query, target, get_aln_str, align, d_path, aln_path, right_extend, q_len + t_len + 1); + fill_align(query, target, align, d_path, aln_path, right_extend, q_len + t_len + 1); return 1; } V[k + k_offset] = x; @@ -147,7 +143,7 @@ static int Align(const char* query, const int q_len, const char* target, const i return 0; } -static void dw_in_one_direction(const char* query, const int query_size, const char* target, const int target_size, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, SW_Parameters& swp, OutputStore& result, const int right_extend, double error_rate) { +static void dw_in_one_direction(const char* const query, const int query_size, const char* const target, const int target_size, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const SW_Parameters& swp, OutputStore& result, const int right_extend, const double error_rate) { const idx_t ALN_SIZE(swp.segment_size); const idx_t U_SIZE(swp.row_size); const idx_t V_SIZE(swp.column_size); @@ -170,7 +166,7 @@ static void dw_in_one_direction(const char* query, const int query_size, const c } U.assign(U_SIZE, 0); V.assign(V_SIZE, 0); - if (!Align(seq1, seg_size, seq2, seg_size, seg_size * 0.3, 400, align, U, V, d_path, aln_path, right_extend, error_rate)) { + if (!Align(seq1, seg_size, seq2, seg_size, seg_size * 0.3, align, U, V, d_path, aln_path, right_extend, error_rate)) { break; } int i(0), j(0), k, num_matches(0); @@ -217,7 +213,7 @@ static void dw_in_one_direction(const char* query, const int query_size, const c } } -static int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, SW_Parameters& swp, double error_rate, const int min_aln_size) { +static int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const SW_Parameters& swp, const double error_rate, const int min_aln_size) { result.init(); align.init(); // left extend @@ -226,53 +222,61 @@ static int dw(const char* query, const int query_size, const int query_start, co // right extend dw_in_one_direction(query + query_start, query_size - query_start, target + target_start, target_size - target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); // merge the results - int i, j, k, idx = 0; - const char* encode2char("ACGT-"); + int i, j, k, idx(0); + const char* const encode2char("ACGT-"); for (k = result.left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { - unsigned char ch(result.left_store1[k]); - r_assert(ch <= 4); + int ch(result.left_store1[k]); + if (ch < 0 || 4 < ch) { + ERROR("Left1: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.left_store_size); + } ch = encode2char[ch]; result.out_store1[idx] = ch; if (ch != '-') { ++i; } ch = result.left_store2[k]; - r_assert(ch <= 4); + if (ch < 0 || 4 < ch) { + ERROR("Left2: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.left_store_size); + } ch = encode2char[ch]; result.out_store2[idx] = ch; if (ch != '-') { ++j; } } - result.query_start = query_start - i; - if (result.query_start < 0) { - std::cerr << "query_start = " << query_start << ", i = " << i << "\n"; - r_assert(result.query_start >= 0); + if (query_start < i) { + ERROR("query_start %d, i %d", query_start, i); + } else if (target_start < j) { + ERROR("target_start %d, j %d", target_start, j); } + result.query_start = query_start - i; result.target_start = target_start - j; - r_assert(result.target_start >= 0); for (k = 0, i = 0, j = 0; k < result.right_store_size; ++k, ++idx) { - unsigned char ch(result.right_store1[k]); - r_assert(ch <= 4); + int ch(result.right_store1[k]); + if (ch < 0 || 4 < ch) { + ERROR("Right1: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.right_store_size); + } ch = encode2char[ch]; result.out_store1[idx] = ch; if (ch != '-') { ++i; } ch = result.right_store2[k]; - r_assert(ch <= 4); + if (ch < 0 || 4 < ch) { + ERROR("Right2: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.right_store_size); + } ch = encode2char[ch]; result.out_store2[idx] = ch; if (ch != '-') { ++j; } } + if (idx < min_aln_size) { + return 0; + } result.out_store_size = idx; result.query_end = query_start + i; result.target_end = target_start + j; - if (result.out_store_size < min_aln_size) { - return 0; - } int mat(0), mis(0), ins(0), del(0); for (j = 0; j < result.out_store_size; ++j) { if (result.out_store1[j] == result.out_store2[j]) { diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 4856696..ff80516 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -13,12 +13,13 @@ struct SW_Parameters { idx_t row_size; idx_t column_size; idx_t segment_aln_size; - SW_Parameters(const idx_t i, const idx_t j, const idx_t k, const idx_t l) : segment_size(i), row_size(j), column_size(k), segment_aln_size(l) { } + idx_t max_aln_size; + SW_Parameters(const idx_t i, const idx_t j, const idx_t k, const idx_t l, const idx_t m) : segment_size(i), row_size(j), column_size(k), segment_aln_size(l), max_aln_size(m) { } }; inline SW_Parameters get_sw_parameters_small() { - // 1000 instead of 500 for large - return SW_Parameters(500, 4096, 4096, 4096); + // 1000 instead of 500 for "large" + return SW_Parameters(500, 4096, 4096, 4096, 2 * MAX_SEQ_SIZE); } struct Alignment { @@ -110,7 +111,7 @@ class DiffRunningData { std::vector d_path; std::vector aln_path; public: - explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in), align(swp_in.segment_aln_size), result(swp_in.segment_aln_size) { } + explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in), align(swp_in.segment_aln_size), result(swp_in.max_aln_size) { } ~DiffRunningData() { } }; From 625a676c43e63a6fa7a4cb79f3ddc3a5ef19271e Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 23 Apr 2019 09:29:04 -0500 Subject: [PATCH 32/56] finished changes from defined (large) array buffers to vectors and strings however, this does appear to have slowed thigns down a bit, I suspect mainly because of the clearing/recreating of strings, but that can be addressed now that we're off static arrays --- src/common/alignment.h | 48 +--- src/mecat2cns/dw.cpp | 327 +++++++++++---------------- src/mecat2cns/dw.h | 113 +++------ src/mecat2cns/mecat_correction.cpp | 66 +++--- src/mecat2cns/overlaps_partition.cpp | 2 - src/mecat2cns/packed_db.h | 2 +- src/mecat2cns/reads_correction_aux.h | 5 +- 7 files changed, 216 insertions(+), 347 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index 2af3585..994788d 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -385,55 +385,21 @@ class M5Record { } }; -#define m5qid(m) ((m).qid) -#define m5qsize(m) ((m).qsize) -#define m5qoff(m) ((m).qstart) -#define m5qend(m) ((m).qend) -#define m5qdir(m) ((m).qdir) -#define m5sid(m) ((m).sid) -#define m5ssize(m) ((m).ssize) -#define m5soff(m) ((m).sstart) -#define m5send(m) ((m).send) -#define m5sdir(m) ((m).sdir) -#define m5score(m) ((m).score) -#define m5mat(m) ((m).mat) -#define m5mis(m) ((m).mis) -#define m5ins(m) ((m).ins) -#define m5dels(m) ((m).dels) -#define m5mapq(m) ((m).mapq) -//#define m5qaln(m) ((m).m5qaln()) -//#define m5pat(m) ((m).m5pat()) -//#define m5saln(m) ((m).m5saln()) -#define m5ident(m) ((m).ident) -#define m5qext(m) ((m).qext) -#define m5sext(m) ((m).sext) - inline int M5RecordOvlpSize(const M5Record& m) { - const int oq(m5qend(m) - m5qoff(m)); - const int os(m5send(m) - m5soff(m)); + const int oq(m.m5qend() - m.m5qoff()); + const int os(m.m5send() - m.m5soff()); return std::max(oq, os); } -//struct Overlap -//{ -// idx_t qid, qoff, qend, qsize, qext; -// int qdir; -// idx_t sid, soff, send, ssize, sext; -// int sdir; -//}; - typedef ExtensionCandidate Overlap; -struct CompareOverlapBySid -{ - bool operator()(const Overlap& a, const Overlap& b) - { - return a.sid < b.sid; - } +struct CompareOverlapBySid { + bool operator()(const Overlap& a, const Overlap& b) { + return a.sid < b.sid; + } }; -struct CnsResult -{ +struct CnsResult { idx_t id; idx_t range[2]; std::string seq; diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 50b27d6..2b8f47b 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -10,88 +10,73 @@ static int CompareDPathData2(const void* const a, const void* const b) { return (d1->d != d2->d) ? (d1->d - d2->d) : (d1->k - d2->k); } -static void fill_align(const char* const query, const char* const target, Alignment& align, std::vector& d_path, std::vector& aln_path, const int right_extend, const size_t aln_path_max) { - align.init(); +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& d_path, std::vector& aln_path, const int extend_forward, const size_t aln_path_max) { + align.clear(); align.aln_q_e = d_path.back().x2; align.aln_t_e = d_path.back().y2; align.dist = d_path.back().d; - align.aln_str_size = (align.aln_q_e + align.aln_t_e + align.dist) / 2; // get align path aln_path.clear(); - DPathData2 seek(align.dist, d_path.back().k); + // no need to search for the first one + const DPathData2* d_path_aux(&d_path[0] + d_path.size() - 1); + aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); + aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); + DPathData2 seek(align.dist - 1, d_path.back().pre_k); for (; seek.d >= 0 && aln_path.size() < aln_path_max; --seek.d) { // there may be a better approach here than bsearch() - const DPathData2* const d_path_aux((const DPathData2*)bsearch(&seek, &d_path[0], d_path.size(), sizeof(DPathData2), CompareDPathData2)); + d_path_aux = (const DPathData2*)bsearch(&seek, &d_path[0], d_path_aux - &d_path[0], sizeof(DPathData2), CompareDPathData2); aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); seek.k = d_path_aux->pre_k; } std::vector::const_reverse_iterator a(aln_path.rbegin()); const std::vector::const_reverse_iterator end_a(aln_path.rend()); - int current_x(a->x); - int current_y(a->y); - align.aln_q_s = current_x; - align.aln_t_s = current_y; - int aln_pos(0); - // starting increment is safe as we're guaranteed two entries at least + align.aln_q_s = a->x; + align.aln_t_s = a->y; + int current_x(a->x), current_y(a->y); for (++a; a != end_a; ++a) { const int new_x(a->x); const int new_y(a->y); - const int dx(new_x - current_x); - const int dy(new_y - current_y); - if (dx == 0 && dy == 0) { - continue; - } else if (dx == 0 && dy != 0) { - if (right_extend) { - for (int i(0); i < dy; ++i) { - align.q_aln_str[aln_pos + i] = GAP_ALN; - align.t_aln_str[aln_pos + i] = target[current_y + i]; - } + if (current_x != new_x && current_y != new_y) { + if (extend_forward) { + align.q_aln_str.append(query, q_offset + current_x, new_x - current_x); + align.t_aln_str.append(target, t_offset + current_y, new_y - current_y); + current_x = new_x; + current_y = new_y; } else { - for (int i(0); i < dy; ++i) { - align.q_aln_str[aln_pos + i] = GAP_ALN; - align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + // don't know a clever way to append a reversed string + for (; current_x < new_x; ++current_x) { + align.q_aln_str += query[q_offset - current_x]; } - } - aln_pos += dy; - } else if (dx != 0 && dy == 0) { - if (right_extend) { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[current_x + i]; - align.t_aln_str[aln_pos + i] = GAP_ALN; + for (; current_y < new_y; ++current_y) { + align.t_aln_str += target[t_offset - current_y]; } + } + } else if (current_x != new_x) { + align.t_aln_str.append(new_x - current_x, GAP_ALN); + if (extend_forward) { + align.q_aln_str.append(query, q_offset + current_x, new_x - current_x); + current_x = new_x; } else { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; - align.t_aln_str[aln_pos + i] = GAP_ALN; + for (; current_x < new_x; ++current_x) { + align.q_aln_str += query[q_offset - current_x]; } } - aln_pos += dx; - } else { - if (right_extend) { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[current_x + i]; - } - for (int i(0); i < dy; ++i) { - align.t_aln_str[aln_pos + i] = target[current_y + i]; - } + } else if (current_y != new_y) { + align.q_aln_str.append(new_y - current_y, GAP_ALN); + if (extend_forward) { + align.t_aln_str.append(target, t_offset + current_y, new_y - current_y); + current_y = new_y; } else { - for (int i(0); i < dx; ++i) { - align.q_aln_str[aln_pos + i] = query[-(current_x + i)]; - } - for (int i(0); i < dy; ++i) { - align.t_aln_str[aln_pos + i] = target[-(current_y + i)]; + for (; current_y < new_y; ++current_y) { + align.t_aln_str += target[t_offset - current_y]; } } - aln_pos += dy; } - current_x = new_x; - current_y = new_y; } - align.aln_str_size = aln_pos; } -static int Align(const char* const query, const int q_len, const char* const target, const int t_len, const int band_tolerance, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int right_extend, const double error_rate) { +static int Align(const std::string& query, const int q_offset, const int q_len, const std::string& target, const int t_offset, const int t_len, const int band_tolerance, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(2 * error_rate * (q_len + t_len)); const int band_size(band_tolerance * 2); d_path.clear(); @@ -99,42 +84,36 @@ static int Align(const char* const query, const int q_len, const char* const tar for (int d(0); d < k_offset && max_k - min_k <= band_size; ++d) { for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; - if (k == min_k || (k != max_k && V[k - 1 + k_offset] < V[k + 1 + k_offset])) { + if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { pre_k = k + 1; - x = V[k + 1 + k_offset]; + x = V[k_offset + k + 1]; } else { pre_k = k - 1; - x = V[k - 1 + k_offset] + 1; + x = V[k_offset + k - 1] + 1; } int y(x - k); const int x1(x), y1(y); - if (right_extend) { - while (x < q_len && y < t_len && query[x] == target[y]) { - ++x; - ++y; - } + if (extend_forward) { + for (; x < q_len && y < t_len && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } } else { - while (x < q_len && y < t_len && query[-x] == target[-y]) { - ++x; - ++y; - } + for (; x < q_len && y < t_len && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); if (x >= q_len || y >= t_len) { - fill_align(query, target, align, d_path, aln_path, right_extend, q_len + t_len + 1); + fill_align(query, q_offset, target, t_offset, align, d_path, aln_path, extend_forward, q_len + t_len + 1); return 1; } - V[k + k_offset] = x; - U[k + k_offset] = x + y; + V[k_offset + k] = x; + U[k_offset + k] = x + y; best_m = std::max(best_m, x + y); } // for banding int new_min_k(max_k); int new_max_k(min_k); - for (int k2(min_k); k2 <= max_k; k2 += 2) { - if (U[k2 + k_offset] >= best_m - band_tolerance) { - new_min_k = std::min(new_min_k, k2); - new_max_k = std::max(new_max_k, k2); + for (int k(min_k); k <= max_k; k += 2) { + if (U[k_offset + k] >= best_m - band_tolerance) { + new_min_k = std::min(new_min_k, k); + new_max_k = std::max(new_max_k, k); } } max_k = new_max_k + 1; @@ -143,34 +122,31 @@ static int Align(const char* const query, const int q_len, const char* const tar return 0; } -static void dw_in_one_direction(const char* const query, const int query_size, const char* const target, const int target_size, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const SW_Parameters& swp, OutputStore& result, const int right_extend, const double error_rate) { - const idx_t ALN_SIZE(swp.segment_size); - const idx_t U_SIZE(swp.row_size); - const idx_t V_SIZE(swp.column_size); - int extend_size(std::min(query_size, target_size)); // size left to extend - int seg_size(ALN_SIZE); +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const SW_Parameters& swp, OutputStore& result, const int extend_forward, const double error_rate) { + const int seg_size(swp.segment_size); int extend1(0), extend2(0); for (int not_at_end(1); not_at_end;) { - if (extend_size <= ALN_SIZE + 100) { - seg_size = extend_size; - not_at_end = 0; - } - const char* seq1; - const char* seq2; - if (right_extend) { - seq1 = query + extend1; - seq2 = target + extend2; + // size left to extend + int extend_size; + if (extend_forward) { + extend_size = std::min(query.size() - q_offset - extend1, target.size() - t_offset - extend2); } else { - seq1 = query - extend1; - seq2 = target - extend2; + extend_size = std::min(q_offset - extend1, t_offset - extend2); } - U.assign(U_SIZE, 0); - V.assign(V_SIZE, 0); - if (!Align(seq1, seg_size, seq2, seg_size, seg_size * 0.3, align, U, V, d_path, aln_path, right_extend, error_rate)) { - break; + U.assign(swp.row_size, 0); + V.assign(swp.column_size, 0); + if (extend_size > seg_size + 100) { + if (!Align(query, q_offset + (extend_forward ? extend1 : -extend1), seg_size, target, t_offset + (extend_forward ? extend2 : -extend2), seg_size, seg_size * 0.3, align, U, V, d_path, aln_path, extend_forward, error_rate)) { + break; + } + } else { + if (!Align(query, q_offset + (extend_forward ? extend1 : -extend1), extend_size, target, t_offset + (extend_forward ? extend2 : -extend2), extend_size, extend_size * 0.3, align, U, V, d_path, aln_path, extend_forward, error_rate)) { + break; + } + not_at_end = 0; } - int i(0), j(0), k, num_matches(0); - for (k = align.aln_str_size - 1; k > -1 && num_matches < 4; --k) { + int k, i(0), j(0), num_matches(0); + for (k = align.q_aln_str.size() - 1; -1 < k && num_matches < 4; --k) { if (align.q_aln_str[k] != GAP_ALN) { ++i; } @@ -184,128 +160,100 @@ static void dw_in_one_direction(const char* const query, const int query_size, c } } if (not_at_end) { - ++k; - i += ALN_SIZE - align.aln_q_e; - if (i == ALN_SIZE) { + if (align.aln_q_e == i) { break; } - j += ALN_SIZE - align.aln_t_e; - extend1 += ALN_SIZE - i; - extend2 += ALN_SIZE - j; + extend1 += align.aln_q_e - i; + extend2 += align.aln_t_e - j; + ++k; + } else if (align.aln_q_e == 0) { + break; } else { - if (align.aln_q_e == 0) { - break; - } - extend1 += align.aln_q_e; - extend2 += align.aln_t_e; - k = align.aln_str_size; + k = align.q_aln_str.size(); } - if (right_extend) { - memcpy(result.right_store1 + result.right_store_size, align.q_aln_str, k); - memcpy(result.right_store2 + result.right_store_size, align.t_aln_str, k); - result.right_store_size += k; + if (extend_forward) { + result.right_store1 += align.q_aln_str.substr(0, k); + result.right_store2 += align.t_aln_str.substr(0, k); } else { - memcpy(result.left_store1 + result.left_store_size, align.q_aln_str, k); - memcpy(result.left_store2 + result.left_store_size, align.t_aln_str, k); - result.left_store_size += k; + result.left_store1 += align.q_aln_str.substr(0, k); + result.left_store2 += align.t_aln_str.substr(0, k); } - extend_size = std::min(query_size - extend1, target_size - extend2); } } -static int dw(const char* query, const int query_size, const int query_start, const char* target, const int target_size, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const SW_Parameters& swp, const double error_rate, const int min_aln_size) { - result.init(); - align.init(); - // left extend - dw_in_one_direction(query + query_start - 1, query_start, target + target_start - 1, target_start, U, V, align, d_path, aln_path, swp, result, 0, error_rate); - align.init(); - // right extend - dw_in_one_direction(query + query_start, query_size - query_start, target + target_start, target_size - target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const SW_Parameters& swp, const double error_rate, const size_t min_aln_size) { + result.clear(); + // reverse extend + dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, aln_path, swp, result, 0, error_rate); + // forward extend + dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); // merge the results - int i, j, k, idx(0); const char* const encode2char("ACGT-"); - for (k = result.left_store_size - 1, i = 0, j = 0; -1 < k; --k, ++idx) { + int i, j, k; + for (k = result.left_store1.size() - 1, i = 0, j = 0; -1 < k; --k) { int ch(result.left_store1[k]); if (ch < 0 || 4 < ch) { - ERROR("Left1: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.left_store_size); - } - ch = encode2char[ch]; - result.out_store1[idx] = ch; - if (ch != '-') { + ERROR("Left1: Out of range 0-4: %d (%d, %s)", ch, k, result.left_store1.c_str()); + } else if (ch != 4) { // not '-' ++i; } + result.out_store1 += encode2char[ch]; ch = result.left_store2[k]; if (ch < 0 || 4 < ch) { - ERROR("Left2: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.left_store_size); - } - ch = encode2char[ch]; - result.out_store2[idx] = ch; - if (ch != '-') { + ERROR("Left2: Out of range 0-4: %d (%d, %s)", ch, k, result.left_store2.c_str()); + } else if (ch != 4) { // not '-' ++j; } + result.out_store2 += encode2char[ch]; } if (query_start < i) { - ERROR("query_start %d, i %d", query_start, i); + ERROR("query_start %d, i %d, left_store_size1 %lu", query_start, i, result.left_store1.size()); } else if (target_start < j) { - ERROR("target_start %d, j %d", target_start, j); + ERROR("target_start %d, j %d, left_store_size2 %lu", target_start, j, result.left_store2.size()); } result.query_start = query_start - i; result.target_start = target_start - j; - for (k = 0, i = 0, j = 0; k < result.right_store_size; ++k, ++idx) { + for (k = 0, i = 0, j = 0; k < static_cast(result.right_store1.size()); ++k) { int ch(result.right_store1[k]); if (ch < 0 || 4 < ch) { - ERROR("Right1: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.right_store_size); - } - ch = encode2char[ch]; - result.out_store1[idx] = ch; - if (ch != '-') { + ERROR("Right1: Out of range 0-4: %d (%d, %s)", ch, k, result.right_store1.c_str()); + } else if (ch != 4) { // not '-' ++i; } + result.out_store1 += encode2char[ch]; ch = result.right_store2[k]; if (ch < 0 || 4 < ch) { - ERROR("Right2: Out of range 0-4: %d (%d, %d, %d)", ch, k, idx, result.right_store_size); - } - ch = encode2char[ch]; - result.out_store2[idx] = ch; - if (ch != '-') { + ERROR("Right2: Out of range 0-4: %d (%d, %s)", ch, k, result.right_store2.c_str()); + } else if (ch != 4) { // not '-' ++j; } + result.out_store2 += encode2char[ch]; } - if (idx < min_aln_size) { + if (result.out_store1.size() < min_aln_size) { return 0; } - result.out_store_size = idx; result.query_end = query_start + i; result.target_end = target_start + j; - int mat(0), mis(0), ins(0), del(0); - for (j = 0; j < result.out_store_size; ++j) { - if (result.out_store1[j] == result.out_store2[j]) { - ++mat; - result.out_match_pattern[j] = '|'; - } else if (result.out_store1[j] == '-') { - ++ins; - result.out_match_pattern[j] = '*'; - } else if (result.out_store2[j] == '-') { - ++del; - result.out_match_pattern[j] = '*'; + for (size_t m(0); m < result.out_store1.size(); ++m) { + if (result.out_store1[m] == result.out_store2[m]) { + ++result.mat; + result.out_match_pattern += '|'; + } else if (result.out_store1[m] == '-') { + ++result.ins; + result.out_match_pattern += '*'; + } else if (result.out_store2[m] == '-') { + ++result.del; + result.out_match_pattern += '*'; } else { - ++mis; - result.out_match_pattern[j] = '*'; + ++result.mis; + result.out_match_pattern += '*'; } } - result.out_store1[result.out_store_size] = 0; - result.out_store2[result.out_store_size] = 0; - result.out_match_pattern[result.out_store_size] = 0; - result.mat = mat; - result.mis = mis; - result.ins = ins; - result.del = del; - result.ident = double(100) * mat / result.out_store_size; return 1; } -bool GetAlignment(const char* const query, const int query_start, const int query_size, const char* const target, const int target_start, const int target_size, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { - if (!dw(query, query_size, query_start, target, target_size, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.swp, error_rate, min_aln_size)) { +bool GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { + if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.swp, error_rate, min_aln_size)) { return 0; } const int consecutive_match_region_size(4); @@ -313,8 +261,8 @@ bool GetAlignment(const char* const query, const int query_start, const int quer int qrb(0); // q starting pads int trb(0); // t starting pads int eit(0); // matching run length - int k; - for (k = 0; k < drd.result.out_store_size; ++k) { + size_t k; + for (k = 0; k < drd.result.out_store1.size(); ++k) { const char qc(drd.result.out_store1[k]); const char tc(drd.result.out_store2[k]); if (qc != '-') { @@ -335,11 +283,11 @@ bool GetAlignment(const char* const query, const int query_start, const int quer } qrb -= consecutive_match_region_size; trb -= consecutive_match_region_size; - const int start_aln_id(k - consecutive_match_region_size); + const size_t start_aln_id(k - consecutive_match_region_size); // trim trailing end of alignment int qre(0); // q ending pads int tre(0); // t ending pads - for (k = drd.result.out_store_size - 1, eit = 0; start_aln_id < k; --k) { + for (k = drd.result.out_store1.size() - 1, eit = 0; start_aln_id < k; --k) { const char qc(drd.result.out_store1[k]); const char tc(drd.result.out_store2[k]); if (qc != '-') { @@ -357,18 +305,17 @@ bool GetAlignment(const char* const query, const int query_start, const int quer } qre -= consecutive_match_region_size; tre -= consecutive_match_region_size; - const int end_aln_id(k + consecutive_match_region_size + 1); - m5qsize(m5) = query_size; - m5qoff(m5) = drd.result.query_start + qrb; - m5qend(m5) = drd.result.query_end - qre; - m5qdir(m5) = FWD; - m5ssize(m5) = target_size; - m5soff(m5) = drd.result.target_start + trb; - m5send(m5) = drd.result.target_end - tre; - m5sdir(m5) = FWD; - const int aln_size(end_aln_id - start_aln_id); - m5.m5qaln().assign(drd.result.out_store1 + start_aln_id, aln_size); - m5.m5saln().assign(drd.result.out_store2 + start_aln_id, aln_size); - m5.m5pat().assign(drd.result.out_match_pattern + start_aln_id, aln_size); + const size_t aln_size(k + consecutive_match_region_size + 1 - start_aln_id); + m5.m5qsize() = query.size(); + m5.m5qoff() = drd.result.query_start + qrb; + m5.m5qend() = drd.result.query_end - qre; + m5.m5qdir() = FWD; + m5.m5ssize() = target.size(); + m5.m5soff() = drd.result.target_start + trb; + m5.m5send() = drd.result.target_end - tre; + m5.m5sdir() = FWD; + m5.m5qaln() = drd.result.out_store1.substr(start_aln_id, aln_size); + m5.m5saln() = drd.result.out_store2.substr(start_aln_id, aln_size); + m5.m5pat() = drd.result.out_match_pattern.substr(start_aln_id, aln_size); return 1; } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index ff80516..960cc87 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -9,78 +9,53 @@ #include "../common/defs.h" struct SW_Parameters { - idx_t segment_size; - idx_t row_size; - idx_t column_size; - idx_t segment_aln_size; - idx_t max_aln_size; - SW_Parameters(const idx_t i, const idx_t j, const idx_t k, const idx_t l, const idx_t m) : segment_size(i), row_size(j), column_size(k), segment_aln_size(l), max_aln_size(m) { } + int segment_size; + int row_size; + int column_size; + SW_Parameters(const int i, const int j, const int k) : segment_size(i), row_size(j), column_size(k) { } }; inline SW_Parameters get_sw_parameters_small() { // 1000 instead of 500 for "large" - return SW_Parameters(500, 4096, 4096, 4096, 2 * MAX_SEQ_SIZE); + return SW_Parameters(500, 4096, 4096); } -struct Alignment { - int aln_str_size; +class Alignment { + public: int dist; - int aln_q_s; - int aln_q_e; - int aln_t_s; - int aln_t_e; - char* q_aln_str; - char* t_aln_str; - void init() { - aln_str_size = 0; - aln_q_s = aln_q_e = 0; - aln_t_s = aln_t_e = 0; - } - Alignment(const idx_t max_aln_size) { - safe_malloc(q_aln_str, char, max_aln_size); - safe_malloc(t_aln_str, char, max_aln_size); - } - ~Alignment() { - safe_free(q_aln_str); - safe_free(t_aln_str); + int aln_q_s, aln_q_e; + int aln_t_s, aln_t_e; + std::string q_aln_str; + std::string t_aln_str; + public: + explicit Alignment() { } + ~Alignment() { } + void clear() { + q_aln_str.clear(); + t_aln_str.clear(); } }; -struct OutputStore { - char* left_store1; - char* left_store2; - char* right_store1; - char* right_store2; - char* out_store1; - char* out_store2; - char* out_match_pattern; - int left_store_size; - int right_store_size; - int out_store_size; +class OutputStore { + public: int query_start, query_end; int target_start, target_end; int mat, mis, ins, del; - double ident; - OutputStore(const idx_t max_aln_size) { - safe_malloc(left_store1, char, max_aln_size); - safe_malloc(left_store2, char, max_aln_size); - safe_malloc(right_store1, char, max_aln_size); - safe_malloc(right_store2, char, max_aln_size); - safe_malloc(out_store1, char, max_aln_size); - safe_malloc(out_store2, char, max_aln_size); - safe_malloc(out_match_pattern, char, max_aln_size); - } - ~OutputStore() { - safe_free(left_store1); - safe_free(left_store2); - safe_free(right_store1); - safe_free(right_store2); - safe_free(out_store1); - safe_free(out_store2); - safe_free(out_match_pattern); - } - void init() { - left_store_size = right_store_size = out_store_size = 0; + std::string left_store1, left_store2; + std::string right_store1, right_store2; + std::string out_store1, out_store2; + std::string out_match_pattern; + public: + explicit OutputStore() { } + ~OutputStore() { } + void clear() { + left_store1.clear(); + left_store2.clear(); + right_store1.clear(); + right_store2.clear(); + out_store1.clear(); + out_store2.clear(); + out_match_pattern.clear(); } }; @@ -103,7 +78,7 @@ struct PathPoint { class DiffRunningData { public: - SW_Parameters swp; + const SW_Parameters swp; Alignment align; OutputStore result; std::string query, target; @@ -111,24 +86,10 @@ class DiffRunningData { std::vector d_path; std::vector aln_path; public: - explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in), align(swp_in.segment_aln_size), result(swp_in.max_aln_size) { } + explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in) { } ~DiffRunningData() { } }; -struct CandidateStartPosition { - idx_t qoff; - idx_t toff; - idx_t tstart; - idx_t tsize; - idx_t tid; - int left_q, left_t; - int right_q, right_t; - int num1, num2; - int score; - idx_t toff_in_aln; - char chain; -}; - -bool GetAlignment(const char* query, int query_start, int query_size, const char* target, int target_start, int target_size, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); +bool GetAlignment(const std::string& query, int query_start, const std::string& target, int target_start, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); #endif // DW_H diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index fb8ed54..2784610 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -37,7 +37,7 @@ struct CompareOverlapByOverlapSize } }; -static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, std::vector& cns_table, const char* const org_seq) { +static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, std::vector& cns_table) { r_assert(qaln.size() == saln.size()); const idx_t aln_size(qaln.size()); for (idx_t i(0); i < aln_size;) { @@ -242,7 +242,7 @@ static void consensus_worker(const std::vector& cns_table, std::ve } } -static void decode_and_append_sequence(std::string& s, const char* const seq, idx_t i, const idx_t end_i) { +static void decode_and_append_sequence(std::string& s, const std::string& seq, idx_t i, const idx_t end_i) { s.reserve(s.size() + end_i - i); for (; i < end_i; ++i) { s += "ACGT"[static_cast(seq[i])]; @@ -252,7 +252,7 @@ static void decode_and_append_sequence(std::string& s, const char* const seq, id // same as consensus_worker, but produces entire read as one entry; // uncorrected sections are just copied as is; -static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::vector& tstr, std::vector& cns_results) { +static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::string& tstr, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); @@ -262,9 +262,9 @@ static void consensus_worker_one_read(const std::vector& cns_table std::vector::const_iterator last_a(eranges.end()); for (; a != end_a; last_a = a++) { if (last_a != end_a) { // add in-between range to cns_result - decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, a->start); + decode_and_append_sequence(cns_result.seq, tstr, last_a->end, a->start); } else if (a->start > 0) { // add beginning of read - decode_and_append_sequence(cns_result.seq, tstr.data(), 0, a->start); + decode_and_append_sequence(cns_result.seq, tstr, 0, a->start); } const int begin_i(a->start - 1); const int end_i(a->end); @@ -273,7 +273,7 @@ static void consensus_worker_one_read(const std::vector& cns_table const idx_t last_end(i != begin_i ? i : a->start); for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } // add low coverage area as-is - decode_and_append_sequence(cns_result.seq, tstr.data(), last_end, i); + decode_and_append_sequence(cns_result.seq, tstr, last_end, i); if (i == end_i) { break; } @@ -289,12 +289,12 @@ static void consensus_worker_one_read(const std::vector& cns_table } } // add uncorrected sequence - decode_and_append_sequence(cns_result.seq, tstr.data(), start, i); + decode_and_append_sequence(cns_result.seq, tstr, start, i); } } // add end of read if (last_a != end_a) { - decode_and_append_sequence(cns_result.seq, tstr.data(), last_a->end, tstr.size()); + decode_and_append_sequence(cns_result.seq, tstr, last_a->end, tstr.size()); } cns_result.range[0] = 0; cns_result.range[1] = cns_result.seq.size(); @@ -309,8 +309,8 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); const idx_t read_size(overlaps[read_id].ssize); - std::vector& qstr(pctd.query); - std::vector& tstr(pctd.target); + std::string& qstr(pctd.query); + std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); @@ -329,11 +329,11 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); const idx_t sext(ovlp.sext); - const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.15, min_align_size)); if (r) { normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); + meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); + cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } } std::vector mranges, eranges; @@ -352,8 +352,8 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData CnsAlns& cns_vec = pctd.cns_alns; std::vector& cns_results = pctd.cns_results; const idx_t read_size = overlaps[read_id].ssize; - std::vector& qstr = pctd.query; - std::vector& tstr = pctd.target; + std::string& qstr = pctd.query; + std::string& tstr = pctd.target; reads.GetSequence(read_id, 1, tstr); std::string& nqstr = pctd.qaln; std::string& ntstr = pctd.saln; @@ -383,12 +383,12 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData idx_t qext = ovlp.qext; idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; - bool r = GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size); - if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), ovlp.qsize, m5soff(m5), m5send(m5), ovlp.ssize, min_mapping_ratio)) + bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size); + if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), ovlp.qsize, m5.m5soff(), m5.m5send(), ovlp.ssize, min_mapping_ratio)) { normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); + meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); + cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } } @@ -431,8 +431,8 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); const idx_t ssize(reads.read_size(read_id)); - std::vector& qstr(pctd.query); - std::vector& tstr(pctd.target); + std::string& qstr(pctd.query); + std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); @@ -455,14 +455,14 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), ec.sext, tstr.size(), drd, m5, 0.15, min_align_size)); - if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { - if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { + const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); + if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), qsize, m5.m5soff(), m5.m5send(), ssize, min_mapping_ratio)) { + if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { ++num_added; used_ids.insert(ec.qid); normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); + meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); + cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } } } @@ -484,8 +484,8 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); const idx_t ssize(reads.read_size(read_id)); - std::vector& qstr = pctd.query; - std::vector& tstr = pctd.target; + std::string& qstr(pctd.query); + std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); @@ -510,14 +510,14 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr.data(), qext, qstr.size(), tstr.data(), sext, tstr.size(), drd, m5, 0.20, min_align_size)); - if (r && check_ovlp_mapping_range(m5qoff(m5), m5qend(m5), qsize, m5soff(m5), m5send(m5), ssize, min_mapping_ratio)) { - if (check_cov_stats(id_list, m5soff(m5), m5send(m5))) { + const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size)); + if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), qsize, m5.m5soff(), m5.m5send(), ssize, min_mapping_ratio)) { + if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { ++num_added; used_ids.insert(ec.qid); normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); - meap_add_one_aln(nqstr, ntstr, m5soff(m5), cns_table, tstr.data()); - cns_vec.add_aln(m5soff(m5), m5send(m5), nqstr, ntstr); + meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); + cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } } } diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index b644f92..da50e07 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -12,8 +12,6 @@ #include "reads_correction_aux.h" #include "packed_db.h" // PackedDB -#define error_and_exit(msg) { std::cerr << msg << "\n"; abort(); } - inline static bool query_is_contained(const M4Record& m4, const double min_cov_ratio) { return m4qend(m4) - m4qoff(m4) >= m4qsize(m4) * min_cov_ratio; } diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 0abea70..20097d2 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -33,7 +33,7 @@ class PackedDB { void open_db(const std::string& filename, idx_t memory_footprint); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); - void GetSequence(const idx_t id, const bool forward, std::vector& seq) const { + void GetSequence(const idx_t id, const bool forward, std::string& seq) const { const SeqIndex &si(seq_idx[id]); seq.resize(si.size); if (forward) { diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 752021e..e7de2f0 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -158,10 +158,7 @@ class ConsensusPerThreadData { M5Record m5; CnsAlns cns_alns; std::vector cns_results; - std::vector query; - std::vector target; - std::string qaln; - std::string saln; + std::string query, target, qaln, saln; public: ConsensusPerThreadData() : drd(DiffRunningData(get_sw_parameters_small())) { // we'll definitely be seeing at least this much use, From 5199f2544b084f7bec27bfcc5f855cdda19b14b2 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 29 Apr 2019 12:17:42 -0500 Subject: [PATCH 33/56] remove extra buffer read/write when getting alignments also created unified buffer for output instead of left/right buffers --- src/common/alignment.h | 8 +- src/main.mk | 3 + src/mecat2cns/dw.cpp | 295 ++++++++++++------------- src/mecat2cns/dw.h | 83 +++---- src/mecat2cns/mecat2cns.mk | 4 +- src/mecat2cns/mecat_correction.cpp | 8 +- src/mecat2cns/reads_correction_aux.cpp | 2 +- src/mecat2cns/reads_correction_aux.h | 20 +- 8 files changed, 204 insertions(+), 219 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index 994788d..e800457 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -243,7 +243,7 @@ class M5Record { int dels; // 15) deletion int mapq; // 16) mapQ std::string pm_q; // 17) aligned query - std::string pm_p; // 18) aligned pattern + //std::string pm_p; // 18) aligned pattern std::string pm_s; // 19) aligned subject double ident; // 20) identity percentage idx_t qext; @@ -353,12 +353,6 @@ class M5Record { const std::string& m5qaln() const { return pm_q; } - std::string& m5pat() { - return pm_p; - } - const std::string& m5pat() const { - return pm_p; - } std::string& m5saln() { return pm_s; } diff --git a/src/main.mk b/src/main.mk index 2cb52b3..55dfaa7 100644 --- a/src/main.mk +++ b/src/main.mk @@ -24,3 +24,6 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2ref/mecat2ref.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk + +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg +TGT_LDFLAGS := -pg diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 2b8f47b..6da7296 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,5 +1,5 @@ #include "dw.h" -#include // memcpy(), memset() +#include // copy(), fill() #include // vector<> #define GAP_ALN 4 @@ -10,78 +10,77 @@ static int CompareDPathData2(const void* const a, const void* const b) { return (d1->d != d2->d) ? (d1->d - d2->d) : (d1->k - d2->k); } -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& d_path, std::vector& aln_path, const int extend_forward, const size_t aln_path_max) { - align.clear(); - align.aln_q_e = d_path.back().x2; - align.aln_t_e = d_path.back().y2; - align.dist = d_path.back().d; +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& d_path, const size_t d_path_idx, std::vector& aln_path, const int extend_forward, const size_t aln_path_max) { + const DPathData2* d_path_aux(&d_path[d_path_idx - 1]); + align.dist = d_path_aux->d; + align.aln_q_e = d_path_aux->x2; + align.aln_t_e = d_path_aux->y2; // get align path - aln_path.clear(); + if (aln_path.size() < aln_path_max + 2) { + aln_path.resize(aln_path_max + 2); + } + size_t aln_idx(-1); // no need to search for the first one - const DPathData2* d_path_aux(&d_path[0] + d_path.size() - 1); - aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); - aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); - DPathData2 seek(align.dist - 1, d_path.back().pre_k); - for (; seek.d >= 0 && aln_path.size() < aln_path_max; --seek.d) { + aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); + aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); + DPathData2 seek(d_path_aux->d - 1, d_path_aux->pre_k); + for (; -1 < seek.d && aln_idx < aln_path_max; --seek.d) { // there may be a better approach here than bsearch() d_path_aux = (const DPathData2*)bsearch(&seek, &d_path[0], d_path_aux - &d_path[0], sizeof(DPathData2), CompareDPathData2); - aln_path.push_back(PathPoint(d_path_aux->x2, d_path_aux->y2)); - aln_path.push_back(PathPoint(d_path_aux->x1, d_path_aux->y1)); + aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); + aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); seek.k = d_path_aux->pre_k; } - std::vector::const_reverse_iterator a(aln_path.rbegin()); - const std::vector::const_reverse_iterator end_a(aln_path.rend()); - align.aln_q_s = a->x; - align.aln_t_s = a->y; - int current_x(a->x), current_y(a->y); - for (++a; a != end_a; ++a) { - const int new_x(a->x); - const int new_y(a->y); - if (current_x != new_x && current_y != new_y) { + int current_x(aln_path[aln_idx].x); + int current_y(aln_path[aln_idx].y); + align.reset(aln_path[0].x + aln_path[0].y); + // +1 so [-dx, 0) becomes (-dx, 0], to match [0, dx) + const char* const query_p(query.data() + q_offset + (extend_forward ? 0 : 1)); + const char* const target_p(target.data() + t_offset + (extend_forward ? 0 : 1)); + for (--aln_idx; aln_idx != size_t(-1); --aln_idx) { + const int new_x(aln_path[aln_idx].x); + const int new_y(aln_path[aln_idx].y); + const int dx(new_x - current_x); + const int dy(new_y - current_y); + if (dx && dy) { + // apparently, dx always equals dy in this case if (extend_forward) { - align.q_aln_str.append(query, q_offset + current_x, new_x - current_x); - align.t_aln_str.append(target, t_offset + current_y, new_y - current_y); - current_x = new_x; - current_y = new_y; + std::copy(query_p + current_x, query_p + new_x, align.q_aln_str.begin() + align.size); + std::copy(target_p + current_y, target_p + new_y, align.t_aln_str.begin() + align.size); } else { - // don't know a clever way to append a reversed string - for (; current_x < new_x; ++current_x) { - align.q_aln_str += query[q_offset - current_x]; - } - for (; current_y < new_y; ++current_y) { - align.t_aln_str += target[t_offset - current_y]; - } + std::reverse_copy(query_p - new_x, query_p - current_x, align.q_aln_str.begin() + align.size); + std::reverse_copy(target_p - new_y, target_p - current_y, align.t_aln_str.begin() + align.size); } - } else if (current_x != new_x) { - align.t_aln_str.append(new_x - current_x, GAP_ALN); + align.size += dx; + } else if (dx) { + std::fill(align.t_aln_str.begin() + align.size, align.t_aln_str.begin() + align.size + dx, GAP_ALN); if (extend_forward) { - align.q_aln_str.append(query, q_offset + current_x, new_x - current_x); - current_x = new_x; + std::copy(query_p + current_x, query_p + new_x, align.q_aln_str.begin() + align.size); } else { - for (; current_x < new_x; ++current_x) { - align.q_aln_str += query[q_offset - current_x]; - } + std::reverse_copy(query_p - new_x, query_p - current_x, align.q_aln_str.begin() + align.size); } - } else if (current_y != new_y) { - align.q_aln_str.append(new_y - current_y, GAP_ALN); + align.size += dx; + } else if (dy) { + std::fill(align.q_aln_str.begin() + align.size, align.q_aln_str.begin() + align.size + dy, GAP_ALN); if (extend_forward) { - align.t_aln_str.append(target, t_offset + current_y, new_y - current_y); - current_y = new_y; + std::copy(target_p + current_y, target_p + new_y, align.t_aln_str.begin() + align.size); } else { - for (; current_y < new_y; ++current_y) { - align.t_aln_str += target[t_offset - current_y]; - } + std::reverse_copy(target_p - new_y, target_p - current_y, align.t_aln_str.begin() + align.size); } + align.size += dy; } + current_x = new_x; + current_y = new_y; } } -static int Align(const std::string& query, const int q_offset, const int q_len, const std::string& target, const int t_offset, const int t_len, const int band_tolerance, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int extend_forward, const double error_rate) { - const int k_offset(2 * error_rate * (q_len + t_len)); - const int band_size(band_tolerance * 2); - d_path.clear(); +static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int extend_forward, const double error_rate) { + const int k_offset(segment_size * 4 * error_rate); + const int band_tolerance(segment_size / 10 * 3 + 1); + const int max_band_size(band_tolerance * 2 - 1); + size_t d_path_idx(0); int best_m(-1), min_k(0), max_k(0); - for (int d(0); d < k_offset && max_k - min_k <= band_size; ++d) { + for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { @@ -94,36 +93,47 @@ static int Align(const std::string& query, const int q_offset, const int q_len, int y(x - k); const int x1(x), y1(y); if (extend_forward) { - for (; x < q_len && y < t_len && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } + for (; x < segment_size && y < segment_size && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } } else { - for (; x < q_len && y < t_len && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } + for (; x < segment_size && y < segment_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } - d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); - if (x >= q_len || y >= t_len) { - fill_align(query, q_offset, target, t_offset, align, d_path, aln_path, extend_forward, q_len + t_len + 1); + if (d_path_idx != d_path.size()) { + d_path[d_path_idx].set(d, k, x1, y1, x, y, pre_k); + } else { + d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); + } + ++d_path_idx; + if (x == segment_size || y == segment_size) { + fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, aln_path, extend_forward, segment_size * 2); return 1; } V[k_offset + k] = x; U[k_offset + k] = x + y; - best_m = std::max(best_m, x + y); + if (best_m < x + y) { + best_m = x + y; + } } // for banding int new_min_k(max_k); int new_max_k(min_k); + const int min_u(best_m - band_tolerance); for (int k(min_k); k <= max_k; k += 2) { - if (U[k_offset + k] >= best_m - band_tolerance) { - new_min_k = std::min(new_min_k, k); - new_max_k = std::max(new_max_k, k); + if (min_u < U[k_offset + k]) { + if (new_min_k > k) { + new_min_k = k; + } + if (new_max_k < k) { + new_max_k = k; + } } } - max_k = new_max_k + 1; min_k = new_min_k - 1; + max_k = new_max_k + 1; } return 0; } -static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const SW_Parameters& swp, OutputStore& result, const int extend_forward, const double error_rate) { - const int seg_size(swp.segment_size); +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { int extend1(0), extend2(0); for (int not_at_end(1); not_at_end;) { // size left to extend @@ -133,20 +143,16 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } else { extend_size = std::min(q_offset - extend1, t_offset - extend2); } - U.assign(swp.row_size, 0); - V.assign(swp.column_size, 0); - if (extend_size > seg_size + 100) { - if (!Align(query, q_offset + (extend_forward ? extend1 : -extend1), seg_size, target, t_offset + (extend_forward ? extend2 : -extend2), seg_size, seg_size * 0.3, align, U, V, d_path, aln_path, extend_forward, error_rate)) { - break; - } - } else { - if (!Align(query, q_offset + (extend_forward ? extend1 : -extend1), extend_size, target, t_offset + (extend_forward ? extend2 : -extend2), extend_size, extend_size * 0.3, align, U, V, d_path, aln_path, extend_forward, error_rate)) { - break; - } + if (extend_size < segment_size + 101) { not_at_end = 0; } + U.assign(U.size(), 0); + V.assign(V.size(), 0); + if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? extend1 : -extend1), target, t_offset + (extend_forward ? extend2 : -extend2), align, U, V, d_path, aln_path, extend_forward, error_rate)) { + break; + } int k, i(0), j(0), num_matches(0); - for (k = align.q_aln_str.size() - 1; -1 < k && num_matches < 4; --k) { + for (k = align.size - 1; -1 < k && num_matches < 4; --k) { if (align.q_aln_str[k] != GAP_ALN) { ++i; } @@ -169,91 +175,65 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } else if (align.aln_q_e == 0) { break; } else { - k = align.q_aln_str.size(); + k = align.size; } if (extend_forward) { - result.right_store1 += align.q_aln_str.substr(0, k); - result.right_store2 += align.t_aln_str.substr(0, k); + std::copy(align.q_aln_str.begin(), align.q_aln_str.begin() + k, &result.q_buffer[result.buffer_start + result.right_size]); + std::copy(align.t_aln_str.begin(), align.t_aln_str.begin() + k, &result.t_buffer[result.buffer_start + result.right_size]); + result.right_size += k; } else { - result.left_store1 += align.q_aln_str.substr(0, k); - result.left_store2 += align.t_aln_str.substr(0, k); + result.left_size += k; + std::copy(align.q_aln_str.begin(), align.q_aln_str.begin() + k, &result.q_buffer[result.buffer_start - result.left_size]); + std::copy(align.t_aln_str.begin(), align.t_aln_str.begin() + k, &result.t_buffer[result.buffer_start - result.left_size]); } + } } -static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const SW_Parameters& swp, const double error_rate, const size_t min_aln_size) { - result.clear(); - // reverse extend - dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, aln_path, swp, result, 0, error_rate); - // forward extend - dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, aln_path, swp, result, 1, error_rate); - // merge the results - const char* const encode2char("ACGT-"); - int i, j, k; - for (k = result.left_store1.size() - 1, i = 0, j = 0; -1 < k; --k) { - int ch(result.left_store1[k]); - if (ch < 0 || 4 < ch) { - ERROR("Left1: Out of range 0-4: %d (%d, %s)", ch, k, result.left_store1.c_str()); - } else if (ch != 4) { // not '-' +static void count_basepairs(const OutputStore& result, int k, const int end_k, int i, int j) { + for (; k != end_k; ++k) { + int ch(result.q_buffer[k]); + assert(-1 < ch && ch < 5); + if (ch != GAP_ALN) { ++i; } - result.out_store1 += encode2char[ch]; - ch = result.left_store2[k]; - if (ch < 0 || 4 < ch) { - ERROR("Left2: Out of range 0-4: %d (%d, %s)", ch, k, result.left_store2.c_str()); - } else if (ch != 4) { // not '-' + ch = result.t_buffer[k]; + assert(-1 < ch && ch < 5); + if (ch != GAP_ALN) { ++j; } - result.out_store2 += encode2char[ch]; } - if (query_start < i) { - ERROR("query_start %d, i %d, left_store_size1 %lu", query_start, i, result.left_store1.size()); - } else if (target_start < j) { - ERROR("target_start %d, j %d, left_store_size2 %lu", target_start, j, result.left_store2.size()); +} + +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { + result.reset_buffer(query_start + target_start, query.size() + target.size()); + // reverse extend + dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, aln_path, segment_size, result, 0, error_rate); + // forward extend + dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, aln_path, segment_size, result, 1, error_rate); + if (result.left_size + result.right_size < min_aln_size) { + return 0; } + int i, j; + // initialize i and j outside subroutine to avoid warning + count_basepairs(result, result.buffer_start - result.left_size, result.buffer_start, i = 0, j = 0); result.query_start = query_start - i; result.target_start = target_start - j; - for (k = 0, i = 0, j = 0; k < static_cast(result.right_store1.size()); ++k) { - int ch(result.right_store1[k]); - if (ch < 0 || 4 < ch) { - ERROR("Right1: Out of range 0-4: %d (%d, %s)", ch, k, result.right_store1.c_str()); - } else if (ch != 4) { // not '-' - ++i; - } - result.out_store1 += encode2char[ch]; - ch = result.right_store2[k]; - if (ch < 0 || 4 < ch) { - ERROR("Right2: Out of range 0-4: %d (%d, %s)", ch, k, result.right_store2.c_str()); - } else if (ch != 4) { // not '-' - ++j; - } - result.out_store2 += encode2char[ch]; - } - if (result.out_store1.size() < min_aln_size) { - return 0; - } + count_basepairs(result, result.buffer_start, result.buffer_start + result.right_size, i = 0, j = 0); result.query_end = query_start + i; result.target_end = target_start + j; - for (size_t m(0); m < result.out_store1.size(); ++m) { - if (result.out_store1[m] == result.out_store2[m]) { - ++result.mat; - result.out_match_pattern += '|'; - } else if (result.out_store1[m] == '-') { - ++result.ins; - result.out_match_pattern += '*'; - } else if (result.out_store2[m] == '-') { - ++result.del; - result.out_match_pattern += '*'; - } else { - ++result.mis; - result.out_match_pattern += '*'; - } - } return 1; } -bool GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { - if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.swp, error_rate, min_aln_size)) { +static void decode_sequence(std::string& out_seq, const std::vector& in_seq, const size_t offset, const size_t size) { + out_seq.resize(size); + for (size_t i(0); i != size; ++i) { + out_seq[i] = "ACGT-"[static_cast(in_seq[offset + i])]; + } +} + +int GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { + if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.segment_size, error_rate, min_aln_size)) { return 0; } const int consecutive_match_region_size(4); @@ -261,14 +241,14 @@ bool GetAlignment(const std::string& query, const int query_start, const std::st int qrb(0); // q starting pads int trb(0); // t starting pads int eit(0); // matching run length - size_t k; - for (k = 0; k < drd.result.out_store1.size(); ++k) { - const char qc(drd.result.out_store1[k]); - const char tc(drd.result.out_store2[k]); - if (qc != '-') { + int k; + for (k = drd.result.buffer_start - drd.result.left_size; k != drd.result.buffer_start + drd.result.right_size; ++k) { + const char qc(drd.result.q_buffer[k]); + const char tc(drd.result.t_buffer[k]); + if (qc != GAP_ALN) { ++qrb; } - if (tc != '-') { + if (tc != GAP_ALN) { ++trb; } if (qc != tc) { @@ -283,29 +263,28 @@ bool GetAlignment(const std::string& query, const int query_start, const std::st } qrb -= consecutive_match_region_size; trb -= consecutive_match_region_size; - const size_t start_aln_id(k - consecutive_match_region_size); + const int start_aln_id(k - consecutive_match_region_size); // trim trailing end of alignment int qre(0); // q ending pads int tre(0); // t ending pads - for (k = drd.result.out_store1.size() - 1, eit = 0; start_aln_id < k; --k) { - const char qc(drd.result.out_store1[k]); - const char tc(drd.result.out_store2[k]); - if (qc != '-') { + eit = 0; // still matching run length + for (k = drd.result.buffer_start + drd.result.right_size - 1;; --k) { + const char qc(drd.result.q_buffer[k]); + const char tc(drd.result.t_buffer[k]); + if (qc != GAP_ALN) { ++qre; } - if (tc != '-') { + if (tc != GAP_ALN) { ++tre; } if (qc != tc) { eit = 0; } else if (++eit == consecutive_match_region_size) { - --k; break; } } qre -= consecutive_match_region_size; tre -= consecutive_match_region_size; - const size_t aln_size(k + consecutive_match_region_size + 1 - start_aln_id); m5.m5qsize() = query.size(); m5.m5qoff() = drd.result.query_start + qrb; m5.m5qend() = drd.result.query_end - qre; @@ -314,8 +293,8 @@ bool GetAlignment(const std::string& query, const int query_start, const std::st m5.m5soff() = drd.result.target_start + trb; m5.m5send() = drd.result.target_end - tre; m5.m5sdir() = FWD; - m5.m5qaln() = drd.result.out_store1.substr(start_aln_id, aln_size); - m5.m5saln() = drd.result.out_store2.substr(start_aln_id, aln_size); - m5.m5pat() = drd.result.out_match_pattern.substr(start_aln_id, aln_size); + const size_t aln_size(k + consecutive_match_region_size - start_aln_id); + decode_sequence(m5.m5qaln(), drd.result.q_buffer, start_aln_id, aln_size); + decode_sequence(m5.m5saln(), drd.result.t_buffer, start_aln_id, aln_size); return 1; } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 960cc87..e863135 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -9,9 +9,8 @@ #include "../common/defs.h" struct SW_Parameters { - int segment_size; - int row_size; - int column_size; + int segment_size; // probably best if it's a multiple of 10 + int row_size, column_size; SW_Parameters(const int i, const int j, const int k) : segment_size(i), row_size(j), column_size(k) { } }; @@ -22,74 +21,84 @@ inline SW_Parameters get_sw_parameters_small() { class Alignment { public: - int dist; - int aln_q_s, aln_q_e; - int aln_t_s, aln_t_e; - std::string q_aln_str; - std::string t_aln_str; + // size tracks actual buffer use + int dist, aln_q_e, aln_t_e, size; + // these are buffers we only expand + std::vector q_aln_str, t_aln_str; public: explicit Alignment() { } ~Alignment() { } - void clear() { - q_aln_str.clear(); - t_aln_str.clear(); + void reset(const size_t new_max_size) { + size = 0; + if (q_aln_str.size() < new_max_size) { + q_aln_str.resize(new_max_size); + t_aln_str.resize(new_max_size); + } } }; class OutputStore { public: + // these track actual buffer use + int buffer_start, left_size, right_size; int query_start, query_end; int target_start, target_end; - int mat, mis, ins, del; - std::string left_store1, left_store2; - std::string right_store1, right_store2; - std::string out_store1, out_store2; - std::string out_match_pattern; + // these are buffers that we only expand + std::vector q_buffer, t_buffer; + // for the record, inserts are q_buffer == 4, deletes are t_buffer == 4; + // matches/mismatches are pretty obvious public: explicit OutputStore() { } ~OutputStore() { } - void clear() { - left_store1.clear(); - left_store2.clear(); - right_store1.clear(); - right_store2.clear(); - out_store1.clear(); - out_store2.clear(); - out_match_pattern.clear(); + void reset_buffer(const int i, const size_t new_max_size) { + buffer_start = i; + left_size = right_size = 0; + if (q_buffer.size() < new_max_size) { + q_buffer.resize(new_max_size); + t_buffer.resize(new_max_size); + } } }; -struct DPathData { - int x1, y1, x2, y2, pre_k; - explicit DPathData() { } - explicit DPathData(const int i, const int j, const int k, const int l, const int m) : x1(i), y1(j), x2(k), y2(l), pre_k(m) { } -}; - -struct DPathData2 : public DPathData { - int d, k; +struct DPathData2 { + int d, k, x1, y1, x2, y2, pre_k; explicit DPathData2(const int i, const int j) : d(i), k(j) { } - explicit DPathData2(const int i, const int j, const int k, const int l, const int m, const int n, const int p) : DPathData(k, l, m, n, p), d(i), k(j) { } + explicit DPathData2(const int i, const int j, const int ki, const int l, const int m, const int n, const int p) : d(i), k(j), x1(ki), y1(l), x2(m), y2(n), pre_k(p) { } + void set(const int i, const int j, const int ki, const int l, const int m, const int n, const int p) { + d = i; + k = j; + x1 = ki; + y1 = l; + x2 = m; + y2 = n; + pre_k = p; + } }; struct PathPoint { int x, y; + explicit PathPoint() { } explicit PathPoint(const int i, const int j) : x(i), y(j) { } + void set(const int i, const int j) { + x = i; + y = j; + } }; class DiffRunningData { public: - const SW_Parameters swp; + const int segment_size; Alignment align; OutputStore result; - std::string query, target; + std::vector query, target; std::vector DynQ, DynT; std::vector d_path; std::vector aln_path; public: - explicit DiffRunningData(const SW_Parameters& swp_in) : swp(swp_in) { } + explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), DynQ(swp.row_size), DynT(swp.column_size) { } ~DiffRunningData() { } }; -bool GetAlignment(const std::string& query, int query_start, const std::string& target, int target_start, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); +int GetAlignment(const std::string& query, int query_start, const std::string& target, int target_start, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); #endif // DW_H diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index fb17ae4..775e943 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,8 +20,8 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -TGT_LDFLAGS := -L${TARGET_DIR} +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg +TGT_LDFLAGS := -L${TARGET_DIR} -pg TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 2784610..dbadbf1 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -331,7 +331,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa const idx_t sext(ovlp.sext); const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.15, min_align_size)); if (r) { - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } @@ -386,7 +386,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size); if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), ovlp.qsize, m5.m5soff(), m5.m5send(), ovlp.ssize, min_mapping_ratio)) { - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } @@ -460,7 +460,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } @@ -515,7 +515,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, true); + normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); } diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 16c5706..af0d4bf 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -1,7 +1,7 @@ #include "reads_correction_aux.h" #include // string -void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, const bool push) +void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, const int push) { qnorm.clear(); tnorm.clear(); diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index e7de2f0..77172f2 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -146,17 +146,17 @@ struct CmpExtensionCandidateCompressedBySidAndScore { class ConsensusPerThreadData { public: - // num_candidates, candidates initialized by allocate_ecs() - // next_candidate initialized by ConsensusThreadData::restart() - idx_t num_candidates, next_candidate; // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed // for candidate runs (to reduce memory usage) void* candidates; - DiffRunningData drd; // XXX - reduce memory footprint - std::vector cns_table; - std::vector id_list; + // num_candidates, candidates initialized by allocate_ecs() + // next_candidate initialized by ConsensusThreadData::restart() + idx_t num_candidates, next_candidate; + DiffRunningData drd; M5Record m5; CnsAlns cns_alns; + std::vector cns_table; + std::vector id_list; std::vector cns_results; std::string query, target, qaln, saln; public: @@ -173,14 +173,14 @@ class ConsensusThreadData { ReadsCorrectionOptions& rco; PackedDB& reads; std::ostream& out; + ConsensusPerThreadData* data; pthread_mutex_t out_lock; idx_t ec_offset; // this doesn't work as a vector - all the pointers end up pointing // to the same values, and eventually it seg faults (possibly a // compiler optimization bug) - ConsensusPerThreadData* data; public: - ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), ec_offset(0), data(new ConsensusPerThreadData[prco.num_threads]), last_thread_id_(-1), num_threads_written_(0) { + ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), data(new ConsensusPerThreadData[prco.num_threads]), ec_offset(0), last_thread_id_(-1), num_threads_written_(0) { done_file_ = input_file_name; done_file_ += ".done"; ckpt_file_ = input_file_name; @@ -274,12 +274,12 @@ class ConsensusThreadData { } } private: - int last_thread_id_, num_threads_written_; pthread_mutex_t id_lock_; + int last_thread_id_, num_threads_written_; std::string done_file_, ckpt_file_, ckpt_file_tmp_; }; -void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, bool push); +void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, int push); void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, idx_t nec); void allocate_ecs(ConsensusThreadData &data, ExtensionCandidateCompressed* ec_list, idx_t nec); From 56383b1295073ca1416800cf0764f4fbc8ba7b3f Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 1 May 2019 13:49:01 -0500 Subject: [PATCH 34/56] moved to copy() over memcpy, unified left/right buffers but now getting free errors in the boost routines, for some reason --- src/common/alignment.h | 174 +---------------------- src/main.mk | 4 +- src/mecat2cns/dw.cpp | 196 +++++++++++++++----------- src/mecat2cns/dw.h | 15 +- src/mecat2cns/main.cpp | 3 +- src/mecat2cns/mecat2cns.mk | 4 +- src/mecat2cns/mecat_correction.cpp | 62 ++++---- src/mecat2cns/packed_db.cpp | 9 +- src/mecat2cns/packed_db.h | 4 +- src/mecat2cns/reads_correction_aux.h | 39 +++-- src/mecat2cns/reads_correction_m4.cpp | 3 +- 11 files changed, 185 insertions(+), 328 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index e800457..e9a2fa4 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -13,6 +13,8 @@ struct ExtensionCandidate int score; }; +typedef ExtensionCandidate Overlap; + // candidates only use a few of these values, so make a smaller structure for them; // sid and qid are used a lot, and we sort on score, so stash qdir inside msb of qext @@ -224,178 +226,8 @@ m4_to_candidate(const M4Record& m4, ExtensionCandidate& ec) ec.score = m4vscore(m4); } -class M5Record { - public: - idx_t qid; // 1) qname - idx_t qsize; // 2) qlength - idx_t qstart; // 3) qstart - idx_t qend; // 4) qend - int qdir; // 5) qstrand - idx_t sid; // 6) sname - idx_t ssize; // 7) slength - idx_t sstart; // 8) sstart - idx_t send; // 9) send - int sdir; // 10) sstrand - int score; // 11) score - int mat; // 12) match - int mis; // 13) mismatch - int ins; // 14) insertion - int dels; // 15) deletion - int mapq; // 16) mapQ - std::string pm_q; // 17) aligned query - //std::string pm_p; // 18) aligned pattern - std::string pm_s; // 19) aligned subject - double ident; // 20) identity percentage - idx_t qext; - idx_t sext; - public: - explicit M5Record() { } - ~M5Record() { } - idx_t& m5qid() { - return qid; - } - const idx_t m5qid() const { - return qid; - } - idx_t& m5qsize() { - return qsize; - } - const idx_t m5qsize() const { - return qsize; - } - idx_t& m5qoff() { - return qstart; - } - const idx_t m5qoff() const { - return qstart; - } - idx_t& m5qend() { - return qend; - } - const idx_t m5qend() const { - return qend; - } - int& m5qdir() { - return qdir; - } - const int m5qdir() const { - return qdir; - } - idx_t& m5sid() { - return sid; - } - const idx_t m5sid() const { - return sid; - } - idx_t& m5ssize() { - return ssize; - } - const idx_t& m5ssize() const { - return ssize; - } - idx_t& m5soff() { - return sstart; - } - const idx_t& m5soff() const { - return sstart; - } - idx_t& m5send() { - return send; - } - const idx_t m5send() const { - return send; - } - int& m5sdir() { - return sdir; - } - const int m5sdir() const { - return sdir; - } - int& m5score() { - return score; - } - const int m5score() const { - return score; - } - int& m5mat() { - return mat; - } - const int m5mat() const { - return mat; - } - int& m5mis() { - return mis; - } - const int m5mis() const { - return mis; - } - int& m5ins() { - return ins; - } - const int m5ins() const { - return ins; - } - int& m5dels() { - return dels; - } - const int m5dels() const { - return dels; - } - int& m5mapq() { - return mapq; - } - const int m5mapq() const { - return mapq; - } - std::string& m5qaln() { - return pm_q; - } - const std::string& m5qaln() const { - return pm_q; - } - std::string& m5saln() { - return pm_s; - } - const std::string& m5saln() const { - return pm_s; - } - double& m5ident() { - return ident; - } - const double m5ident() const { - return ident; - } - idx_t& m5qext() { - return qext; - } - const idx_t m5qext() const { - return qext; - } - idx_t& m5sext() { - return sext; - } - const idx_t m5sext() const { - return sext; - } -}; - -inline int M5RecordOvlpSize(const M5Record& m) { - const int oq(m.m5qend() - m.m5qoff()); - const int os(m.m5send() - m.m5soff()); - return std::max(oq, os); -} - -typedef ExtensionCandidate Overlap; - -struct CompareOverlapBySid { - bool operator()(const Overlap& a, const Overlap& b) { - return a.sid < b.sid; - } -}; - struct CnsResult { - idx_t id; - idx_t range[2]; + idx_t id, range[2]; std::string seq; }; diff --git a/src/main.mk b/src/main.mk index 55dfaa7..d3892b2 100644 --- a/src/main.mk +++ b/src/main.mk @@ -25,5 +25,5 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -TGT_LDFLAGS := -pg +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -g +TGT_LDFLAGS := -pg -g diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 6da7296..0ac1687 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -12,13 +12,12 @@ static int CompareDPathData2(const void* const a, const void* const b) { static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& d_path, const size_t d_path_idx, std::vector& aln_path, const int extend_forward, const size_t aln_path_max) { const DPathData2* d_path_aux(&d_path[d_path_idx - 1]); - align.dist = d_path_aux->d; align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; - // get align path if (aln_path.size() < aln_path_max + 2) { aln_path.resize(aln_path_max + 2); } + // get align path size_t aln_idx(-1); // no need to search for the first one aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); @@ -31,12 +30,11 @@ static void fill_align(const std::string& query, const int q_offset, const std:: aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); seek.k = d_path_aux->pre_k; } + align.reset(align.aln_q_e + align.aln_t_e); int current_x(aln_path[aln_idx].x); int current_y(aln_path[aln_idx].y); - align.reset(aln_path[0].x + aln_path[0].y); - // +1 so [-dx, 0) becomes (-dx, 0], to match [0, dx) - const char* const query_p(query.data() + q_offset + (extend_forward ? 0 : 1)); - const char* const target_p(target.data() + t_offset + (extend_forward ? 0 : 1)); + const char* const query_p(query.data() + q_offset); + const char* const target_p(target.data() + t_offset); for (--aln_idx; aln_idx != size_t(-1); --aln_idx) { const int new_x(aln_path[aln_idx].x); const int new_y(aln_path[aln_idx].y); @@ -45,27 +43,31 @@ static void fill_align(const std::string& query, const int q_offset, const std:: if (dx && dy) { // apparently, dx always equals dy in this case if (extend_forward) { - std::copy(query_p + current_x, query_p + new_x, align.q_aln_str.begin() + align.size); - std::copy(target_p + current_y, target_p + new_y, align.t_aln_str.begin() + align.size); + std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); + std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); + align.size += dx; } else { - std::reverse_copy(query_p - new_x, query_p - current_x, align.q_aln_str.begin() + align.size); - std::reverse_copy(target_p - new_y, target_p - current_y, align.t_aln_str.begin() + align.size); + align.size += dx; + const int offset(align.q_aln_str.size() - align.size); + std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); + std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); } - align.size += dx; } else if (dx) { - std::fill(align.t_aln_str.begin() + align.size, align.t_aln_str.begin() + align.size + dx, GAP_ALN); if (extend_forward) { - std::copy(query_p + current_x, query_p + new_x, align.q_aln_str.begin() + align.size); + std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); + std::fill(&align.t_aln_str[align.size], &align.t_aln_str[align.size + dx], GAP_ALN); } else { - std::reverse_copy(query_p - new_x, query_p - current_x, align.q_aln_str.begin() + align.size); + std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[align.q_aln_str.size() - align.size - dx]); + std::fill(&align.t_aln_str[0] + align.t_aln_str.size() - align.size - dx, &align.t_aln_str[0] + align.t_aln_str.size() - align.size, GAP_ALN); } align.size += dx; } else if (dy) { - std::fill(align.q_aln_str.begin() + align.size, align.q_aln_str.begin() + align.size + dy, GAP_ALN); if (extend_forward) { - std::copy(target_p + current_y, target_p + new_y, align.t_aln_str.begin() + align.size); + std::fill(&align.q_aln_str[align.size], &align.q_aln_str[align.size + dy], GAP_ALN); + std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); } else { - std::reverse_copy(target_p - new_y, target_p - current_y, align.t_aln_str.begin() + align.size); + std::fill(&align.q_aln_str[0] + align.q_aln_str.size() - align.size - dy, &align.q_aln_str[0] + align.q_aln_str.size() - align.size, GAP_ALN); + std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[align.t_aln_str.size() - align.size - dy]); } align.size += dy; } @@ -77,10 +79,14 @@ static void fill_align(const std::string& query, const int q_offset, const std:: static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(segment_size * 4 * error_rate); const int band_tolerance(segment_size / 10 * 3 + 1); + // max band size is max offset between query and target const int max_band_size(band_tolerance * 2 - 1); size_t d_path_idx(0); int best_m(-1), min_k(0), max_k(0); + // XXX - we could create an index into d_path by d, which would speed up the bsearch() above + // (in fact, we wouldn't even have to store d in it anymore) for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { + // k is the offset between query and target for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { @@ -91,7 +97,9 @@ static int Align(const int segment_size, const std::string& query, const int q_o x = V[k_offset + k - 1] + 1; } int y(x - k); + // start of exact match const int x1(x), y1(y); + // find the other end of exact match if (extend_forward) { for (; x < segment_size && y < segment_size && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } } else { @@ -103,6 +111,7 @@ static int Align(const int segment_size, const std::string& query, const int q_o d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); } ++d_path_idx; + // see if we got as much as we can if (x == segment_size || y == segment_size) { fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, aln_path, extend_forward, segment_size * 2); return 1; @@ -134,73 +143,92 @@ static int Align(const int segment_size, const std::string& query, const int q_o } static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { - int extend1(0), extend2(0); - for (int not_at_end(1); not_at_end;) { + int q_extend(0), t_extend(0), not_at_end(1); + const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset); + const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset); + do { // size left to extend - int extend_size; - if (extend_forward) { - extend_size = std::min(query.size() - q_offset - extend1, target.size() - t_offset - extend2); - } else { - extend_size = std::min(q_offset - extend1, t_offset - extend2); - } - if (extend_size < segment_size + 101) { + const int extend_size(std::min(q_extend_max - q_extend, t_extend_max - t_extend)); + if (extend_size < segment_size + 101) { // close enough to the end not_at_end = 0; } - U.assign(U.size(), 0); + U.assign(U.size(), 0); // XXX - are these two initializations necessary? V.assign(V.size(), 0); - if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? extend1 : -extend1), target, t_offset + (extend_forward ? extend2 : -extend2), align, U, V, d_path, aln_path, extend_forward, error_rate)) { - break; + if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, aln_path, extend_forward, error_rate)) { + return; } - int k, i(0), j(0), num_matches(0); - for (k = align.size - 1; -1 < k && num_matches < 4; --k) { - if (align.q_aln_str[k] != GAP_ALN) { - ++i; - } - if (align.t_aln_str[k] != GAP_ALN) { - ++j; - } - if (align.q_aln_str[k] == align.t_aln_str[k]) { - ++num_matches; + int k; + if (not_at_end) { + // go backwards until we find a good match (4 consecutive + // matching basepairs), counting non-gap basepairs + int q_bps(0), t_bps(0), num_matches(0); + if (extend_forward) { + for (k = align.size - 1; -1 < k; --k) { + if (align.q_aln_str[k] != GAP_ALN) { + ++q_bps; + } + if (align.t_aln_str[k] != GAP_ALN) { + ++t_bps; + } + if (align.q_aln_str[k] != align.t_aln_str[k]) { + num_matches = 0; + } else if (++num_matches == 4) { + break; + } + } } else { - num_matches = 0; + for (k = align.size; 0 < k; --k) { + if (align.q_aln_str[align.q_aln_str.size() - k] != GAP_ALN) { + ++q_bps; + } + if (align.t_aln_str[align.t_aln_str.size() - k] != GAP_ALN) { + ++t_bps; + } + if (align.q_aln_str[align.q_aln_str.size() - k] != align.t_aln_str[align.t_aln_str.size() - k]) { + num_matches = 0; + } else if (++num_matches == 4) { + break; + } + } } - } - if (not_at_end) { - if (align.aln_q_e == i) { - break; + if (align.aln_q_e == q_bps) { // no good match + return; } - extend1 += align.aln_q_e - i; - extend2 += align.aln_t_e - j; - ++k; - } else if (align.aln_q_e == 0) { - break; + // only extend to the good match + q_extend += align.aln_q_e - q_bps; + t_extend += align.aln_t_e - t_bps; + } else if (align.aln_q_e == 0) { // no good match + return; } else { k = align.size; } if (extend_forward) { - std::copy(align.q_aln_str.begin(), align.q_aln_str.begin() + k, &result.q_buffer[result.buffer_start + result.right_size]); - std::copy(align.t_aln_str.begin(), align.t_aln_str.begin() + k, &result.t_buffer[result.buffer_start + result.right_size]); + const int i(result.buffer_start + result.right_size); + std::copy(&align.q_aln_str[0], &align.q_aln_str[k], &result.q_buffer[i]); + std::copy(&align.t_aln_str[0], &align.t_aln_str[k], &result.t_buffer[i]); result.right_size += k; } else { result.left_size += k; - std::copy(align.q_aln_str.begin(), align.q_aln_str.begin() + k, &result.q_buffer[result.buffer_start - result.left_size]); - std::copy(align.t_aln_str.begin(), align.t_aln_str.begin() + k, &result.t_buffer[result.buffer_start - result.left_size]); + const int i(result.buffer_start - result.left_size); + char* const q_aln_end(&align.q_aln_str[0] + align.q_aln_str.size()); + char* const t_aln_end(&align.t_aln_str[0] + align.t_aln_str.size()); + std::copy(q_aln_end - k, q_aln_end, &result.q_buffer[i]); + std::copy(t_aln_end - k, t_aln_end, &result.t_buffer[i]); } - - } + } while (not_at_end); } -static void count_basepairs(const OutputStore& result, int k, const int end_k, int i, int j) { - for (; k != end_k; ++k) { - int ch(result.q_buffer[k]); +static void count_basepairs(const OutputStore& result, int i, const int end_i, int& q_bps, int& t_bps) { + for (; i != end_i; ++i) { + int ch(result.q_buffer[i]); assert(-1 < ch && ch < 5); if (ch != GAP_ALN) { - ++i; + ++q_bps; } - ch = result.t_buffer[k]; + ch = result.t_buffer[i]; assert(-1 < ch && ch < 5); if (ch != GAP_ALN) { - ++j; + ++t_bps; } } } @@ -214,14 +242,14 @@ static int dw(const std::string& query, const int query_start, const std::string if (result.left_size + result.right_size < min_aln_size) { return 0; } - int i, j; + int q_bps, t_bps; // initialize i and j outside subroutine to avoid warning - count_basepairs(result, result.buffer_start - result.left_size, result.buffer_start, i = 0, j = 0); - result.query_start = query_start - i; - result.target_start = target_start - j; - count_basepairs(result, result.buffer_start, result.buffer_start + result.right_size, i = 0, j = 0); - result.query_end = query_start + i; - result.target_end = target_start + j; + count_basepairs(result, result.buffer_start - result.left_size, result.buffer_start, q_bps = 0, t_bps = 0); + result.query_start = query_start - q_bps; + result.target_start = target_start - t_bps; + count_basepairs(result, result.buffer_start, result.buffer_start + result.right_size, q_bps = 0, t_bps = 0); + result.query_end = query_start + q_bps; + result.target_end = target_start + t_bps; return 1; } @@ -233,18 +261,20 @@ static void decode_sequence(std::string& out_seq, const std::vector& in_se } int GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { - if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, drd.result, drd.segment_size, error_rate, min_aln_size)) { + OutputStore& result(drd.result); + if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, result, drd.segment_size, error_rate, min_aln_size)) { return 0; } + // create return m5 record const int consecutive_match_region_size(4); // trim starting end of alignment int qrb(0); // q starting pads int trb(0); // t starting pads int eit(0); // matching run length int k; - for (k = drd.result.buffer_start - drd.result.left_size; k != drd.result.buffer_start + drd.result.right_size; ++k) { - const char qc(drd.result.q_buffer[k]); - const char tc(drd.result.t_buffer[k]); + for (k = result.buffer_start - result.left_size; k != result.buffer_start + result.right_size; ++k) { + const char qc(result.q_buffer[k]); + const char tc(result.t_buffer[k]); if (qc != GAP_ALN) { ++qrb; } @@ -268,9 +298,9 @@ int GetAlignment(const std::string& query, const int query_start, const std::str int qre(0); // q ending pads int tre(0); // t ending pads eit = 0; // still matching run length - for (k = drd.result.buffer_start + drd.result.right_size - 1;; --k) { - const char qc(drd.result.q_buffer[k]); - const char tc(drd.result.t_buffer[k]); + for (k = result.buffer_start + result.right_size - 1;; --k) { + const char qc(result.q_buffer[k]); + const char tc(result.t_buffer[k]); if (qc != GAP_ALN) { ++qre; } @@ -285,16 +315,12 @@ int GetAlignment(const std::string& query, const int query_start, const std::str } qre -= consecutive_match_region_size; tre -= consecutive_match_region_size; - m5.m5qsize() = query.size(); - m5.m5qoff() = drd.result.query_start + qrb; - m5.m5qend() = drd.result.query_end - qre; - m5.m5qdir() = FWD; - m5.m5ssize() = target.size(); - m5.m5soff() = drd.result.target_start + trb; - m5.m5send() = drd.result.target_end - tre; - m5.m5sdir() = FWD; + m5.qoff = result.query_start + qrb; + m5.qend = result.query_end - qre; + m5.soff = result.target_start + trb; + m5.send = result.target_end - tre; const size_t aln_size(k + consecutive_match_region_size - start_aln_id); - decode_sequence(m5.m5qaln(), drd.result.q_buffer, start_aln_id, aln_size); - decode_sequence(m5.m5saln(), drd.result.t_buffer, start_aln_id, aln_size); + decode_sequence(m5.qaln, result.q_buffer, start_aln_id, aln_size); + decode_sequence(m5.saln, result.t_buffer, start_aln_id, aln_size); return 1; } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index e863135..e220b56 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -1,12 +1,9 @@ #ifndef DW_H #define DW_H -#include #include // string #include // vector<> - -#include "../common/alignment.h" -#include "../common/defs.h" +#include "../common/defs.h" // idx_t struct SW_Parameters { int segment_size; // probably best if it's a multiple of 10 @@ -22,7 +19,7 @@ inline SW_Parameters get_sw_parameters_small() { class Alignment { public: // size tracks actual buffer use - int dist, aln_q_e, aln_t_e, size; + int aln_q_e, aln_t_e, size; // these are buffers we only expand std::vector q_aln_str, t_aln_str; public: @@ -99,6 +96,14 @@ class DiffRunningData { ~DiffRunningData() { } }; +class M5Record { + public: + idx_t qoff, qend, soff, send; + std::string qaln, saln; + M5Record() { } + ~M5Record() { } +}; + int GetAlignment(const std::string& query, int query_start, const std::string& target, int target_start, DiffRunningData& drd, M5Record& m5, double error_rate, int min_aln_size); #endif // DW_H diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 90d883c..8d1c7a7 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -33,7 +33,8 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt unlink(script_file.c_str()); std::ofstream out; open_fstream(out, script_file.c_str(), std::ios::out); - out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; + //out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; + out << "#!/bin/bash\n" << prog << make_options(new_options) << "\n"; if (!out) { std::cerr << "Error writing to " << script_file << "\n"; exit(1); diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index 775e943..d41de42 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,8 +20,8 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -TGT_LDFLAGS := -L${TARGET_DIR} -pg +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -g +TGT_LDFLAGS := -L${TARGET_DIR} -pg -g TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index dbadbf1..7732983 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -11,8 +11,8 @@ // returns type of coverage present -static inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item, const int min_cov) { - const int cov(ceil((cns_item.mat_cnt + cns_item.ins_cnt) * 0.8)); +static inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item) { + const int cov((cns_item.mat_cnt + cns_item.ins_cnt) * 0.8); uint1 ident; if (cns_item.mat_cnt >= cov) { // coverage is 80% or more matches ident = FMAT; @@ -21,19 +21,17 @@ static inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item, co } else { // neither of the above ident = UNDS; } - if (2 * cns_item.del_cnt >= cov) { // deletes are 40% or more than the coverage + if (cns_item.del_cnt * 2 >= cov) { // deletes are 40% or more than the coverage ident |= FDEL; } return ident; } -struct CompareOverlapByOverlapSize -{ - bool operator()(const Overlap& a ,const Overlap& b) - { - const idx_t ovlp_a = std::max(a.qend - a.qoff, a.send - a.soff); - const idx_t ovlp_b = std::max(b.qend - b.qoff, b.send - b.soff); - return ovlp_a > ovlp_b; +struct CompareOverlapByOverlapSize { + bool operator()(const Overlap& a, const Overlap& b) { + const idx_t ovlp_a(std::max(a.qend - a.qoff, a.send - a.soff)); + const idx_t ovlp_b(std::max(b.qend - b.qoff, b.send - b.soff)); + return ovlp_b < ovlp_a; } }; @@ -74,13 +72,13 @@ static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, con ag.consensus(min_cov * 0.4, cns); } -static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target, const int min_cov) { +static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target) { if (static_cast(cns_id_vec.size()) < cns_list_size) { cns_id_vec.resize(cns_list_size); } // get types of coverage for (int i(0); i < cns_list_size; ++i) { - cns_id_vec[i] = identify_one_consensus_item(cns_list[start_soff + i], min_cov); + cns_id_vec[i] = identify_one_consensus_item(cns_list[start_soff + i]); } std::string cns; target.clear(); @@ -95,7 +93,7 @@ static void meap_consensus_one_segment(const std::vector& cns_list for (++i; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } int need_refinement(0); // check to see if anything in-between has questionable coverage - for (int k(start); k < i; ++k) { + for (int k(start); k != i; ++k) { if (cns_id_vec[k] & unds_or_fdel) { need_refinement = 1; break; @@ -233,7 +231,7 @@ static void consensus_worker(const std::vector& cns_table, std::ve // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start >= min_size_95) { - meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq); if (cns_seq.size() >= static_cast(min_size)) { output_cns_result(cns_results, cns_result, start, i, cns_seq); } @@ -281,7 +279,7 @@ static void consensus_worker_one_read(const std::vector& cns_table // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start > min_size_95) { - meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq, min_cov); + meap_consensus_one_segment(cns_table, i - start, id_list, start, cns_vec, aux_qstr, aux_tstr, cns_seq); if (cns_seq.size() >= static_cast(min_size)) { // add corrected sequence cns_result.seq += cns_seq; @@ -331,9 +329,9 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa const idx_t sext(ovlp.sext); const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.15, min_align_size)); if (r) { - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); - meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); - cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); + normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); + meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); + cns_vec.add_aln(m5.soff, m5.send, nqstr, ntstr); } } std::vector mranges, eranges; @@ -384,11 +382,11 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size); - if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), ovlp.qsize, m5.m5soff(), m5.m5send(), ovlp.ssize, min_mapping_ratio)) + if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, ovlp.qsize, m5.soff, m5.send, ovlp.ssize, min_mapping_ratio)) { - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); - meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); - cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); + normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); + meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); + cns_vec.add_aln(m5.soff, m5.send, nqstr, ntstr); } } @@ -456,13 +454,13 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); - if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), qsize, m5.m5soff(), m5.m5send(), ssize, min_mapping_ratio)) { - if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { + if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { + if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); - meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); - cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); + normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); + meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); + cns_vec.add_aln(m5.soff, m5.send, nqstr, ntstr); } } } @@ -511,13 +509,13 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size)); - if (r && check_ovlp_mapping_range(m5.m5qoff(), m5.m5qend(), qsize, m5.m5soff(), m5.m5send(), ssize, min_mapping_ratio)) { - if (check_cov_stats(id_list, m5.m5soff(), m5.m5send())) { + if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { + if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; used_ids.insert(ec.qid); - normalize_gaps(m5.m5qaln(), m5.m5saln(), nqstr, ntstr, 1); - meap_add_one_aln(nqstr, ntstr, m5.m5soff(), cns_table); - cns_vec.add_aln(m5.m5soff(), m5.m5send(), nqstr, ntstr); + normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); + meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); + cns_vec.add_aln(m5.soff, m5.send, nqstr, ntstr); } } } diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 984b0cd..9aa1767 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -22,10 +22,9 @@ void PackedDB::add_one_seq(const Sequence& seq) { if (max_db_size < needed_size) { idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } - uint1* new_pac(NULL); - safe_calloc(new_pac, uint1, (new_size + 3) / 4); + uint1* const new_pac(new uint1[(new_size + 3) / 4]); memcpy(new_pac, pac, (db_size + 3) / 4); - safe_free(pac); + delete[] pac; pac = new_pac; max_db_size = new_size; } @@ -182,7 +181,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { const idx_t file_size(pstream.tellg()); max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { - safe_calloc(pac, uint1, max_db_size); + pac = new uint1[max_db_size]; } size_t read_count; if (!pstream.read((char*)&read_count, sizeof(size_t))) { @@ -262,7 +261,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co LOG(stderr, "using %ld bytes for %lu reads, %ld aligns (out of %ld)", total_size, read_ids.size(), i, nec); if (max_db_size == 0) { max_db_size = total_size; - safe_calloc(pac, uint1, max_db_size); + pac = new uint1[max_db_size]; } // now read in the reads std::set::const_iterator a(read_ids.begin()); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 20097d2..4b5f237 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -21,7 +21,7 @@ class PackedDB { explicit PackedDB() : pac(0), db_size(0), max_db_size(0) { } ~PackedDB() { if (pac) { - safe_free(pac); + delete[] pac; } } // returns number of reads @@ -68,7 +68,7 @@ class PackedDB { void add_one_seq(const Sequence& seq); void destroy() { if (pac) { - safe_free(pac); + delete[] pac; pac = NULL; } seq_idx.clear(); diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 77172f2..8f9019a 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -1,11 +1,10 @@ #ifndef _READS_CORRECTION_AUX_H #define _READS_CORRECTION_AUX_H -#include -#include +#include // vector<> #include -#include "dw.h" +#include "dw.h" // DiffRunningData, M5Record, get_parameters_small() #include "packed_db.h" #include "options.h" @@ -19,44 +18,40 @@ struct CnsTableItem { #define MAX_CNS_OVLPS 100 -class MappingRange { - public: +struct MappingRange { int start, end; explicit MappingRange() : start(0), end(0) { } explicit MappingRange(const int s, const int e) : start(s), end(e) { } - ~MappingRange() { } }; class CnsAln : public MappingRange { public: - explicit CnsAln() : aln_idx(0) { } - explicit CnsAln(const int i, const int j, const int k, const std::string& q, const std::string& s) : MappingRange(i, j), aln_idx(k), qaln(s), saln(q) { } + explicit CnsAln(const int i, const int j, const std::string& q, const std::string& s) : MappingRange(i, j), aln_idx(0), qaln(s), saln(q) { } ~CnsAln() { } - // don't know why this skips the first basepair int retrieve_aln_subseqs(const int sb, const int se, std::string& qstr, std::string& tstr, int& sb_out) { - const int aln_size(static_cast(saln.size()) - 1); - if (se <= start || sb >= end || aln_idx >= aln_size) { + const int aln_size(saln.size() - 1); + if (se <= start || sb >= end || aln_idx == aln_size) { return 0; } sb_out = std::max(start, sb); - qstr.clear(); - tstr.clear(); while (start < sb && aln_idx < aln_size) { - ++aln_idx; - if (saln[aln_idx] != GAP) { + if (saln[++aln_idx] != GAP) { ++start; } } - qstr += qaln[aln_idx]; - tstr += saln[aln_idx]; + // should we test for start < sb, and return 0 if so, + // rather than returning the last basepair of the alignment? + const int aln_start(aln_idx); while (start < se && aln_idx < aln_size) { - ++aln_idx; - if (saln[aln_idx] != GAP) { + if (saln[++aln_idx] != GAP) { ++start; } - qstr += qaln[aln_idx]; - tstr += saln[aln_idx]; } + // this looks like it could return the same basepair twice - + // once at the end of a call, once at the start of the next + const int aln_length(aln_idx - aln_start + 1); + qstr.assign(qaln, aln_start, aln_length); + tstr.assign(saln, aln_start, aln_length); return 1; } private: @@ -82,7 +77,7 @@ class CnsAlns { } void add_aln(const int soff, const int send, const std::string& qstr, const std::string& tstr) { r_assert(qstr.size() == tstr.size()); - cns_alns_.push_back(CnsAln(soff, send, 0, qstr, tstr)); + cns_alns_.push_back(CnsAln(soff, send, qstr, tstr)); } void get_mapping_ranges(std::vector& ranges) const { ranges.clear(); diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index 0617493..b821394 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -1,12 +1,13 @@ #include "reads_correction_m4.h" -#include +#include // sort() #include // string #include #include "mecat_correction.h" #include "overlaps_partition.h" #include "overlaps_store.h" +#include "reads_correction_aux.h" void* reads_correction_func_m4(void* arg) { ConsensusThreadData& data(*(static_cast(arg))); From 6b1fa4fe06bc3e63d6efafe0cef13edf96293b7c Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 1 May 2019 16:46:07 -0500 Subject: [PATCH 35/56] in the middle of testing, this is a save point --- src/mecat2cns/dw.cpp | 49 +++++++++++++++--------------- src/mecat2cns/dw.h | 10 +++--- src/mecat2cns/mecat_correction.cpp | 2 ++ 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 0ac1687..fb044d2 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -5,30 +5,28 @@ #define GAP_ALN 4 static int CompareDPathData2(const void* const a, const void* const b) { - const DPathData2* const d1((const DPathData2*)a); - const DPathData2* const d2((const DPathData2*)b); - return (d1->d != d2->d) ? (d1->d - d2->d) : (d1->k - d2->k); + return ((const DPathData2* const)a)->k - ((const DPathData2* const)b)->k; } -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& d_path, const size_t d_path_idx, std::vector& aln_path, const int extend_forward, const size_t aln_path_max) { +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const size_t d_path_idx, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { const DPathData2* d_path_aux(&d_path[d_path_idx - 1]); align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; - if (aln_path.size() < aln_path_max + 2) { - aln_path.resize(aln_path_max + 2); - } // get align path + if (static_cast(aln_path.size()) < (d + 1) * 2) { + aln_path.resize((d + 1) * 2); + } size_t aln_idx(-1); // no need to search for the first one aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); - DPathData2 seek(d_path_aux->d - 1, d_path_aux->pre_k); - for (; -1 < seek.d && aln_idx < aln_path_max; --seek.d) { + DPathData2 seek; + for (; d != 0; --d) { + seek.k = d_path_aux->pre_k; // there may be a better approach here than bsearch() - d_path_aux = (const DPathData2*)bsearch(&seek, &d_path[0], d_path_aux - &d_path[0], sizeof(DPathData2), CompareDPathData2); + d_path_aux = (const DPathData2*)bsearch(&seek, &d_path[d_path_index[d - 1]], d_path_index[d] - d_path_index[d - 1], sizeof(DPathData2), CompareDPathData2); aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); - seek.k = d_path_aux->pre_k; } align.reset(align.aln_q_e + align.aln_t_e); int current_x(aln_path[aln_idx].x); @@ -76,16 +74,18 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(segment_size * 4 * error_rate); + if (static_cast(d_path_index.size()) < k_offset) { + d_path_index.resize(k_offset); + } const int band_tolerance(segment_size / 10 * 3 + 1); // max band size is max offset between query and target const int max_band_size(band_tolerance * 2 - 1); size_t d_path_idx(0); int best_m(-1), min_k(0), max_k(0); - // XXX - we could create an index into d_path by d, which would speed up the bsearch() above - // (in fact, we wouldn't even have to store d in it anymore) for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { + d_path_index[d] = d_path_idx; // starting point of each "d" set of entries // k is the offset between query and target for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; @@ -106,14 +106,14 @@ static int Align(const int segment_size, const std::string& query, const int q_o for (; x < segment_size && y < segment_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } if (d_path_idx != d_path.size()) { - d_path[d_path_idx].set(d, k, x1, y1, x, y, pre_k); + d_path[d_path_idx].set(k, x1, y1, x, y, pre_k); } else { - d_path.push_back(DPathData2(d, k, x1, y1, x, y, pre_k)); + d_path.push_back(DPathData2(k, x1, y1, x, y, pre_k)); } ++d_path_idx; // see if we got as much as we can if (x == segment_size || y == segment_size) { - fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, aln_path, extend_forward, segment_size * 2); + fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, d_path_index, d, aln_path, extend_forward); return 1; } V[k_offset + k] = x; @@ -142,7 +142,7 @@ static int Align(const int segment_size, const std::string& query, const int q_o return 0; } -static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { int q_extend(0), t_extend(0), not_at_end(1); const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset); const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset); @@ -152,10 +152,9 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co if (extend_size < segment_size + 101) { // close enough to the end not_at_end = 0; } - U.assign(U.size(), 0); // XXX - are these two initializations necessary? + U.assign(U.size(), 0); // are these two initializations necessary? V.assign(V.size(), 0); - if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, aln_path, extend_forward, error_rate)) { - return; + if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, d_path_index, aln_path, extend_forward, error_rate)) { return; } int k; if (not_at_end) { @@ -233,12 +232,12 @@ static void count_basepairs(const OutputStore& result, int i, const int end_i, i } } -static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { result.reset_buffer(query_start + target_start, query.size() + target.size()); // reverse extend - dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, aln_path, segment_size, result, 0, error_rate); + dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 0, error_rate); // forward extend - dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, aln_path, segment_size, result, 1, error_rate); + dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 1, error_rate); if (result.left_size + result.right_size < min_aln_size) { return 0; } @@ -262,7 +261,7 @@ static void decode_sequence(std::string& out_seq, const std::vector& in_se int GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { OutputStore& result(drd.result); - if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.aln_path, result, drd.segment_size, error_rate, min_aln_size)) { + if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.d_path_index, drd.aln_path, result, drd.segment_size, error_rate, min_aln_size)) { return 0; } // create return m5 record diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index e220b56..35bd5ec 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -58,11 +58,10 @@ class OutputStore { }; struct DPathData2 { - int d, k, x1, y1, x2, y2, pre_k; - explicit DPathData2(const int i, const int j) : d(i), k(j) { } - explicit DPathData2(const int i, const int j, const int ki, const int l, const int m, const int n, const int p) : d(i), k(j), x1(ki), y1(l), x2(m), y2(n), pre_k(p) { } - void set(const int i, const int j, const int ki, const int l, const int m, const int n, const int p) { - d = i; + int k, x1, y1, x2, y2, pre_k; + explicit DPathData2() { } + explicit DPathData2(const int j, const int ki, const int l, const int m, const int n, const int p) : k(j), x1(ki), y1(l), x2(m), y2(n), pre_k(p) { } + void set(const int j, const int ki, const int l, const int m, const int n, const int p) { k = j; x1 = ki; y1 = l; @@ -89,6 +88,7 @@ class DiffRunningData { OutputStore result; std::vector query, target; std::vector DynQ, DynT; + std::vector d_path_index; std::vector d_path; std::vector aln_path; public: diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 7732983..7f82451 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -453,7 +453,9 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); +std::cerr << "(" << ec.sid << " " << ec.sext << " " << ssize << ") (" << ec.qid << " " << ec.qext() << " " << qsize << ") " << ec.qdir() << " " << ec.score << "\n"; const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); +std::cerr << m5.soff << " " << m5.send << " " << m5.qoff << " " << m5.qend << " " << m5.saln.size() << " " << m5.qaln.size() << "\n" << m5.saln << "\n" << m5.qaln << "\n"; if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; From 1e82d1452c9ea42294780f0cbf185e00f61eb9ec Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 2 May 2019 21:49:34 -0500 Subject: [PATCH 36/56] still working on bug, but got rid of bsearch in dw.cpp --- src/mecat2cns/dw.cpp | 165 ++++++++++++++----------- src/mecat2cns/dw.h | 2 + src/mecat2cns/mecat_correction.cpp | 3 +- src/mecat2cns/reads_correction_can.cpp | 2 +- 4 files changed, 96 insertions(+), 76 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index fb044d2..1b83b4d 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -4,11 +4,8 @@ #define GAP_ALN 4 -static int CompareDPathData2(const void* const a, const void* const b) { - return ((const DPathData2* const)a)->k - ((const DPathData2* const)b)->k; -} - static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const size_t d_path_idx, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { + // the final one is always last on the stack const DPathData2* d_path_aux(&d_path[d_path_idx - 1]); align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; @@ -17,16 +14,13 @@ static void fill_align(const std::string& query, const int q_offset, const std:: aln_path.resize((d + 1) * 2); } size_t aln_idx(-1); - // no need to search for the first one - aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); - aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); - DPathData2 seek; - for (; d != 0; --d) { - seek.k = d_path_aux->pre_k; - // there may be a better approach here than bsearch() - d_path_aux = (const DPathData2*)bsearch(&seek, &d_path[d_path_index[d - 1]], d_path_index[d] - d_path_index[d - 1], sizeof(DPathData2), CompareDPathData2); + for (;;) { aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); + if (--d == -1) { + break; + } + d_path_aux = &d_path[d_path_index[d] + (d_path_aux->pre_k - d_path[d_path_index[d]].k) / 2]; } align.reset(align.aln_q_e + align.aln_t_e); int current_x(aln_path[aln_idx].x); @@ -86,6 +80,10 @@ static int Align(const int segment_size, const std::string& query, const int q_o int best_m(-1), min_k(0), max_k(0); for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { d_path_index[d] = d_path_idx; // starting point of each "d" set of entries + const size_t possible_growth((max_k - min_k) / 2 + 1); + if (d_path.size() < d_path_idx + possible_growth) { + d_path.resize(d_path_idx + possible_growth); + } // k is the offset between query and target for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; @@ -105,12 +103,7 @@ static int Align(const int segment_size, const std::string& query, const int q_o } else { for (; x < segment_size && y < segment_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } - if (d_path_idx != d_path.size()) { - d_path[d_path_idx].set(k, x1, y1, x, y, pre_k); - } else { - d_path.push_back(DPathData2(k, x1, y1, x, y, pre_k)); - } - ++d_path_idx; + d_path[d_path_idx++].set(k, x1, y1, x, y, pre_k); // see if we got as much as we can if (x == segment_size || y == segment_size) { fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, d_path_index, d, aln_path, extend_forward); @@ -143,9 +136,10 @@ static int Align(const int segment_size, const std::string& query, const int q_o } static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { - int q_extend(0), t_extend(0), not_at_end(1); const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset); const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset); + // amount we've already extended + int q_extend(0), t_extend(0), not_at_end(1); do { // size left to extend const int extend_size(std::min(q_extend_max - q_extend, t_extend_max - t_extend)); @@ -154,7 +148,8 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } U.assign(U.size(), 0); // are these two initializations necessary? V.assign(V.size(), 0); - if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, d_path_index, aln_path, extend_forward, error_rate)) { return; + if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, d_path_index, aln_path, extend_forward, error_rate)) { + return; } int k; if (not_at_end) { @@ -162,7 +157,8 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co // matching basepairs), counting non-gap basepairs int q_bps(0), t_bps(0), num_matches(0); if (extend_forward) { - for (k = align.size - 1; -1 < k; --k) { + // XXX - change these to == GAP_ALN, like below + for (k = align.size - 1; k != -1; --k) { if (align.q_aln_str[k] != GAP_ALN) { ++q_bps; } @@ -176,14 +172,14 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } } } else { - for (k = align.size; 0 < k; --k) { - if (align.q_aln_str[align.q_aln_str.size() - k] != GAP_ALN) { + for (k = align.q_aln_str.size() - align.size; k != static_cast(align.q_aln_str.size()); ++k) { + if (align.q_aln_str[k] != GAP_ALN) { ++q_bps; } - if (align.t_aln_str[align.t_aln_str.size() - k] != GAP_ALN) { + if (align.t_aln_str[k] != GAP_ALN) { ++t_bps; } - if (align.q_aln_str[align.q_aln_str.size() - k] != align.t_aln_str[align.t_aln_str.size() - k]) { + if (align.q_aln_str[k] != align.t_aln_str[k]) { num_matches = 0; } else if (++num_matches == 4) { break; @@ -193,41 +189,41 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co if (align.aln_q_e == q_bps) { // no good match return; } - // only extend to the good match + // only extend to the good match; that said, not + // sure why we're tossing the good match itself q_extend += align.aln_q_e - q_bps; t_extend += align.aln_t_e - t_bps; } else if (align.aln_q_e == 0) { // no good match return; - } else { + } else if (extend_forward) { k = align.size; + } else { + k = align.q_aln_str.size() - align.size; } if (extend_forward) { const int i(result.buffer_start + result.right_size); - std::copy(&align.q_aln_str[0], &align.q_aln_str[k], &result.q_buffer[i]); - std::copy(&align.t_aln_str[0], &align.t_aln_str[k], &result.t_buffer[i]); + std::copy(&align.q_aln_str[0], &align.q_aln_str[0] + k, &result.q_buffer[i]); + std::copy(&align.t_aln_str[0], &align.t_aln_str[0] + k, &result.t_buffer[i]); result.right_size += k; } else { - result.left_size += k; + result.left_size += align.q_aln_str.size() - k; const int i(result.buffer_start - result.left_size); - char* const q_aln_end(&align.q_aln_str[0] + align.q_aln_str.size()); - char* const t_aln_end(&align.t_aln_str[0] + align.t_aln_str.size()); - std::copy(q_aln_end - k, q_aln_end, &result.q_buffer[i]); - std::copy(t_aln_end - k, t_aln_end, &result.t_buffer[i]); + std::copy(&align.q_aln_str[k], &align.q_aln_str[0] + align.q_aln_str.size(), &result.q_buffer[i]); + std::copy(&align.t_aln_str[k], &align.t_aln_str[0] + align.q_aln_str.size(), &result.t_buffer[i]); } } while (not_at_end); } -static void count_basepairs(const OutputStore& result, int i, const int end_i, int& q_bps, int& t_bps) { - for (; i != end_i; ++i) { - int ch(result.q_buffer[i]); - assert(-1 < ch && ch < 5); - if (ch != GAP_ALN) { - ++q_bps; +// remove gaps from the basepair count +static void remove_gaps(const OutputStore& result, const int start_i, const int end_i, int& q_bps, int& t_bps) { + for (int i(start_i); i != end_i; ++i) { + if (result.q_buffer[i] == GAP_ALN) { + --q_bps; } - ch = result.t_buffer[i]; - assert(-1 < ch && ch < 5); - if (ch != GAP_ALN) { - ++t_bps; + } + for (int i(start_i); i != end_i; ++i) { + if (result.t_buffer[i] == GAP_ALN) { + --t_bps; } } } @@ -242,11 +238,14 @@ static int dw(const std::string& query, const int query_start, const std::string return 0; } int q_bps, t_bps; - // initialize i and j outside subroutine to avoid warning - count_basepairs(result, result.buffer_start - result.left_size, result.buffer_start, q_bps = 0, t_bps = 0); + q_bps = t_bps = result.left_size; + remove_gaps(result, result.buffer_start - result.left_size, result.buffer_start, q_bps, t_bps); +std::cerr << query_start << " " << query.size() << " " << target_start << " " << target.size() << " | " << q_bps << " " << t_bps << " | "; result.query_start = query_start - q_bps; result.target_start = target_start - t_bps; - count_basepairs(result, result.buffer_start, result.buffer_start + result.right_size, q_bps = 0, t_bps = 0); + q_bps = t_bps = result.right_size; + remove_gaps(result, result.buffer_start, result.buffer_start + result.right_size, q_bps, t_bps); +std::cerr << q_bps << " " << t_bps << "\n"; result.query_end = query_start + q_bps; result.target_end = target_start + t_bps; return 1; @@ -267,59 +266,79 @@ int GetAlignment(const std::string& query, const int query_start, const std::str // create return m5 record const int consecutive_match_region_size(4); // trim starting end of alignment - int qrb(0); // q starting pads - int trb(0); // t starting pads + int qrb(0); // q starting basepair offset to good sequence + int trb(0); // t starting basepair offset to good sequence int eit(0); // matching run length - int k; - for (k = result.buffer_start - result.left_size; k != result.buffer_start + result.right_size; ++k) { + const int start_k(result.buffer_start - result.left_size); + const int end_k(result.buffer_start + result.right_size); + int k(start_k); + for (; k != end_k; ++k) { const char qc(result.q_buffer[k]); const char tc(result.t_buffer[k]); - if (qc != GAP_ALN) { - ++qrb; - } - if (tc != GAP_ALN) { - ++trb; - } if (qc != tc) { eit = 0; + // we don't count gaps + if (qc == GAP_ALN) { + --qrb; + } else if (tc == GAP_ALN) { + --trb; + } } else if (++eit == consecutive_match_region_size) { - ++k; break; } } if (eit < consecutive_match_region_size) { // no good match return 0; } - qrb -= consecutive_match_region_size; - trb -= consecutive_match_region_size; - const int start_aln_id(k - consecutive_match_region_size); + // +1 for the ++k we skipped with the break + const int start_aln_id(k + 1 - consecutive_match_region_size); + qrb += start_aln_id - start_k; + trb += start_aln_id - start_k; // trim trailing end of alignment - int qre(0); // q ending pads - int tre(0); // t ending pads + int qre(0); // q ending basepair offset to good sequence + int tre(0); // t ending basepair offset to good sequence eit = 0; // still matching run length - for (k = result.buffer_start + result.right_size - 1;; --k) { + for (k = end_k - 1;; --k) { const char qc(result.q_buffer[k]); const char tc(result.t_buffer[k]); - if (qc != GAP_ALN) { - ++qre; - } - if (tc != GAP_ALN) { - ++tre; - } if (qc != tc) { eit = 0; + // we don't count gaps + if (qc == GAP_ALN) { + --qre; + } else if (tc == GAP_ALN) { + --tre; + } } else if (++eit == consecutive_match_region_size) { break; } } - qre -= consecutive_match_region_size; - tre -= consecutive_match_region_size; + // -1 for the --k we skipped with the break + k += consecutive_match_region_size - 1; + qre += end_k - k; + tre += end_k - k; +std::cerr << "k: " << start_k << " " << end_k << " " << start_aln_id << " " << k << "\n"; +std::cerr << "result: " << result.query_start << " " << result.query_end << " " << result.target_start << " " << result.target_end << " (" << result.buffer_start << " " << result.left_size << " " << result.right_size << ") | " << qrb << " " << qre << " " << trb << " " << tre << "\n"; m5.qoff = result.query_start + qrb; m5.qend = result.query_end - qre; m5.soff = result.target_start + trb; m5.send = result.target_end - tre; - const size_t aln_size(k + consecutive_match_region_size - start_aln_id); + // +1 to make this end-inclusive + const size_t aln_size(k - start_aln_id + 1); decode_sequence(m5.qaln, result.q_buffer, start_aln_id, aln_size); decode_sequence(m5.saln, result.t_buffer, start_aln_id, aln_size); +// XXX - one result was missing query[query_start - 1], which should not happen +if (result.query_start < 0) { +std::string s(query_start, 0); +for (int i(0); i != query_start; ++i) { +s[i] = "ACGT-"[static_cast(query[i])]; +} +std::cerr << s << "\n\n" << m5.qaln.substr(0, result.buffer_start - start_aln_id) << "\n\n"; +s.assign(query.size() - query_start, 0); +for (size_t i(0); i != query.size() - query_start; ++i) { +s[i] = "ACGT-"[static_cast(query[query_start + i])]; +} +std::cerr << s << "\n\n" << m5.qaln.substr(result.buffer_start - start_aln_id, aln_size - (result.buffer_start - start_aln_id)) << "\n\n"; +} return 1; } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 35bd5ec..32890ea 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -89,6 +89,8 @@ class DiffRunningData { std::vector query, target; std::vector DynQ, DynT; std::vector d_path_index; + // maybe make d_path a deque rather than a vector, for growth + // speed and (likely) reduced memory waste? std::vector d_path; std::vector aln_path; public: diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 7f82451..f92d2a4 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -453,9 +453,8 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); -std::cerr << "(" << ec.sid << " " << ec.sext << " " << ssize << ") (" << ec.qid << " " << ec.qext() << " " << qsize << ") " << ec.qdir() << " " << ec.score << "\n"; +std::cerr << "can: " << ec.sid << " " << ec.qid << " " << ec.sext << " " << ec.qext() << " " << ec.qdir() << " " << ec.score << "\n"; const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); -std::cerr << m5.soff << " " << m5.send << " " << m5.qoff << " " << m5.qend << " " << m5.saln.size() << " " << m5.qaln.size() << "\n" << m5.saln << "\n" << m5.qaln << "\n"; if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 7cc15e4..9ce0c56 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -14,7 +14,7 @@ static void* reads_correction_func_can(void* const arg) { ConsensusThreadData& data(*(static_cast(arg))); const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); - const ExtensionCandidateCompressed* const candidates((ExtensionCandidateCompressed*)pdata.candidates); + const ExtensionCandidateCompressed* const candidates((const ExtensionCandidateCompressed*)pdata.candidates); idx_t i(pdata.next_candidate); if (data.rco.tech == TECH_PACBIO) { while (i != pdata.num_candidates) { From 7d62c5d3e0df036e847848bb99328d9f52816b47 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 3 May 2019 15:20:47 -0500 Subject: [PATCH 37/56] fixed bugs, improved speed of alignment finding finally nailed all the bugs (I hope) I created by changing dw.cpp, and the changes should speed up alignment creation as well as reduce memory usage --- src/main.mk | 4 +- src/mecat2cns/dw.cpp | 145 +++++++++++++---------------- src/mecat2cns/dw.h | 35 ++++--- src/mecat2cns/main.cpp | 3 +- src/mecat2cns/mecat2cns.mk | 4 +- src/mecat2cns/mecat_correction.cpp | 1 - 6 files changed, 90 insertions(+), 102 deletions(-) diff --git a/src/main.mk b/src/main.mk index d3892b2..55dfaa7 100644 --- a/src/main.mk +++ b/src/main.mk @@ -25,5 +25,5 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -g -TGT_LDFLAGS := -pg -g +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg +TGT_LDFLAGS := -pg diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 1b83b4d..0b8ff2c 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -4,30 +4,28 @@ #define GAP_ALN 4 -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const size_t d_path_idx, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { - // the final one is always last on the stack - const DPathData2* d_path_aux(&d_path[d_path_idx - 1]); +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; // get align path if (static_cast(aln_path.size()) < (d + 1) * 2) { aln_path.resize((d + 1) * 2); } - size_t aln_idx(-1); + int aln_idx(-1); for (;;) { aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); if (--d == -1) { break; } - d_path_aux = &d_path[d_path_index[d] + (d_path_aux->pre_k - d_path[d_path_index[d]].k) / 2]; + d_path_aux = &d_path[d_path_index[d].d_offset + (d_path_aux->pre_k - d_path_index[d].min_k) / 2]; } align.reset(align.aln_q_e + align.aln_t_e); int current_x(aln_path[aln_idx].x); int current_y(aln_path[aln_idx].y); - const char* const query_p(query.data() + q_offset); - const char* const target_p(target.data() + t_offset); - for (--aln_idx; aln_idx != size_t(-1); --aln_idx) { + const char* const query_p(query.data() + q_offset + (extend_forward ? 0 : 1)); + const char* const target_p(target.data() + t_offset + (extend_forward ? 0 : 1)); + for (--aln_idx; aln_idx != -1; --aln_idx) { const int new_x(aln_path[aln_idx].x); const int new_y(aln_path[aln_idx].y); const int dx(new_x - current_x); @@ -68,7 +66,9 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { + U.assign(U.size(), 0); + V.assign(V.size(), 0); const int k_offset(segment_size * 4 * error_rate); if (static_cast(d_path_index.size()) < k_offset) { d_path_index.resize(k_offset); @@ -76,10 +76,11 @@ static int Align(const int segment_size, const std::string& query, const int q_o const int band_tolerance(segment_size / 10 * 3 + 1); // max band size is max offset between query and target const int max_band_size(band_tolerance * 2 - 1); - size_t d_path_idx(0); + int d_path_idx(0); int best_m(-1), min_k(0), max_k(0); for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { - d_path_index[d] = d_path_idx; // starting point of each "d" set of entries + // starting point of each "d" set of entries + d_path_index[d].set(d_path_idx, min_k); const size_t possible_growth((max_k - min_k) / 2 + 1); if (d_path.size() < d_path_idx + possible_growth) { d_path.resize(d_path_idx + possible_growth); @@ -103,12 +104,13 @@ static int Align(const int segment_size, const std::string& query, const int q_o } else { for (; x < segment_size && y < segment_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } - d_path[d_path_idx++].set(k, x1, y1, x, y, pre_k); + d_path[d_path_idx].set(x1, y1, x, y, pre_k); // see if we got as much as we can if (x == segment_size || y == segment_size) { - fill_align(query, q_offset, target, t_offset, align, d_path, d_path_idx, d_path_index, d, aln_path, extend_forward); + fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[d_path_idx], d_path_index, d, aln_path, extend_forward); return 1; } + ++d_path_idx; V[k_offset + k] = x; U[k_offset + k] = x + y; if (best_m < x + y) { @@ -135,9 +137,9 @@ static int Align(const int segment_size, const std::string& query, const int q_o return 0; } -static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { - const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset); - const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset); +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { + const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset + 1); + const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset + 1); // amount we've already extended int q_extend(0), t_extend(0), not_at_end(1); do { @@ -146,8 +148,6 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co if (extend_size < segment_size + 101) { // close enough to the end not_at_end = 0; } - U.assign(U.size(), 0); // are these two initializations necessary? - V.assign(V.size(), 0); if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, d_path_index, aln_path, extend_forward, error_rate)) { return; } @@ -157,40 +157,47 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co // matching basepairs), counting non-gap basepairs int q_bps(0), t_bps(0), num_matches(0); if (extend_forward) { - // XXX - change these to == GAP_ALN, like below for (k = align.size - 1; k != -1; --k) { - if (align.q_aln_str[k] != GAP_ALN) { - ++q_bps; - } - if (align.t_aln_str[k] != GAP_ALN) { - ++t_bps; - } - if (align.q_aln_str[k] != align.t_aln_str[k]) { + const char qc(align.q_aln_str[k]); + const char tc(align.t_aln_str[k]); + if (qc != tc) { num_matches = 0; + if (qc == GAP_ALN) { + --q_bps; + } else if (tc == GAP_ALN) { + --t_bps; + } } else if (++num_matches == 4) { break; } } + q_bps += align.size - k; + t_bps += align.size - k; } else { - for (k = align.q_aln_str.size() - align.size; k != static_cast(align.q_aln_str.size()); ++k) { - if (align.q_aln_str[k] != GAP_ALN) { - ++q_bps; - } - if (align.t_aln_str[k] != GAP_ALN) { - ++t_bps; - } - if (align.q_aln_str[k] != align.t_aln_str[k]) { + const int offset(align.q_aln_str.size() - align.size); + for (k = offset; k != static_cast(align.q_aln_str.size()); ++k) { + const char qc(align.q_aln_str[k]); + const char tc(align.t_aln_str[k]); + if (qc != tc) { num_matches = 0; + if (qc == GAP_ALN) { + --q_bps; + } else if (tc == GAP_ALN) { + --t_bps; + } } else if (++num_matches == 4) { + ++k; // don't include final match break; } } + q_bps += k - offset; + t_bps += k - offset; } if (align.aln_q_e == q_bps) { // no good match return; } - // only extend to the good match; that said, not - // sure why we're tossing the good match itself + // don't extend into the good match; keeping good match for + // next alignment, maybe? q_extend += align.aln_q_e - q_bps; t_extend += align.aln_t_e - t_bps; } else if (align.aln_q_e == 0) { // no good match @@ -214,40 +221,31 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } while (not_at_end); } -// remove gaps from the basepair count -static void remove_gaps(const OutputStore& result, const int start_i, const int end_i, int& q_bps, int& t_bps) { - for (int i(start_i); i != end_i; ++i) { - if (result.q_buffer[i] == GAP_ALN) { - --q_bps; - } - } - for (int i(start_i); i != end_i; ++i) { - if (result.t_buffer[i] == GAP_ALN) { - --t_bps; +static int gap_count(const std::vector& buffer, int i, const int end_i) { + int j(0); + for (; i != end_i; ++i) { + if (buffer[i] == GAP_ALN) { + ++j; } } + return j; } -static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { result.reset_buffer(query_start + target_start, query.size() + target.size()); - // reverse extend + // reverse extend (left side) dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 0, error_rate); - // forward extend + // forward extend (right side) dw_in_one_direction(query, query_start, target, target_start, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 1, error_rate); if (result.left_size + result.right_size < min_aln_size) { return 0; } - int q_bps, t_bps; - q_bps = t_bps = result.left_size; - remove_gaps(result, result.buffer_start - result.left_size, result.buffer_start, q_bps, t_bps); -std::cerr << query_start << " " << query.size() << " " << target_start << " " << target.size() << " | " << q_bps << " " << t_bps << " | "; - result.query_start = query_start - q_bps; - result.target_start = target_start - t_bps; - q_bps = t_bps = result.right_size; - remove_gaps(result, result.buffer_start, result.buffer_start + result.right_size, q_bps, t_bps); -std::cerr << q_bps << " " << t_bps << "\n"; - result.query_end = query_start + q_bps; - result.target_end = target_start + t_bps; + const int left_start(result.buffer_start - result.left_size); + const int right_end(result.buffer_start + result.right_size); + result.query_start = query_start - result.left_size + gap_count(result.q_buffer, left_start, result.buffer_start); + result.query_end = query_start + result.right_size - gap_count(result.q_buffer, result.buffer_start, right_end); + result.target_start = target_start - result.left_size + gap_count(result.t_buffer, left_start, result.buffer_start); + result.target_end = target_start + result.right_size - gap_count(result.t_buffer, result.buffer_start, right_end); return 1; } @@ -259,11 +257,11 @@ static void decode_sequence(std::string& out_seq, const std::vector& in_se } int GetAlignment(const std::string& query, const int query_start, const std::string& target, const int target_start, DiffRunningData& drd, M5Record& m5, const double error_rate, const int min_aln_size) { - OutputStore& result(drd.result); - if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.d_path_index, drd.aln_path, result, drd.segment_size, error_rate, min_aln_size)) { + if (!dw(query, query_start, target, target_start, drd.DynQ, drd.DynT, drd.align, drd.d_path, drd.d_path_index, drd.aln_path, drd.result, drd.segment_size, error_rate, min_aln_size)) { return 0; } // create return m5 record + const OutputStore& result(drd.result); const int consecutive_match_region_size(4); // trim starting end of alignment int qrb(0); // q starting basepair offset to good sequence @@ -287,7 +285,7 @@ int GetAlignment(const std::string& query, const int query_start, const std::str break; } } - if (eit < consecutive_match_region_size) { // no good match + if (eit != consecutive_match_region_size) { // no good match return 0; } // +1 for the ++k we skipped with the break @@ -313,32 +311,17 @@ int GetAlignment(const std::string& query, const int query_start, const std::str break; } } - // -1 for the --k we skipped with the break - k += consecutive_match_region_size - 1; + // --k we skipped in the break is countered by +1 for end-inclusion + k += consecutive_match_region_size; qre += end_k - k; tre += end_k - k; -std::cerr << "k: " << start_k << " " << end_k << " " << start_aln_id << " " << k << "\n"; -std::cerr << "result: " << result.query_start << " " << result.query_end << " " << result.target_start << " " << result.target_end << " (" << result.buffer_start << " " << result.left_size << " " << result.right_size << ") | " << qrb << " " << qre << " " << trb << " " << tre << "\n"; m5.qoff = result.query_start + qrb; m5.qend = result.query_end - qre; m5.soff = result.target_start + trb; m5.send = result.target_end - tre; // +1 to make this end-inclusive - const size_t aln_size(k - start_aln_id + 1); + const size_t aln_size(k - start_aln_id); decode_sequence(m5.qaln, result.q_buffer, start_aln_id, aln_size); decode_sequence(m5.saln, result.t_buffer, start_aln_id, aln_size); -// XXX - one result was missing query[query_start - 1], which should not happen -if (result.query_start < 0) { -std::string s(query_start, 0); -for (int i(0); i != query_start; ++i) { -s[i] = "ACGT-"[static_cast(query[i])]; -} -std::cerr << s << "\n\n" << m5.qaln.substr(0, result.buffer_start - start_aln_id) << "\n\n"; -s.assign(query.size() - query_start, 0); -for (size_t i(0); i != query.size() - query_start; ++i) { -s[i] = "ACGT-"[static_cast(query[query_start + i])]; -} -std::cerr << s << "\n\n" << m5.qaln.substr(result.buffer_start - start_aln_id, aln_size - (result.buffer_start - start_aln_id)) << "\n\n"; -} return 1; } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 32890ea..5397914 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -57,24 +57,31 @@ class OutputStore { } }; -struct DPathData2 { - int k, x1, y1, x2, y2, pre_k; - explicit DPathData2() { } - explicit DPathData2(const int j, const int ki, const int l, const int m, const int n, const int p) : k(j), x1(ki), y1(l), x2(m), y2(n), pre_k(p) { } - void set(const int j, const int ki, const int l, const int m, const int n, const int p) { - k = j; - x1 = ki; - y1 = l; - x2 = m; - y2 = n; - pre_k = p; +struct DPathData { + int x1, y1, x2, y2, pre_k; + explicit DPathData() { } + explicit DPathData(const int i, const int j, const int k, const int l, const int m) : x1(i), y1(j), x2(k), y2(l), pre_k(m) { } + void set(const int i, const int j, const int k, const int l, const int m) { + x1 = i; + y1 = j; + x2 = k; + y2 = l; + pre_k = m; + } +}; + +struct DPathIndex { + int d_offset, min_k; + explicit DPathIndex() { } + void set(const int i, const int j) { + d_offset = i; + min_k = j; } }; struct PathPoint { int x, y; explicit PathPoint() { } - explicit PathPoint(const int i, const int j) : x(i), y(j) { } void set(const int i, const int j) { x = i; y = j; @@ -88,10 +95,10 @@ class DiffRunningData { OutputStore result; std::vector query, target; std::vector DynQ, DynT; - std::vector d_path_index; + std::vector d_path_index; // maybe make d_path a deque rather than a vector, for growth // speed and (likely) reduced memory waste? - std::vector d_path; + std::vector d_path; std::vector aln_path; public: explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), DynQ(swp.row_size), DynT(swp.column_size) { } diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 8d1c7a7..90d883c 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -33,8 +33,7 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt unlink(script_file.c_str()); std::ofstream out; open_fstream(out, script_file.c_str(), std::ios::out); - //out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; - out << "#!/bin/bash\n" << prog << make_options(new_options) << "\n"; + out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; if (!out) { std::cerr << "Error writing to " << script_file << "\n"; exit(1); diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index d41de42..775e943 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,8 +20,8 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -g -TGT_LDFLAGS := -L${TARGET_DIR} -pg -g +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg +TGT_LDFLAGS := -L${TARGET_DIR} -pg TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index f92d2a4..7732983 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -453,7 +453,6 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); -std::cerr << "can: " << ec.sid << " " << ec.qid << " " << ec.sext << " " << ec.qext() << " " << ec.qdir() << " " << ec.score << "\n"; const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { From 7025c3c2114558ad58d57e30b7130cb4cc3d1d51 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 3 May 2019 16:48:45 -0500 Subject: [PATCH 38/56] increase RM from 100k to 500k also testing d_path as a deque rather than a vector --- src/mecat2cns/dw.cpp | 9 +++++---- src/mecat2cns/dw.h | 3 ++- src/mecat2pw/pw_impl.cpp | 2 +- src/mecat2ref/mecat2ref.cpp | 2 +- src/mecat2ref/mecat2ref_defs.h | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 0b8ff2c..5ae7275 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,10 +1,11 @@ #include "dw.h" #include // copy(), fill() +#include // deque<> #include // vector<> #define GAP_ALN 4 -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::deque& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; // get align path @@ -66,7 +67,7 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { U.assign(U.size(), 0); V.assign(V.size(), 0); const int k_offset(segment_size * 4 * error_rate); @@ -137,7 +138,7 @@ static int Align(const int segment_size, const std::string& query, const int q_o return 0; } -static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset + 1); const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset + 1); // amount we've already extended @@ -231,7 +232,7 @@ static int gap_count(const std::vector& buffer, int i, const int end_i) { return j; } -static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { result.reset_buffer(query_start + target_start, query.size() + target.size()); // reverse extend (left side) dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 0, error_rate); diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 5397914..1fc3eab 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -1,6 +1,7 @@ #ifndef DW_H #define DW_H +#include // deque<> #include // string #include // vector<> #include "../common/defs.h" // idx_t @@ -98,7 +99,7 @@ class DiffRunningData { std::vector d_path_index; // maybe make d_path a deque rather than a vector, for growth // speed and (likely) reduced memory waste? - std::vector d_path; + std::deque d_path; std::vector aln_path; public: explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), DynQ(swp.row_size), DynT(swp.column_size) { } diff --git a/src/mecat2pw/pw_impl.cpp b/src/mecat2pw/pw_impl.cpp index 493ad6e..2654829 100644 --- a/src/mecat2pw/pw_impl.cpp +++ b/src/mecat2pw/pw_impl.cpp @@ -15,7 +15,7 @@ #include #include // unlink() -#define RM 100000 +#define RM 500000 #define DN 500 #define BC 10 #define SM 40 diff --git a/src/mecat2ref/mecat2ref.cpp b/src/mecat2ref/mecat2ref.cpp index 36c94fb..34f9061 100644 --- a/src/mecat2ref/mecat2ref.cpp +++ b/src/mecat2ref/mecat2ref.cpp @@ -9,7 +9,7 @@ #include #include #include -#define RM 100000 +#define RM 500000 #include "output.h" #include "../common/defs.h" diff --git a/src/mecat2ref/mecat2ref_defs.h b/src/mecat2ref/mecat2ref_defs.h index 79ea59f..6b63587 100644 --- a/src/mecat2ref/mecat2ref_defs.h +++ b/src/mecat2ref/mecat2ref_defs.h @@ -13,7 +13,7 @@ #include #include -#define RM 100000 +#define RM 500000 #define ZV 1000 #define ZVS 2000 #define ZVL 1000L From 80fdb916f10d1d776a6b10fed56390f9a74cedf5 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Sat, 4 May 2019 00:40:28 -0500 Subject: [PATCH 39/56] back to static allocation turns out dynamic allocation comes with a large cost - 33% slower, and not appreciably less memory usage. The other changes made a major speed increase, though, 2.5-3x increase. --- src/main.mk | 3 +-- src/mecat2cns/dw.cpp | 27 ++++++----------------- src/mecat2cns/dw.h | 35 ++++++++++++++---------------- src/mecat2cns/mecat2cns.mk | 4 ++-- src/mecat2cns/mecat_correction.cpp | 4 ++-- 5 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/main.mk b/src/main.mk index 55dfaa7..fb23300 100644 --- a/src/main.mk +++ b/src/main.mk @@ -25,5 +25,4 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -TGT_LDFLAGS := -pg +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 5ae7275..c149491 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,17 +1,13 @@ #include "dw.h" #include // copy(), fill() -#include // deque<> #include // vector<> #define GAP_ALN 4 -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::deque& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { align.aln_q_e = d_path_aux->x2; align.aln_t_e = d_path_aux->y2; // get align path - if (static_cast(aln_path.size()) < (d + 1) * 2) { - aln_path.resize((d + 1) * 2); - } int aln_idx(-1); for (;;) { aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); @@ -21,7 +17,7 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } d_path_aux = &d_path[d_path_index[d].d_offset + (d_path_aux->pre_k - d_path_index[d].min_k) / 2]; } - align.reset(align.aln_q_e + align.aln_t_e); + align.reset(); int current_x(aln_path[aln_idx].x); int current_y(aln_path[aln_idx].y); const char* const query_p(query.data() + q_offset + (extend_forward ? 0 : 1)); @@ -67,25 +63,16 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { U.assign(U.size(), 0); V.assign(V.size(), 0); const int k_offset(segment_size * 4 * error_rate); - if (static_cast(d_path_index.size()) < k_offset) { - d_path_index.resize(k_offset); - } const int band_tolerance(segment_size / 10 * 3 + 1); - // max band size is max offset between query and target const int max_band_size(band_tolerance * 2 - 1); - int d_path_idx(0); - int best_m(-1), min_k(0), max_k(0); + int d_path_idx(0), best_m(-1), min_k(0), max_k(0); for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries d_path_index[d].set(d_path_idx, min_k); - const size_t possible_growth((max_k - min_k) / 2 + 1); - if (d_path.size() < d_path_idx + possible_growth) { - d_path.resize(d_path_idx + possible_growth); - } // k is the offset between query and target for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; @@ -138,7 +125,7 @@ static int Align(const int segment_size, const std::string& query, const int q_o return 0; } -static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { +static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { const int q_extend_max(extend_forward ? query.size() - q_offset : q_offset + 1); const int t_extend_max(extend_forward ? target.size() - t_offset : t_offset + 1); // amount we've already extended @@ -232,8 +219,8 @@ static int gap_count(const std::vector& buffer, int i, const int end_i) { return j; } -static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::deque& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { - result.reset_buffer(query_start + target_start, query.size() + target.size()); +static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { + result.reset_buffer(query_start + target_start); // reverse extend (left side) dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 0, error_rate); // forward extend (right side) diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 1fc3eab..f453f04 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -1,20 +1,19 @@ #ifndef DW_H #define DW_H -#include // deque<> #include // string #include // vector<> #include "../common/defs.h" // idx_t struct SW_Parameters { int segment_size; // probably best if it's a multiple of 10 - int row_size, column_size; - SW_Parameters(const int i, const int j, const int k) : segment_size(i), row_size(j), column_size(k) { } + int row_size, column_size, d_path_size; + SW_Parameters(const int i, const int j, const int k, const int l) : segment_size(i), row_size(j), column_size(k), d_path_size(l) { } }; inline SW_Parameters get_sw_parameters_small() { // 1000 instead of 500 for "large" - return SW_Parameters(500, 4096, 4096); + return SW_Parameters(500, 4096, 4096, 500000); } class Alignment { @@ -25,13 +24,13 @@ class Alignment { std::vector q_aln_str, t_aln_str; public: explicit Alignment() { } + explicit Alignment(const size_t new_max_size) { + q_aln_str.resize(new_max_size); + t_aln_str.resize(new_max_size); + } ~Alignment() { } - void reset(const size_t new_max_size) { + void reset() { size = 0; - if (q_aln_str.size() < new_max_size) { - q_aln_str.resize(new_max_size); - t_aln_str.resize(new_max_size); - } } }; @@ -47,14 +46,14 @@ class OutputStore { // matches/mismatches are pretty obvious public: explicit OutputStore() { } + explicit OutputStore(const size_t new_max_size) { + q_buffer.resize(new_max_size); + t_buffer.resize(new_max_size); + } ~OutputStore() { } - void reset_buffer(const int i, const size_t new_max_size) { + void reset_buffer(const int i) { buffer_start = i; left_size = right_size = 0; - if (q_buffer.size() < new_max_size) { - q_buffer.resize(new_max_size); - t_buffer.resize(new_max_size); - } } }; @@ -94,15 +93,13 @@ class DiffRunningData { const int segment_size; Alignment align; OutputStore result; - std::vector query, target; std::vector DynQ, DynT; + std::vector d_path; std::vector d_path_index; - // maybe make d_path a deque rather than a vector, for growth - // speed and (likely) reduced memory waste? - std::deque d_path; std::vector aln_path; public: - explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), DynQ(swp.row_size), DynT(swp.column_size) { } + // work on more accurate sizing of these buffers + explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), align((swp.segment_size + 100) * 2), result(MAX_SEQ_SIZE * 2), DynQ(swp.row_size), DynT(swp.column_size), d_path(swp.d_path_size), d_path_index(swp.segment_size * 2), aln_path(swp.segment_size * 4) { } ~DiffRunningData() { } }; diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index 775e943..fb17ae4 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,8 +20,8 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -pg -TGT_LDFLAGS := -L${TARGET_DIR} -pg +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 +TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 7732983..db559f6 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -73,7 +73,7 @@ static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, con } static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target) { - if (static_cast(cns_id_vec.size()) < cns_list_size) { + if (cns_id_vec.size() < static_cast(cns_list_size)) { cns_id_vec.resize(cns_list_size); } // get types of coverage @@ -400,7 +400,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData // once coverage gets high enough, stop adding coverage and return false static inline int check_cov_stats(std::vector& cov_stats, const int soff, const int send) { - if (static_cast(cov_stats.size()) < send) { + if (cov_stats.size() < static_cast(send)) { cov_stats.resize(send, 0); } int n(0); From dab77e24b70da8e3ce9056762e5ff101a95f1a22 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Sat, 4 May 2019 14:05:31 -0500 Subject: [PATCH 40/56] perform better static sizing of GetAlignment() buffers --- src/mecat2cns/dw.cpp | 18 +++++----- src/mecat2cns/dw.h | 50 ++++++++++++---------------- src/mecat2cns/packed_db.h | 25 +++++++++----- src/mecat2cns/reads_correction_aux.h | 11 ++++-- 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index c149491..519243d 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -63,11 +63,11 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int segment_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { - U.assign(U.size(), 0); - V.assign(V.size(), 0); - const int k_offset(segment_size * 4 * error_rate); - const int band_tolerance(segment_size / 10 * 3 + 1); +static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { + const int k_offset(extend_size * 4 * error_rate); + U.assign(k_offset * 2, 0); + V.assign(k_offset * 2, 0); + const int band_tolerance(extend_size / 10 * 3 + 1); const int max_band_size(band_tolerance * 2 - 1); int d_path_idx(0), best_m(-1), min_k(0), max_k(0); for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { @@ -88,13 +88,13 @@ static int Align(const int segment_size, const std::string& query, const int q_o const int x1(x), y1(y); // find the other end of exact match if (extend_forward) { - for (; x < segment_size && y < segment_size && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } + for (; x < extend_size && y < extend_size && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } } else { - for (; x < segment_size && y < segment_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } + for (; x < extend_size && y < extend_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } } d_path[d_path_idx].set(x1, y1, x, y, pre_k); // see if we got as much as we can - if (x == segment_size || y == segment_size) { + if (x == extend_size || y == extend_size) { fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[d_path_idx], d_path_index, d, aln_path, extend_forward); return 1; } @@ -133,7 +133,7 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co do { // size left to extend const int extend_size(std::min(q_extend_max - q_extend, t_extend_max - t_extend)); - if (extend_size < segment_size + 101) { // close enough to the end + if (extend_size <= segment_size + SEGMENT_BORDER) { // close enough to the end not_at_end = 0; } if (!Align(not_at_end ? segment_size : extend_size, query, q_offset + (extend_forward ? q_extend : -q_extend), target, t_offset + (extend_forward ? t_extend : -t_extend), align, U, V, d_path, d_path_index, aln_path, extend_forward, error_rate)) { diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index f453f04..4e28c9c 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -5,25 +5,12 @@ #include // vector<> #include "../common/defs.h" // idx_t -struct SW_Parameters { - int segment_size; // probably best if it's a multiple of 10 - int row_size, column_size, d_path_size; - SW_Parameters(const int i, const int j, const int k, const int l) : segment_size(i), row_size(j), column_size(k), d_path_size(l) { } -}; - -inline SW_Parameters get_sw_parameters_small() { - // 1000 instead of 500 for "large" - return SW_Parameters(500, 4096, 4096, 500000); -} - class Alignment { public: // size tracks actual buffer use int aln_q_e, aln_t_e, size; - // these are buffers we only expand std::vector q_aln_str, t_aln_str; public: - explicit Alignment() { } explicit Alignment(const size_t new_max_size) { q_aln_str.resize(new_max_size); t_aln_str.resize(new_max_size); @@ -36,21 +23,20 @@ class Alignment { class OutputStore { public: - // these track actual buffer use + // these track actual buffer use - buffer use starts at + // buffer_start, with left going down, and right going up int buffer_start, left_size, right_size; int query_start, query_end; int target_start, target_end; - // these are buffers that we only expand std::vector q_buffer, t_buffer; - // for the record, inserts are q_buffer == 4, deletes are t_buffer == 4; - // matches/mismatches are pretty obvious public: explicit OutputStore() { } - explicit OutputStore(const size_t new_max_size) { + ~OutputStore() { } + // can't figure out how to pass this in on initialization + void resize(const size_t new_max_size) { q_buffer.resize(new_max_size); t_buffer.resize(new_max_size); } - ~OutputStore() { } void reset_buffer(const int i) { buffer_start = i; left_size = right_size = 0; @@ -88,19 +74,27 @@ struct PathPoint { } }; +// if the end of query/target is within this or less, extend the whole distance +#define SEGMENT_BORDER 100 + class DiffRunningData { public: - const int segment_size; - Alignment align; - OutputStore result; - std::vector DynQ, DynT; - std::vector d_path; - std::vector d_path_index; - std::vector aln_path; + const int segment_size; // 500 is "small", 1000 is "large" + // in Align(), k_offset = extend_size * 4 * error_rate; error_rate is .15 or .2, + // extend_size can be as high as segment_size + SEGMENT_BORDER + Alignment align; // can be twice k_offset + OutputStore result; // can be twice max read size + std::vector DynQ, DynT; // can be twice k_offset + std::vector d_path; // can be 1/2 k_offset^2 + std::vector d_path_index; // can be k_offset + std::vector aln_path; // can be twice k_offset public: - // work on more accurate sizing of these buffers - explicit DiffRunningData(const SW_Parameters& swp) : segment_size(swp.segment_size), align((swp.segment_size + 100) * 2), result(MAX_SEQ_SIZE * 2), DynQ(swp.row_size), DynT(swp.column_size), d_path(swp.d_path_size), d_path_index(swp.segment_size * 2), aln_path(swp.segment_size * 4) { } + explicit DiffRunningData() : segment_size(500), align((segment_size + SEGMENT_BORDER) * 2), DynQ((segment_size + SEGMENT_BORDER) * 2), DynT((segment_size + SEGMENT_BORDER) * 2), d_path((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER) / 2), d_path_index(segment_size + SEGMENT_BORDER), aln_path((segment_size + SEGMENT_BORDER) * 2) { } ~DiffRunningData() { } + // can't figure out how to pass this in on initialization, short of a global + void set_size(const idx_t max_read_size) { + result.resize(max_read_size * 2); + } }; class M5Record { diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 4b5f237..99d5660 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -20,9 +20,7 @@ class PackedDB { public: explicit PackedDB() : pac(0), db_size(0), max_db_size(0) { } ~PackedDB() { - if (pac) { - delete[] pac; - } + delete[] pac; } // returns number of reads static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); @@ -54,6 +52,20 @@ class PackedDB { idx_t read_size(const idx_t read_id) const { return seq_idx[read_id].size; } + idx_t max_read_size() const { + if (seq_idx.empty()) { + return 0; + } + std::vector::const_iterator a(seq_idx.begin()); + const std::vector::const_iterator end_a(seq_idx.end()); + idx_t max_size(a->size); + for (++a; a != end_a; ++a) { + if (max_size < a->size) { + max_size = a->size; + } + } + return max_size; + } private: static void set_char(std::vector& p, const idx_t idx, const u1_t c) { p[idx >> 2] |= c << ((~idx & 3) << 1); @@ -67,10 +79,7 @@ class PackedDB { } void add_one_seq(const Sequence& seq); void destroy() { - if (pac) { - delete[] pac; - pac = NULL; - } + delete[] pac; seq_idx.clear(); max_db_size = db_size = 0; } @@ -78,7 +87,7 @@ class PackedDB { u1_t* pac; idx_t db_size; idx_t max_db_size; - PODArray seq_idx; + std::vector seq_idx; std::ifstream pstream; }; diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 8f9019a..a63845a 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -4,7 +4,7 @@ #include // vector<> #include -#include "dw.h" // DiffRunningData, M5Record, get_parameters_small() +#include "dw.h" // DiffRunningData, M5Record #include "packed_db.h" #include "options.h" @@ -155,12 +155,15 @@ class ConsensusPerThreadData { std::vector cns_results; std::string query, target, qaln, saln; public: - ConsensusPerThreadData() : drd(DiffRunningData(get_sw_parameters_small())) { + ConsensusPerThreadData() { // we'll definitely be seeing at least this much use, // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); } ~ConsensusPerThreadData() { } + void set_size(const idx_t max_read_size) { + drd.set_size(max_read_size); + } }; class ConsensusThreadData { @@ -183,6 +186,10 @@ class ConsensusThreadData { ckpt_file_tmp_ = ckpt_file_ + ".tmp"; pthread_mutex_init(&out_lock, NULL); pthread_mutex_init(&id_lock_, NULL); + const idx_t max_read_size(reads.max_read_size()); + for (int i(0); i != rco.num_threads; ++i) { + data[i].set_size(max_read_size); + } } ~ConsensusThreadData() { delete[] data; From 36cf1e7d6fb642fe922b7f475a540c670441d616 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Sun, 5 May 2019 09:52:58 -0500 Subject: [PATCH 41/56] added exit files to allow detection of failed execution of sub-tasks --- src/mecat2cns/dw.h | 7 ++++-- src/mecat2cns/main.cpp | 46 ++++++++++++++++++++++--------------- src/mecat2cns/packed_db.cpp | 19 +++++++++------ src/mecat2cns/packed_db.h | 22 +++++------------- 4 files changed, 51 insertions(+), 43 deletions(-) diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 4e28c9c..d4efe90 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -77,9 +77,12 @@ struct PathPoint { // if the end of query/target is within this or less, extend the whole distance #define SEGMENT_BORDER 100 +// use static sizing here rather than dynamic, as dynamic is 50% slower in practice; +// that said, reserving the space instead of initializing it may be viable + class DiffRunningData { public: - const int segment_size; // 500 is "small", 1000 is "large" + const int segment_size; // 500 is "small", 1000 is "large" // in Align(), k_offset = extend_size * 4 * error_rate; error_rate is .15 or .2, // extend_size can be as high as segment_size + SEGMENT_BORDER Alignment align; // can be twice k_offset @@ -89,7 +92,7 @@ class DiffRunningData { std::vector d_path_index; // can be k_offset std::vector aln_path; // can be twice k_offset public: - explicit DiffRunningData() : segment_size(500), align((segment_size + SEGMENT_BORDER) * 2), DynQ((segment_size + SEGMENT_BORDER) * 2), DynT((segment_size + SEGMENT_BORDER) * 2), d_path((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER) / 2), d_path_index(segment_size + SEGMENT_BORDER), aln_path((segment_size + SEGMENT_BORDER) * 2) { } + explicit DiffRunningData() : segment_size(500), align((segment_size + SEGMENT_BORDER) * 2), DynQ((segment_size + SEGMENT_BORDER) * 2), DynT((segment_size + SEGMENT_BORDER) * 2), d_path((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER + 1) / 2), d_path_index(segment_size + SEGMENT_BORDER), aln_path((segment_size + SEGMENT_BORDER) * 2) { } ~DiffRunningData() { } // can't figure out how to pass this in on initialization, short of a global void set_size(const idx_t max_read_size) { diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 90d883c..db96b4e 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -5,15 +5,17 @@ #include "packed_db.h" // PackedDB #include // S_IRUSR, S_IXUSR -#include -#include +#include // list<> +#include // ostringstream #include // string #include // chmod() #include // ... unlink() #include // pair<> -#include +#include // vector<> -static void grid_start(const char* const prog, const ReadsCorrectionOptions &options, const int i) { +static void grid_start(const char* const prog, const ReadsCorrectionOptions &options, const int i, const std::string& exit_file) { + // make sure exit marker is not present + unlink(exit_file.c_str()); // create grid script, have grid run it ReadsCorrectionOptions new_options(options); new_options.job_index = i; @@ -33,7 +35,7 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt unlink(script_file.c_str()); std::ofstream out; open_fstream(out, script_file.c_str(), std::ios::out); - out << "#!/bin/bash\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; + out << "#!/bin/bash\nset -e\ntrap 'touch " << exit_file << "' EXIT\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; if (!out) { std::cerr << "Error writing to " << script_file << "\n"; exit(1); @@ -45,16 +47,21 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt assert(system(cmd.c_str()) == 0); } -// pass by value so we can modify list to easily avoid checking previously -// found results files -static void wait_for_files(std::list results) { - const std::list::const_iterator end_a(results.end()); - while (!results.empty()) { +// exit files get modified during loop + +static void wait_for_files(std::list& exit_files) { + const std::list::const_iterator end_a(exit_files.end()); + while (!exit_files.empty()) { sleep(60); - std::list::iterator a(results.begin()); + std::list::iterator a(exit_files.begin()); while (a != end_a) { if (access(a->c_str(), F_OK) == 0) { - a = results.erase(a); + // now test to see if it worked, by removed ".exit" from end + a->resize(a->size() - 5); + if (access(a->c_str(), F_OK) != 0) { // failed! + ERROR("Failed: %s does not exist", a->c_str()); + } + a = exit_files.erase(a); } else { ++a; } @@ -101,9 +108,9 @@ int main(int argc, char** argv) { return 0; } else if (access("partition.done", F_OK) == 0) { // split already done } else if (rco.grid_options || rco.grid_options_split) { - grid_start(argv[0], rco, -1); + grid_start(argv[0], rco, -1, "partition.done.exit"); std::list partition_results; - partition_results.push_back("partition.done"); + partition_results.push_back("partition.done.exit"); wait_for_files(partition_results); } else { if (rco.input_type == INPUT_TYPE_CAN) { @@ -135,19 +142,22 @@ int main(int argc, char** argv) { generate_partition_index_file_name(rco.m4, idx_file_name); std::vector partition_file_vec; load_partition_files_info(idx_file_name.c_str(), partition_file_vec); - std::list results; + std::list exit_files, results; for (size_t i(0); i != partition_file_vec.size(); ++i) { std::ostringstream os; os << rco.corrected_reads << "." << i; - results.push_back(os.str()); + const std::string done_file(os.str()); + const std::string exit_file(os.str() + ".exit"); + results.push_back(done_file); + exit_files.push_back(exit_file); if (access(os.str().c_str(), F_OK) != 0) { - grid_start(argv[0], rco, i); + grid_start(argv[0], rco, i, exit_file); if (rco.grid_start_delay) { sleep(rco.grid_start_delay); } } } - wait_for_files(results); + wait_for_files(exit_files); merge_results(rco.corrected_reads, results); } return 0; diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 9aa1767..d4dbebe 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -13,12 +13,11 @@ #include "../common/alignment.h" void PackedDB::add_one_seq(const Sequence& seq) { - SeqIndex si; - si.file_offset = -1; - si.memory_offset = db_size; - si.size = seq.size(); - seq_idx.push_back(si); - const idx_t needed_size(db_size + si.size); + seq_idx.push_back(SeqIndex(-1, db_size, seq.size())); + if (max_read_size_ < seq.size()) { + max_read_size_ = seq.size(); + } + const idx_t needed_size(db_size + seq.size()); if (max_db_size < needed_size) { idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } @@ -31,7 +30,7 @@ void PackedDB::add_one_seq(const Sequence& seq) { const Sequence::str_t& org_seq(seq.sequence()); const uint1* const table(get_dna_encode_table()); unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < si.size; ++i, ++db_size) { + for (idx_t i(0); i < seq.size(); ++i, ++db_size) { const uint1 c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); } @@ -200,6 +199,9 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { while (index >> si.file_offset >> si.size) { si.memory_offset = si.file_offset * 4; seq_idx.push_back(si); + if (max_read_size_ < si.size) { + max_read_size_ = si.size; + } } close_fstream(index); if (!pstream.read((char*)pac, max_db_size)) { @@ -210,6 +212,9 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { si.memory_offset = -1; while (index >> si.file_offset >> si.size) { seq_idx.push_back(si); + if (max_read_size_ < si.size) { + max_read_size_ = si.size; + } } close_fstream(index); } diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 99d5660..d60a7c9 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -9,16 +9,18 @@ #include "../common/defs.h" #include "../common/sequence.h" // Sequence -#include "../common/alignment.h" // ExtensionCandidate +#include "../common/alignment.h" // ExtensionCandidateCompressed class PackedDB { private: struct SeqIndex { off_t file_offset; idx_t memory_offset, size; + explicit SeqIndex() { } + explicit SeqIndex(const off_t i, const idx_t j, const idx_t k) : file_offset(i), memory_offset(j), size(k) { } }; public: - explicit PackedDB() : pac(0), db_size(0), max_db_size(0) { } + explicit PackedDB() : pac(0), db_size(0), max_db_size(0), max_read_size_(0) { } ~PackedDB() { delete[] pac; } @@ -53,18 +55,7 @@ class PackedDB { return seq_idx[read_id].size; } idx_t max_read_size() const { - if (seq_idx.empty()) { - return 0; - } - std::vector::const_iterator a(seq_idx.begin()); - const std::vector::const_iterator end_a(seq_idx.end()); - idx_t max_size(a->size); - for (++a; a != end_a; ++a) { - if (max_size < a->size) { - max_size = a->size; - } - } - return max_size; + return max_read_size_; } private: static void set_char(std::vector& p, const idx_t idx, const u1_t c) { @@ -85,8 +76,7 @@ class PackedDB { } private: u1_t* pac; - idx_t db_size; - idx_t max_db_size; + idx_t db_size, max_db_size, max_read_size_; std::vector seq_idx; std::ifstream pstream; }; From bcd394e3034bf2f6f64d73af96d61d5c0b40e292 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 7 May 2019 13:44:34 -0500 Subject: [PATCH 42/56] general cleanup, moved error_rate to options though it's not currently settable --- src/mecat2cns/dw.cpp | 94 +++++++++++++++------------- src/mecat2cns/dw.h | 23 ++++--- src/mecat2cns/mecat_correction.cpp | 12 ++-- src/mecat2cns/options.cpp | 4 ++ src/mecat2cns/options.h | 1 + src/mecat2cns/reads_correction_aux.h | 6 +- 6 files changed, 84 insertions(+), 56 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 519243d..b513025 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -17,63 +17,73 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } d_path_aux = &d_path[d_path_index[d].d_offset + (d_path_aux->pre_k - d_path_index[d].min_k) / 2]; } + // walk backwards along align path to fill in sequence with gaps align.reset(); int current_x(aln_path[aln_idx].x); int current_y(aln_path[aln_idx].y); - const char* const query_p(query.data() + q_offset + (extend_forward ? 0 : 1)); - const char* const target_p(target.data() + t_offset + (extend_forward ? 0 : 1)); - for (--aln_idx; aln_idx != -1; --aln_idx) { - const int new_x(aln_path[aln_idx].x); - const int new_y(aln_path[aln_idx].y); - const int dx(new_x - current_x); - const int dy(new_y - current_y); - if (dx && dy) { - // apparently, dx always equals dy in this case - if (extend_forward) { + if (extend_forward) { + const char* const query_p(query.data() + q_offset); + const char* const target_p(target.data() + t_offset); + for (--aln_idx; aln_idx != -1; --aln_idx) { + const int new_x(aln_path[aln_idx].x); + const int new_y(aln_path[aln_idx].y); + const int dx(new_x - current_x); + const int dy(new_y - current_y); + if (dx && dy) { // apparently, dx always equals dy in this case std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); align.size += dx; - } else { + } else if (dx) { + std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); + std::fill(&align.t_aln_str[align.size], &align.t_aln_str[align.size] + dx, GAP_ALN); + align.size += dx; + } else if (dy) { + std::fill(&align.q_aln_str[align.size], &align.q_aln_str[align.size] + dy, GAP_ALN); + std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); + align.size += dy; + } + current_x = new_x; + current_y = new_y; + } + } else { + const char* const query_p(query.data() + q_offset + 1); + const char* const target_p(target.data() + t_offset + 1); + for (--aln_idx; aln_idx != -1; --aln_idx) { + const int new_x(aln_path[aln_idx].x); + const int new_y(aln_path[aln_idx].y); + const int dx(new_x - current_x); + const int dy(new_y - current_y); + if (dx && dy) { // apparently, dx always equals dy in this case align.size += dx; const int offset(align.q_aln_str.size() - align.size); std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); + } else if (dx) { + align.size += dx; + const int offset(align.q_aln_str.size() - align.size); + std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); + std::fill(&align.t_aln_str[offset], &align.t_aln_str[offset] + dx, GAP_ALN); + } else if (dy) { + align.size += dy; + const int offset(align.q_aln_str.size() - align.size); + std::fill(&align.q_aln_str[offset], &align.q_aln_str[offset] + dy, GAP_ALN); + std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); } - } else if (dx) { - if (extend_forward) { - std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); - std::fill(&align.t_aln_str[align.size], &align.t_aln_str[align.size + dx], GAP_ALN); - } else { - std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[align.q_aln_str.size() - align.size - dx]); - std::fill(&align.t_aln_str[0] + align.t_aln_str.size() - align.size - dx, &align.t_aln_str[0] + align.t_aln_str.size() - align.size, GAP_ALN); - } - align.size += dx; - } else if (dy) { - if (extend_forward) { - std::fill(&align.q_aln_str[align.size], &align.q_aln_str[align.size + dy], GAP_ALN); - std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); - } else { - std::fill(&align.q_aln_str[0] + align.q_aln_str.size() - align.size - dy, &align.q_aln_str[0] + align.q_aln_str.size() - align.size, GAP_ALN); - std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[align.t_aln_str.size() - align.size - dy]); - } - align.size += dy; + current_x = new_x; + current_y = new_y; } - current_x = new_x; - current_y = new_y; } } static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(extend_size * 4 * error_rate); - U.assign(k_offset * 2, 0); - V.assign(k_offset * 2, 0); const int band_tolerance(extend_size / 10 * 3 + 1); const int max_band_size(band_tolerance * 2 - 1); int d_path_idx(0), best_m(-1), min_k(0), max_k(0); + V[k_offset + 1] = 0; for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries d_path_index[d].set(d_path_idx, min_k); - // k is the offset between query and target for (int k(min_k); k <= max_k; k += 2) { int x, pre_k; if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { @@ -184,8 +194,8 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co if (align.aln_q_e == q_bps) { // no good match return; } - // don't extend into the good match; keeping good match for - // next alignment, maybe? + // don't extend into the good match; keeping + // good match for next alignment, maybe? q_extend += align.aln_q_e - q_bps; t_extend += align.aln_t_e - t_bps; } else if (align.aln_q_e == 0) { // no good match @@ -196,15 +206,15 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co k = align.q_aln_str.size() - align.size; } if (extend_forward) { - const int i(result.buffer_start + result.right_size); - std::copy(&align.q_aln_str[0], &align.q_aln_str[0] + k, &result.q_buffer[i]); - std::copy(&align.t_aln_str[0], &align.t_aln_str[0] + k, &result.t_buffer[i]); + const int offset(result.buffer_start + result.right_size); + std::copy(&align.q_aln_str[0], &align.q_aln_str[0] + k, &result.q_buffer[offset]); + std::copy(&align.t_aln_str[0], &align.t_aln_str[0] + k, &result.t_buffer[offset]); result.right_size += k; } else { result.left_size += align.q_aln_str.size() - k; - const int i(result.buffer_start - result.left_size); - std::copy(&align.q_aln_str[k], &align.q_aln_str[0] + align.q_aln_str.size(), &result.q_buffer[i]); - std::copy(&align.t_aln_str[k], &align.t_aln_str[0] + align.q_aln_str.size(), &result.t_buffer[i]); + const int offset(result.buffer_start - result.left_size); + std::copy(&align.q_aln_str[k], &align.q_aln_str[0] + align.q_aln_str.size(), &result.q_buffer[offset]); + std::copy(&align.t_aln_str[k], &align.t_aln_str[0] + align.q_aln_str.size(), &result.t_buffer[offset]); } } while (not_at_end); } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index d4efe90..ab224cd 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -1,9 +1,10 @@ #ifndef DW_H #define DW_H +#include "../common/defs.h" // idx_t +#include // ceil() #include // string #include // vector<> -#include "../common/defs.h" // idx_t class Alignment { public: @@ -11,11 +12,12 @@ class Alignment { int aln_q_e, aln_t_e, size; std::vector q_aln_str, t_aln_str; public: - explicit Alignment(const size_t new_max_size) { + explicit Alignment() { } + ~Alignment() { } + void resize(const size_t new_max_size) { q_aln_str.resize(new_max_size); t_aln_str.resize(new_max_size); } - ~Alignment() { } void reset() { size = 0; } @@ -83,7 +85,7 @@ struct PathPoint { class DiffRunningData { public: const int segment_size; // 500 is "small", 1000 is "large" - // in Align(), k_offset = extend_size * 4 * error_rate; error_rate is .15 or .2, + // in Align(), k_offset = extend_size * 4 * error_rate, // extend_size can be as high as segment_size + SEGMENT_BORDER Alignment align; // can be twice k_offset OutputStore result; // can be twice max read size @@ -92,11 +94,18 @@ class DiffRunningData { std::vector d_path_index; // can be k_offset std::vector aln_path; // can be twice k_offset public: - explicit DiffRunningData() : segment_size(500), align((segment_size + SEGMENT_BORDER) * 2), DynQ((segment_size + SEGMENT_BORDER) * 2), DynT((segment_size + SEGMENT_BORDER) * 2), d_path((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER + 1) / 2), d_path_index(segment_size + SEGMENT_BORDER), aln_path((segment_size + SEGMENT_BORDER) * 2) { } + explicit DiffRunningData() : segment_size(500) { } ~DiffRunningData() { } - // can't figure out how to pass this in on initialization, short of a global - void set_size(const idx_t max_read_size) { + // can't figure out how to pass this in on initialization + void set_size(const double error_rate, const idx_t max_read_size) { + const size_t max_size(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)); + align.resize(max_size * 2); result.resize(max_read_size * 2); + DynQ.resize(max_size * 2); + DynT.resize(max_size * 2); + d_path.resize(max_size * (max_size + 1) / 2); + d_path_index.resize(max_size); + aln_path.resize(max_size * 2); } }; diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index db559f6..c39e862 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -313,6 +313,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); + const double error_rate(ctd.rco.error_rate); const int max_added(60); std::vector& cns_table(pctd.cns_table); cns_table.assign(read_size, CnsTableItem()); // reset table @@ -327,7 +328,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); const idx_t sext(ovlp.sext); - const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size)); if (r) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); @@ -356,6 +357,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData std::string& nqstr = pctd.qaln; std::string& ntstr = pctd.saln; const int min_align_size = ctd.rco.min_align_size; + const double error_rate(ctd.rco.error_rate); const double min_mapping_ratio = ctd.rco.min_mapping_ratio - 0.02; idx_t L, R; @@ -381,7 +383,7 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData idx_t qext = ovlp.qext; idx_t sext = ovlp.sext; if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; - bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size); + bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, ovlp.qsize, m5.soff, m5.send, ovlp.ssize, min_mapping_ratio)) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); @@ -435,6 +437,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); + const double error_rate(ctd.rco.error_rate); const double min_mapping_ratio(ctd.rco.min_mapping_ratio - 0.02); int num_added(0); const int max_added(60); @@ -453,7 +456,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, 0.15, min_align_size)); + const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; @@ -488,6 +491,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea std::string& nqstr(pctd.qaln); std::string& ntstr(pctd.saln); const int min_align_size(ctd.rco.min_align_size); + const double error_rate(ctd.rco.error_rate); const double min_mapping_ratio(ctd.rco.min_mapping_ratio - 0.02); int num_added(0); int num_ext(0); @@ -508,7 +512,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, 0.20, min_align_size)); + const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index acc4475..d003aed 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -15,6 +15,7 @@ static int cov_pacbio = 4; static int min_size_pacbio = 2000; static bool print_usage_pacbio = false; static int tech_pacbio = TECH_PACBIO; +static double error_rate_pacbio = .15; static int input_type_nanopore = 1; static int num_threads_nanopore = 1; @@ -24,6 +25,7 @@ static int cov_nanopore = 6; static int min_size_nanopore = 2000; static bool print_usage_nanopore = false; static int tech_nanopore = TECH_NANOPORE; +static double error_rate_nanopore = .2; static int default_tech = TECH_PACBIO; static int num_partition_files = 0; @@ -197,6 +199,7 @@ ConsensusOptions init_consensus_options(const int tech) { t.min_size = min_size_pacbio; t.print_usage_info = print_usage_pacbio; t.tech = tech_pacbio; + t.error_rate = error_rate_pacbio; } else { t.input_type = input_type_nanopore; t.num_threads = num_threads_nanopore; @@ -206,6 +209,7 @@ ConsensusOptions init_consensus_options(const int tech) { t.min_size = min_size_nanopore; t.print_usage_info = print_usage_nanopore; t.tech = tech_nanopore; + t.error_rate = error_rate_nanopore; } return t; } diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index f17a3df..e593bf7 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -29,6 +29,7 @@ struct ConsensusOptions int grid_start_delay; int full_reads; idx_t read_buffer_size; + double error_rate; // .15 for pacbio, .2 for nanopore }; void diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index a63845a..94a7f99 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -161,8 +161,8 @@ class ConsensusPerThreadData { cns_results.reserve(MAX_CNS_RESULTS); } ~ConsensusPerThreadData() { } - void set_size(const idx_t max_read_size) { - drd.set_size(max_read_size); + void set_size(const double error_rate, const idx_t max_read_size) { + drd.set_size(error_rate, max_read_size); } }; @@ -188,7 +188,7 @@ class ConsensusThreadData { pthread_mutex_init(&id_lock_, NULL); const idx_t max_read_size(reads.max_read_size()); for (int i(0); i != rco.num_threads; ++i) { - data[i].set_size(max_read_size); + data[i].set_size(rco.error_rate, max_read_size); } } ~ConsensusThreadData() { From c2aa22a583bae5e8a26d448baef1d09c5df5e53a Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 7 May 2019 14:31:25 -0500 Subject: [PATCH 43/56] cleaned up Align() a bit renamed some variables, made end of band calculations a bit quicker --- src/mecat2cns/dw.cpp | 60 ++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index b513025..bb83cc1 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -75,62 +75,52 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& U, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& combined_match_length, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(extend_size * 4 * error_rate); - const int band_tolerance(extend_size / 10 * 3 + 1); - const int max_band_size(band_tolerance * 2 - 1); - int d_path_idx(0), best_m(-1), min_k(0), max_k(0); - V[k_offset + 1] = 0; - for (int d(0); d < k_offset && max_k - min_k < max_band_size; ++d) { + const int band_tolerance(extend_size / 10 * 3); + const int max_band_size(band_tolerance * 2 + 1); + int d_path_idx(0), best_combined_match_length(0), min_k(0), max_k(0); + V[k_offset + 1] = 0; // initialize starting point + for (int d(0); d != k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries d_path_index[d].set(d_path_idx, min_k); for (int k(min_k); k <= max_k; k += 2) { - int x, pre_k; + int q_pos, pre_k; if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { pre_k = k + 1; - x = V[k_offset + k + 1]; + q_pos = V[k_offset + k + 1]; } else { pre_k = k - 1; - x = V[k_offset + k - 1] + 1; + q_pos = V[k_offset + k - 1] + 1; } - int y(x - k); + int t_pos(q_pos - k); // start of exact match - const int x1(x), y1(y); + const int q_start(q_pos), t_start(t_pos); // find the other end of exact match if (extend_forward) { - for (; x < extend_size && y < extend_size && query[q_offset + x] == target[t_offset + y]; ++x, ++y) { } + for (; q_pos < extend_size && t_pos < extend_size && query[q_offset + q_pos] == target[t_offset + t_pos]; ++q_pos, ++t_pos) { } } else { - for (; x < extend_size && y < extend_size && query[q_offset - x] == target[t_offset - y]; ++x, ++y) { } + for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } } - d_path[d_path_idx].set(x1, y1, x, y, pre_k); + d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); // see if we got as much as we can - if (x == extend_size || y == extend_size) { + if (q_pos == extend_size || t_pos == extend_size) { fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[d_path_idx], d_path_index, d, aln_path, extend_forward); return 1; } ++d_path_idx; - V[k_offset + k] = x; - U[k_offset + k] = x + y; - if (best_m < x + y) { - best_m = x + y; + V[k_offset + k] = q_pos; + combined_match_length[k_offset + k] = q_pos + t_pos; + if (best_combined_match_length < q_pos + t_pos) { + best_combined_match_length = q_pos + t_pos; } } - // for banding - int new_min_k(max_k); - int new_max_k(min_k); - const int min_u(best_m - band_tolerance); - for (int k(min_k); k <= max_k; k += 2) { - if (min_u < U[k_offset + k]) { - if (new_min_k > k) { - new_min_k = k; - } - if (new_max_k < k) { - new_max_k = k; - } - } - } - min_k = new_min_k - 1; - max_k = new_max_k + 1; + // shift ends to one outside "good" band + const int cutoff(best_combined_match_length - band_tolerance); + for (; combined_match_length[k_offset + min_k] < cutoff; min_k += 2) { } + --min_k; + for (; combined_match_length[k_offset + max_k] < cutoff; max_k -= 2) { } + ++max_k; } return 0; } From da5b08c5e71eb120be76f42386c1d7724899f180 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 8 May 2019 19:46:15 -0500 Subject: [PATCH 44/56] minor pre-alloc fix use actual error_rate, not .25, and correct align size, which should be based directly off the extend size, not k_offset --- src/mecat2cns/dw.cpp | 12 +++++------ src/mecat2cns/dw.h | 49 ++++++++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index bb83cc1..a367da1 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -75,23 +75,23 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } } -static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& V, std::vector& combined_match_length, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { +static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& q_extent, std::vector& combined_match_length, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { const int k_offset(extend_size * 4 * error_rate); const int band_tolerance(extend_size / 10 * 3); const int max_band_size(band_tolerance * 2 + 1); int d_path_idx(0), best_combined_match_length(0), min_k(0), max_k(0); - V[k_offset + 1] = 0; // initialize starting point + q_extent[k_offset + 1] = 0; // initialize starting point for (int d(0); d != k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries d_path_index[d].set(d_path_idx, min_k); for (int k(min_k); k <= max_k; k += 2) { int q_pos, pre_k; - if (k == min_k || (k != max_k && V[k_offset + k - 1] < V[k_offset + k + 1])) { + if (k == min_k || (k != max_k && q_extent[k_offset + k - 1] < q_extent[k_offset + k + 1])) { pre_k = k + 1; - q_pos = V[k_offset + k + 1]; + q_pos = q_extent[k_offset + k + 1]; } else { pre_k = k - 1; - q_pos = V[k_offset + k - 1] + 1; + q_pos = q_extent[k_offset + k - 1] + 1; } int t_pos(q_pos - k); // start of exact match @@ -109,7 +109,7 @@ static int Align(const int extend_size, const std::string& query, const int q_of return 1; } ++d_path_idx; - V[k_offset + k] = q_pos; + q_extent[k_offset + k] = q_pos; combined_match_length[k_offset + k] = q_pos + t_pos; if (best_combined_match_length < q_pos + t_pos) { best_combined_match_length = q_pos + t_pos; diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index ab224cd..49a5a54 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -14,9 +14,9 @@ class Alignment { public: explicit Alignment() { } ~Alignment() { } - void resize(const size_t new_max_size) { - q_aln_str.resize(new_max_size); - t_aln_str.resize(new_max_size); + void resize(const size_t max_size) { + q_aln_str.resize(max_size); + t_aln_str.resize(max_size); } void reset() { size = 0; @@ -35,9 +35,9 @@ class OutputStore { explicit OutputStore() { } ~OutputStore() { } // can't figure out how to pass this in on initialization - void resize(const size_t new_max_size) { - q_buffer.resize(new_max_size); - t_buffer.resize(new_max_size); + void resize(const size_t max_size) { + q_buffer.resize(max_size); + t_buffer.resize(max_size); } void reset_buffer(const int i) { buffer_start = i; @@ -84,28 +84,29 @@ struct PathPoint { class DiffRunningData { public: - const int segment_size; // 500 is "small", 1000 is "large" - // in Align(), k_offset = extend_size * 4 * error_rate, - // extend_size can be as high as segment_size + SEGMENT_BORDER - Alignment align; // can be twice k_offset - OutputStore result; // can be twice max read size - std::vector DynQ, DynT; // can be twice k_offset - std::vector d_path; // can be 1/2 k_offset^2 - std::vector d_path_index; // can be k_offset - std::vector aln_path; // can be twice k_offset + static const int segment_size = 500; // 500 is "small", 1000 is "large" + Alignment align; + OutputStore result; + std::vector DynQ, DynT; + std::vector d_path; + std::vector d_path_index; + std::vector aln_path; public: - explicit DiffRunningData() : segment_size(500) { } + explicit DiffRunningData() { } ~DiffRunningData() { } - // can't figure out how to pass this in on initialization + // can't figure out how to pass these in on initialization void set_size(const double error_rate, const idx_t max_read_size) { - const size_t max_size(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)); - align.resize(max_size * 2); + const size_t max_extend_size(segment_size + SEGMENT_BORDER); + // in Align(), k_offset = extend_size * 4 * error_rate, + const size_t max_k_offset(ceil(max_extend_size * 4 * error_rate)); + // allocate largest first (approximately) + d_path.resize(max_k_offset * (max_k_offset + 1) / 2); result.resize(max_read_size * 2); - DynQ.resize(max_size * 2); - DynT.resize(max_size * 2); - d_path.resize(max_size * (max_size + 1) / 2); - d_path_index.resize(max_size); - aln_path.resize(max_size * 2); + aln_path.resize(max_k_offset * 4); + align.resize(max_extend_size * 2); + DynQ.resize(max_k_offset * 2); + DynT.resize(max_k_offset * 2); + d_path_index.resize(max_k_offset); } }; From d53721f80cd4952e1bc39be535a4571ba1973691 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 9 May 2019 14:46:33 -0500 Subject: [PATCH 45/56] char -> uint1, pthread mutex -> c++11 changed a few vector to vectir when they just held values from 0-4; changed pthread mutexes to std::mutex, which requires c++11 --- src/mecat2cns/dw.cpp | 68 +++++++++--------- src/mecat2cns/dw.h | 58 ++++++---------- src/mecat2cns/mecat2cns.mk | 2 +- src/mecat2cns/reads_correction_aux.h | 99 ++++++++++----------------- src/mecat2cns/reads_correction_m4.cpp | 13 ++-- 5 files changed, 101 insertions(+), 139 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index a367da1..9ad7d01 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -18,7 +18,7 @@ static void fill_align(const std::string& query, const int q_offset, const std:: d_path_aux = &d_path[d_path_index[d].d_offset + (d_path_aux->pre_k - d_path_index[d].min_k) / 2]; } // walk backwards along align path to fill in sequence with gaps - align.reset(); + align.clear(); int current_x(aln_path[aln_idx].x); int current_y(aln_path[aln_idx].y); if (extend_forward) { @@ -30,17 +30,17 @@ static void fill_align(const std::string& query, const int q_offset, const std:: const int dx(new_x - current_x); const int dy(new_y - current_y); if (dx && dy) { // apparently, dx always equals dy in this case - std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); - std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); - align.size += dx; + std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.current_size]); + std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.current_size]); + align.current_size += dx; } else if (dx) { - std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.size]); - std::fill(&align.t_aln_str[align.size], &align.t_aln_str[align.size] + dx, GAP_ALN); - align.size += dx; + std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.current_size]); + std::fill(&align.t_aln_str[align.current_size], &align.t_aln_str[align.current_size] + dx, GAP_ALN); + align.current_size += dx; } else if (dy) { - std::fill(&align.q_aln_str[align.size], &align.q_aln_str[align.size] + dy, GAP_ALN); - std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.size]); - align.size += dy; + std::fill(&align.q_aln_str[align.current_size], &align.q_aln_str[align.current_size] + dy, GAP_ALN); + std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.current_size]); + align.current_size += dy; } current_x = new_x; current_y = new_y; @@ -54,18 +54,18 @@ static void fill_align(const std::string& query, const int q_offset, const std:: const int dx(new_x - current_x); const int dy(new_y - current_y); if (dx && dy) { // apparently, dx always equals dy in this case - align.size += dx; - const int offset(align.q_aln_str.size() - align.size); + align.current_size += dx; + const int offset(align.q_aln_str.size() - align.current_size); std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); } else if (dx) { - align.size += dx; - const int offset(align.q_aln_str.size() - align.size); + align.current_size += dx; + const int offset(align.q_aln_str.size() - align.current_size); std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); std::fill(&align.t_aln_str[offset], &align.t_aln_str[offset] + dx, GAP_ALN); } else if (dy) { - align.size += dy; - const int offset(align.q_aln_str.size() - align.size); + align.current_size += dy; + const int offset(align.q_aln_str.size() - align.current_size); std::fill(&align.q_aln_str[offset], &align.q_aln_str[offset] + dy, GAP_ALN); std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); } @@ -145,9 +145,9 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co // matching basepairs), counting non-gap basepairs int q_bps(0), t_bps(0), num_matches(0); if (extend_forward) { - for (k = align.size - 1; k != -1; --k) { - const char qc(align.q_aln_str[k]); - const char tc(align.t_aln_str[k]); + for (k = align.current_size - 1; k != -1; --k) { + const uint1 qc(align.q_aln_str[k]); + const uint1 tc(align.t_aln_str[k]); if (qc != tc) { num_matches = 0; if (qc == GAP_ALN) { @@ -159,13 +159,13 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co break; } } - q_bps += align.size - k; - t_bps += align.size - k; + q_bps += align.current_size - k; + t_bps += align.current_size - k; } else { - const int offset(align.q_aln_str.size() - align.size); + const int offset(align.q_aln_str.size() - align.current_size); for (k = offset; k != static_cast(align.q_aln_str.size()); ++k) { - const char qc(align.q_aln_str[k]); - const char tc(align.t_aln_str[k]); + const uint1 qc(align.q_aln_str[k]); + const uint1 tc(align.t_aln_str[k]); if (qc != tc) { num_matches = 0; if (qc == GAP_ALN) { @@ -191,9 +191,9 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } else if (align.aln_q_e == 0) { // no good match return; } else if (extend_forward) { - k = align.size; + k = align.current_size; } else { - k = align.q_aln_str.size() - align.size; + k = align.q_aln_str.size() - align.current_size; } if (extend_forward) { const int offset(result.buffer_start + result.right_size); @@ -209,7 +209,7 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } while (not_at_end); } -static int gap_count(const std::vector& buffer, int i, const int end_i) { +static int gap_count(const std::vector& buffer, int i, const int end_i) { int j(0); for (; i != end_i; ++i) { if (buffer[i] == GAP_ALN) { @@ -220,7 +220,7 @@ static int gap_count(const std::vector& buffer, int i, const int end_i) { } static int dw(const std::string& query, const int query_start, const std::string& target, const int target_start, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, OutputStore& result, const int segment_size, const double error_rate, const int min_aln_size) { - result.reset_buffer(query_start + target_start); + result.clear(query_start + target_start); // reverse extend (left side) dw_in_one_direction(query, query_start - 1, target, target_start - 1, U, V, align, d_path, d_path_index, aln_path, segment_size, result, 0, error_rate); // forward extend (right side) @@ -237,10 +237,10 @@ static int dw(const std::string& query, const int query_start, const std::string return 1; } -static void decode_sequence(std::string& out_seq, const std::vector& in_seq, const size_t offset, const size_t size) { +static void decode_sequence(std::string& out_seq, const std::vector& in_seq, const size_t offset, const size_t size) { out_seq.resize(size); for (size_t i(0); i != size; ++i) { - out_seq[i] = "ACGT-"[static_cast(in_seq[offset + i])]; + out_seq[i] = "ACGT-"[in_seq[offset + i]]; } } @@ -259,8 +259,8 @@ int GetAlignment(const std::string& query, const int query_start, const std::str const int end_k(result.buffer_start + result.right_size); int k(start_k); for (; k != end_k; ++k) { - const char qc(result.q_buffer[k]); - const char tc(result.t_buffer[k]); + const uint1 qc(result.q_buffer[k]); + const uint1 tc(result.t_buffer[k]); if (qc != tc) { eit = 0; // we don't count gaps @@ -285,8 +285,8 @@ int GetAlignment(const std::string& query, const int query_start, const std::str int tre(0); // t ending basepair offset to good sequence eit = 0; // still matching run length for (k = end_k - 1;; --k) { - const char qc(result.q_buffer[k]); - const char tc(result.t_buffer[k]); + const uint1 qc(result.q_buffer[k]); + const uint1 tc(result.t_buffer[k]); if (qc != tc) { eit = 0; // we don't count gaps diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 49a5a54..a3b6ef8 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -8,18 +8,17 @@ class Alignment { public: - // size tracks actual buffer use - int aln_q_e, aln_t_e, size; - std::vector q_aln_str, t_aln_str; + // current_size tracks actual buffer use + int aln_q_e, aln_t_e, current_size; + std::vector q_aln_str, t_aln_str; public: - explicit Alignment() { } + explicit Alignment(const size_t max_size) : q_aln_str(max_size), t_aln_str(max_size) { } ~Alignment() { } - void resize(const size_t max_size) { - q_aln_str.resize(max_size); - t_aln_str.resize(max_size); + size_t size() const { + return q_aln_str.size(); } - void reset() { - size = 0; + void clear() { + current_size = 0; } }; @@ -30,16 +29,14 @@ class OutputStore { int buffer_start, left_size, right_size; int query_start, query_end; int target_start, target_end; - std::vector q_buffer, t_buffer; + std::vector q_buffer, t_buffer; public: - explicit OutputStore() { } + explicit OutputStore(const size_t max_size) : q_buffer(max_size), t_buffer(max_size) { } ~OutputStore() { } - // can't figure out how to pass this in on initialization - void resize(const size_t max_size) { - q_buffer.resize(max_size); - t_buffer.resize(max_size); + size_t size() const { + return q_buffer.size(); } - void reset_buffer(const int i) { + void clear(const int i) { buffer_start = i; left_size = right_size = 0; } @@ -47,8 +44,6 @@ class OutputStore { struct DPathData { int x1, y1, x2, y2, pre_k; - explicit DPathData() { } - explicit DPathData(const int i, const int j, const int k, const int l, const int m) : x1(i), y1(j), x2(k), y2(l), pre_k(m) { } void set(const int i, const int j, const int k, const int l, const int m) { x1 = i; y1 = j; @@ -60,7 +55,6 @@ struct DPathData { struct DPathIndex { int d_offset, min_k; - explicit DPathIndex() { } void set(const int i, const int j) { d_offset = i; min_k = j; @@ -69,7 +63,6 @@ struct DPathIndex { struct PathPoint { int x, y; - explicit PathPoint() { } void set(const int i, const int j) { x = i; y = j; @@ -92,29 +85,22 @@ class DiffRunningData { std::vector d_path_index; std::vector aln_path; public: - explicit DiffRunningData() { } + explicit DiffRunningData(const double error_rate, const idx_t max_read_size) : + align((segment_size + SEGMENT_BORDER) * 2), + result(max_read_size * 2), + DynQ(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), + DynT(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), + d_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), + d_path_index(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)), + aln_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 4) { } ~DiffRunningData() { } - // can't figure out how to pass these in on initialization - void set_size(const double error_rate, const idx_t max_read_size) { - const size_t max_extend_size(segment_size + SEGMENT_BORDER); - // in Align(), k_offset = extend_size * 4 * error_rate, - const size_t max_k_offset(ceil(max_extend_size * 4 * error_rate)); - // allocate largest first (approximately) - d_path.resize(max_k_offset * (max_k_offset + 1) / 2); - result.resize(max_read_size * 2); - aln_path.resize(max_k_offset * 4); - align.resize(max_extend_size * 2); - DynQ.resize(max_k_offset * 2); - DynT.resize(max_k_offset * 2); - d_path_index.resize(max_k_offset); - } }; class M5Record { public: idx_t qoff, qend, soff, send; std::string qaln, saln; - M5Record() { } + explicit M5Record() { } ~M5Record() { } }; diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index fb17ae4..71bc7f5 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,7 +20,7 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 94a7f99..98b592d 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -3,6 +3,7 @@ #include // vector<> #include +#include // lock_guard<>, mutex #include "dw.h" // DiffRunningData, M5Record #include "packed_db.h" @@ -10,9 +11,7 @@ struct CnsTableItem { char base; - uint1 mat_cnt; - uint1 ins_cnt; - uint1 del_cnt; + uint1 mat_cnt, ins_cnt, del_cnt; CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } }; @@ -26,37 +25,39 @@ struct MappingRange { class CnsAln : public MappingRange { public: - explicit CnsAln(const int i, const int j, const std::string& q, const std::string& s) : MappingRange(i, j), aln_idx(0), qaln(s), saln(q) { } + explicit CnsAln(const int i, const int j, const std::string& q, const std::string& s) : MappingRange(i, j), aln_idx_(0), qaln_(s), saln_(q) { } ~CnsAln() { } int retrieve_aln_subseqs(const int sb, const int se, std::string& qstr, std::string& tstr, int& sb_out) { - const int aln_size(saln.size() - 1); - if (se <= start || sb >= end || aln_idx == aln_size) { + const int aln_size(saln_.size() - 1); + if (se <= start || sb >= end || aln_idx_ == aln_size) { return 0; } sb_out = std::max(start, sb); - while (start < sb && aln_idx < aln_size) { - if (saln[++aln_idx] != GAP) { + while (start < sb && aln_idx_ < aln_size) { + if (saln_[++aln_idx_] != GAP) { ++start; } } // should we test for start < sb, and return 0 if so, // rather than returning the last basepair of the alignment? - const int aln_start(aln_idx); - while (start < se && aln_idx < aln_size) { - if (saln[++aln_idx] != GAP) { + const int aln_start(aln_idx_); + while (start < se && aln_idx_ < aln_size) { + if (saln_[++aln_idx_] != GAP) { ++start; } } // this looks like it could return the same basepair twice - // once at the end of a call, once at the start of the next - const int aln_length(aln_idx - aln_start + 1); - qstr.assign(qaln, aln_start, aln_length); - tstr.assign(saln, aln_start, aln_length); + const int aln_length(aln_idx_ - aln_start + 1); + qstr.assign(qaln_, aln_start, aln_length); + tstr.assign(saln_, aln_start, aln_length); return 1; } private: - int aln_idx; - std::string qaln, saln; + int aln_idx_; + // despite never changing, we can't make these const because they get used in + // a vector which uses default construction and copy mechanics + std::string qaln_, saln_; }; class CnsAlns { @@ -80,11 +81,7 @@ class CnsAlns { cns_alns_.push_back(CnsAln(soff, send, qstr, tstr)); } void get_mapping_ranges(std::vector& ranges) const { - ranges.clear(); - ranges.reserve(cns_alns_.size()); - for (size_t i(0); i < cns_alns_.size(); ++i) { - ranges.push_back(cns_alns_[i]); - } + ranges.assign(cns_alns_.begin(), cns_alns_.end()); } private: std::vector cns_alns_; @@ -155,15 +152,12 @@ class ConsensusPerThreadData { std::vector cns_results; std::string query, target, qaln, saln; public: - ConsensusPerThreadData() { + explicit ConsensusPerThreadData(const double error_rate, const idx_t max_read_size) : drd(error_rate, max_read_size) { // we'll definitely be seeing at least this much use, // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); } ~ConsensusPerThreadData() { } - void set_size(const double error_rate, const idx_t max_read_size) { - drd.set_size(error_rate, max_read_size); - } }; class ConsensusThreadData { @@ -171,35 +165,15 @@ class ConsensusThreadData { ReadsCorrectionOptions& rco; PackedDB& reads; std::ostream& out; - ConsensusPerThreadData* data; - pthread_mutex_t out_lock; + std::mutex out_lock; idx_t ec_offset; - // this doesn't work as a vector - all the pointers end up pointing - // to the same values, and eventually it seg faults (possibly a - // compiler optimization bug) + std::vector data; public: - ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), data(new ConsensusPerThreadData[prco.num_threads]), ec_offset(0), last_thread_id_(-1), num_threads_written_(0) { - done_file_ = input_file_name; - done_file_ += ".done"; - ckpt_file_ = input_file_name; - ckpt_file_ += ".ckpt"; - ckpt_file_tmp_ = ckpt_file_ + ".tmp"; - pthread_mutex_init(&out_lock, NULL); - pthread_mutex_init(&id_lock_, NULL); - const idx_t max_read_size(reads.max_read_size()); - for (int i(0); i != rco.num_threads; ++i) { - data[i].set_size(rco.error_rate, max_read_size); - } - } - ~ConsensusThreadData() { - delete[] data; - pthread_mutex_destroy(&out_lock); - pthread_mutex_destroy(&id_lock_); - } + ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), ec_offset(0), data(prco.num_threads, ConsensusPerThreadData(prco.error_rate, r.max_read_size())), last_thread_id_(-1), num_threads_written_(0), done_file_(std::string(input_file_name) + ".done"), ckpt_file_(std::string(input_file_name) + ".ckpt"), ckpt_file_tmp_(ckpt_file_ + ".tmp") { } + ~ConsensusThreadData() { } int get_thread_id() { - pthread_mutex_lock(&id_lock_); + std::lock_guard lock(id_lock_); const int tid(++last_thread_id_); - pthread_mutex_unlock(&id_lock_); return tid; } void reset_threads() { @@ -212,19 +186,20 @@ class ConsensusThreadData { ConsensusPerThreadData& pdata(data[tid]); std::vector::const_iterator a(pdata.cns_results.begin()); const std::vector::const_iterator end_a(pdata.cns_results.end()); - pthread_mutex_lock(&out_lock); - for (; a != end_a; ++a) { - out << ">" << a->id << "_" << a->range[0] << "_" << a->range[1] << "_" << a->seq.size() << "\n" << a->seq << "\n"; - if (!out) { - ERROR("Error writing output"); + { + std::lock_guard lock(out_lock); + for (; a != end_a; ++a) { + out << ">" << a->id << "_" << a->range[0] << "_" << a->range[1] << "_" << a->seq.size() << "\n" << a->seq << "\n"; + if (!out) { + ERROR("Error writing output"); + } + } + pdata.next_candidate = i; + if (++num_threads_written_ >= rco.num_threads) { + checkpoint(); + num_threads_written_ = 0; } } - pdata.next_candidate = i; - if (++num_threads_written_ >= rco.num_threads) { - checkpoint(); - num_threads_written_ = 0; - } - pthread_mutex_unlock(&out_lock); pdata.cns_results.clear(); } int restart(off_t& output_pos) { @@ -276,7 +251,7 @@ class ConsensusThreadData { } } private: - pthread_mutex_t id_lock_; + std::mutex id_lock_; int last_thread_id_, num_threads_written_; std::string done_file_, ckpt_file_, ckpt_file_tmp_; }; diff --git a/src/mecat2cns/reads_correction_m4.cpp b/src/mecat2cns/reads_correction_m4.cpp index b821394..a495f5f 100644 --- a/src/mecat2cns/reads_correction_m4.cpp +++ b/src/mecat2cns/reads_correction_m4.cpp @@ -36,14 +36,15 @@ void* reads_correction_func_m4(void* arg) { consensus_one_read_m4_nanopore(data, pdata, sid, i, j); } if (pdata.cns_results.size() >= MAX_CNS_RESULTS) { - pthread_mutex_lock(&data.out_lock); - for (std::vector::iterator iter = pdata.cns_results.begin(); iter != pdata.cns_results.end(); ++iter) { - data.out << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n" << iter->seq << "\n"; - if (!data.out) { - ERROR("Error writing output"); + { + std::lock_guard lock(data.out_lock); + for (std::vector::iterator iter = pdata.cns_results.begin(); iter != pdata.cns_results.end(); ++iter) { + data.out << ">" << iter->id << "_" << iter->range[0] << "_" << iter->range[1] << "_" << iter->seq.size() << "\n" << iter->seq << "\n"; + if (!data.out) { + ERROR("Error writing output"); + } } } - pthread_mutex_unlock(&data.out_lock); pdata.cns_results.clear(); } i = j; From 9b7c353ac09ddaf8f8afdf1eecb220b1e6f98c8a Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Thu, 9 May 2019 15:01:49 -0500 Subject: [PATCH 46/56] minor cleanup and reformatting --- src/mecat2cns/reads_correction_aux.h | 48 ++++++++++++++++++---------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 98b592d..60a6e29 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -12,7 +12,7 @@ struct CnsTableItem { char base; uint1 mat_cnt, ins_cnt, del_cnt; - CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } + explicit CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } }; #define MAX_CNS_OVLPS 100 @@ -55,14 +55,14 @@ class CnsAln : public MappingRange { } private: int aln_idx_; - // despite never changing, we can't make these const because they get used in - // a vector which uses default construction and copy mechanics + // despite never changing, we can't make these const because this class + // is used in a vector which uses default construction and copy mechanics std::string qaln_, saln_; }; class CnsAlns { public: - CnsAlns() { } + explicit CnsAlns() { } ~CnsAlns() { } void clear() { cns_alns_.clear(); @@ -169,7 +169,17 @@ class ConsensusThreadData { idx_t ec_offset; std::vector data; public: - ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const char* const input_file_name) : rco(prco), reads(r), out(output), ec_offset(0), data(prco.num_threads, ConsensusPerThreadData(prco.error_rate, r.max_read_size())), last_thread_id_(-1), num_threads_written_(0), done_file_(std::string(input_file_name) + ".done"), ckpt_file_(std::string(input_file_name) + ".ckpt"), ckpt_file_tmp_(ckpt_file_ + ".tmp") { } + ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const std::string& input_file_name) : + rco(prco), + reads(r), + out(output), + ec_offset(0), + data(rco.num_threads, ConsensusPerThreadData(rco.error_rate, reads.max_read_size())), + last_thread_id_(-1), + num_threads_written_(0), + done_file_(input_file_name + ".done"), + ckpt_file_(input_file_name + ".ckpt"), + ckpt_file_tmp_(ckpt_file_ + ".tmp") { } ~ConsensusThreadData() { } int get_thread_id() { std::lock_guard lock(id_lock_); @@ -178,15 +188,17 @@ class ConsensusThreadData { } void reset_threads() { last_thread_id_ = -1; - for (int i(0); i < rco.num_threads; ++i) { - data[i].next_candidate = 0; + std::vector::iterator a(data.begin()); + const std::vector::const_iterator end_a(data.end()); + for (; a != end_a; ++a) { + a->next_candidate = 0; } } void write_buffer(const int tid, const idx_t i) { ConsensusPerThreadData& pdata(data[tid]); - std::vector::const_iterator a(pdata.cns_results.begin()); - const std::vector::const_iterator end_a(pdata.cns_results.end()); { + std::vector::const_iterator a(pdata.cns_results.begin()); + const std::vector::const_iterator end_a(pdata.cns_results.end()); std::lock_guard lock(out_lock); for (; a != end_a; ++a) { out << ">" << a->id << "_" << a->range[0] << "_" << a->range[1] << "_" << a->seq.size() << "\n" << a->seq << "\n"; @@ -203,9 +215,11 @@ class ConsensusThreadData { pdata.cns_results.clear(); } int restart(off_t& output_pos) { - if (access(ckpt_file_.c_str(), F_OK) != 0) { - for (int i(0); i < rco.num_threads; ++i) { - data[i].next_candidate = 0; + std::vector::iterator a(data.begin()); + const std::vector::const_iterator end_a(data.end()); + if (access(ckpt_file_.c_str(), F_OK) != 0) { // fresh start + for (; a != end_a; ++a) { + a->next_candidate = 0; } return 0; } @@ -217,8 +231,8 @@ class ConsensusThreadData { if (!ckpt_in) { ERROR("Restart failed: could not read checkpoint file: %s", ckpt_file_.c_str()); } - for (int i(0); i < rco.num_threads; ++i) { - ckpt_in >> data[i].next_candidate; + for (; a != end_a; ++a) { + ckpt_in >> a->next_candidate; if (!ckpt_in) { ERROR("Restart failed: could not read checkpoint file: %s", ckpt_file_.c_str()); } @@ -238,8 +252,10 @@ class ConsensusThreadData { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); return; } - for (int i(0); i < rco.num_threads; ++i) { - ckpt_out << data[i].next_candidate << "\n"; + std::vector::iterator a(data.begin()); + const std::vector::const_iterator end_a(data.end()); + for (; a != end_a; ++a) { + ckpt_out << a->next_candidate << "\n"; if (!ckpt_out) { LOG(stderr, "Checkpoint failed: write failed: %s", ckpt_file_tmp_.c_str()); return; From 5d8af26e34f5742308f41994c37643f0331750b1 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 15 May 2019 10:59:18 -0500 Subject: [PATCH 47/56] narrowed d_path buffer to reflect cutoff it's not just a right triangle, it's a bounded one --- src/mecat2cns/dw.cpp | 4 +++- src/mecat2cns/dw.h | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 9ad7d01..b15ab38 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -76,8 +76,10 @@ static void fill_align(const std::string& query, const int q_offset, const std:: } static int Align(const int extend_size, const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, std::vector& q_extent, std::vector& combined_match_length, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int extend_forward, const double error_rate) { + // if these constants are changed, DiffRunningData buffer sizes + // in dw.h should also be changed const int k_offset(extend_size * 4 * error_rate); - const int band_tolerance(extend_size / 10 * 3); + const int band_tolerance(extend_size * 3 / 10); const int max_band_size(band_tolerance * 2 + 1); int d_path_idx(0), best_combined_match_length(0), min_k(0), max_k(0); q_extent[k_offset + 1] = 0; // initialize starting point diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index a3b6ef8..1a21779 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -90,7 +90,9 @@ class DiffRunningData { result(max_read_size * 2), DynQ(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), DynT(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), - d_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), + // effectively a right triangle on a rectangle, + // as it's bounded geometric growth (4 * error_rate limited to .3) + d_path(4 * error_rate < .3 ? ceil((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER) * .3 * (.15 + 4 * error_rate - .3)) : ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), d_path_index(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)), aln_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 4) { } ~DiffRunningData() { } From 6cc539cef537dee03a337772b4c0bf536ff6d9b0 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Wed, 15 May 2019 11:36:47 -0500 Subject: [PATCH 48/56] comments, refined d_path size --- src/mecat2cns/dw.cpp | 4 ++-- src/mecat2cns/dw.h | 2 +- src/mecat2cns/mecat2cns.mk | 2 ++ src/mecat2cns/reads_correction_aux.h | 33 +++++++++++++++++++++++--- src/mecat2cns/reads_correction_can.cpp | 30 +---------------------- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index b15ab38..49b7716 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -29,7 +29,7 @@ static void fill_align(const std::string& query, const int q_offset, const std:: const int new_y(aln_path[aln_idx].y); const int dx(new_x - current_x); const int dy(new_y - current_y); - if (dx && dy) { // apparently, dx always equals dy in this case + if (dx && dy) { // dx always equals dy in this case std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.current_size]); std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.current_size]); align.current_size += dx; @@ -53,7 +53,7 @@ static void fill_align(const std::string& query, const int q_offset, const std:: const int new_y(aln_path[aln_idx].y); const int dx(new_x - current_x); const int dy(new_y - current_y); - if (dx && dy) { // apparently, dx always equals dy in this case + if (dx && dy) { // dx always equals dy in this case align.current_size += dx; const int offset(align.q_aln_str.size() - align.current_size); std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 1a21779..67f9590 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -92,7 +92,7 @@ class DiffRunningData { DynT(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), // effectively a right triangle on a rectangle, // as it's bounded geometric growth (4 * error_rate limited to .3) - d_path(4 * error_rate < .3 ? ceil((segment_size + SEGMENT_BORDER) * (segment_size + SEGMENT_BORDER) * .3 * (.15 + 4 * error_rate - .3)) : ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), + d_path(4 * error_rate < .3 ? ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * .3 + 2) / 2 + ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * (4 * error_rate - .3)) : ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), d_path_index(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)), aln_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 4) { } ~DiffRunningData() { } diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index 71bc7f5..ef2b942 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -20,6 +20,8 @@ SOURCES := main.cpp \ SRC_INCDIRS := . libboost +# make sure large files are okay (and off_t is 8 bytes); +# requires c++11 or higher for headers TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lmecat diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 60a6e29..0f33b06 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -1,13 +1,15 @@ #ifndef _READS_CORRECTION_AUX_H #define _READS_CORRECTION_AUX_H -#include // vector<> -#include #include // lock_guard<>, mutex +#include // string +#include +#include // vector<> #include "dw.h" // DiffRunningData, M5Record #include "packed_db.h" #include "options.h" +#include "../common/defs.h" // GAP struct CnsTableItem { char base; @@ -19,7 +21,6 @@ struct CnsTableItem { struct MappingRange { int start, end; - explicit MappingRange() : start(0), end(0) { } explicit MappingRange(const int s, const int e) : start(s), end(e) { } }; @@ -136,6 +137,32 @@ struct CmpExtensionCandidateCompressedBySidAndScore { } }; +// like CmpExtensionCandidateCompressedBySidAndScore, but use a new ordering +// (don't store actual vector in this class, as it gets copied a lot by sort()) + +class CmpExtensionCandidateCompressedNewOrder { + public: + explicit CmpExtensionCandidateCompressedNewOrder(const std::vector& order) : order_(order) { } + bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { + if (a.sid != b.sid) { // primary sort + // for splitting up in allocate_ecs() + return order_[a.sid] < order_[b.sid]; + } else if (a.score != b.score) { // secondary sort + return b.score < a.score; // process best ones first + } else if (a.qid != b.qid) { // group ties by qid + return a.qid < b.qid; // (doesn't matter if new or old) + } else if (a.qext() != b.qext()) { + return a.qext() < b.qext(); + } else if (a.sext != b.sext) { // tertiary sort + return a.sext < b.sext; // make sorting consistent + } else { + return a.qdir() < b.qdir(); + } + } + private: + const std::vector& order_; // [read_id] = new_order +}; + class ConsensusPerThreadData { public: // this is ExtensionCandidate for m4 runs, ExtensionCandidateCompressed diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 9ce0c56..fe6c8cc 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -54,34 +54,6 @@ struct CmpExtensionCandidateCompressedBySid { } }; -// do full ordering, but using new rid instead of old -// (we cannot store anything in this class, as it gets copied a lot) - -class OrderCmp { - public: - explicit OrderCmp(const std::vector& order) : order_(order) { } - ~OrderCmp() { } - // same as CmpExtensionCandidateCompressedBySidAndScore, but with new order - bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { - if (a.sid != b.sid) { // primary sort - // for splitting up in allocate_ecs() - return order_[a.sid] < order_[b.sid]; - } else if (a.score != b.score) { // secondary sort - return b.score < a.score; // process best ones first - } else if (a.qid != b.qid) { // group ties by qid - return a.qid < b.qid; // (doesn't matter if new or old) - } else if (a.qext() != b.qext()) { - return a.qext() < b.qext(); - } else if (a.sext != b.sext) { // tertiary sort - return a.sext < b.sext; // make sorting consistent - } else { - return a.qdir() < b.qdir(); - } - } - private: - const std::vector& order_; // [read_id] = order -}; - class EC_Index { // offset into ec_list (and number of ecs) for each read id public: idx_t offset, count; @@ -153,7 +125,7 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ for (size_t i(0); i != new_order.size(); ++i) { rid_to_order[new_order[i]] = i; } - OrderCmp new_rid_order_cmp(rid_to_order); + CmpExtensionCandidateCompressedNewOrder new_rid_order_cmp(rid_to_order); std::sort(ec_list, ec_list + nec, new_rid_order_cmp); nec = total_ec; } From 6c9bd5500e7235087fbf185236bced82a05a1a54 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 21 May 2019 09:36:08 -0500 Subject: [PATCH 49/56] include cleanup, corrected Align() loop termination got rid of argument.*, which was no longer used, added and removed various #includes to better reflect what was actually needed, changed Align() to finish out the inner loop (k_min to k_max) when it hit the termination condition and choose the best of the terminating k values rather than the first one --- src/common/alignment.h | 5 -- src/mecat2cns/MECAT_AlnGraphBoost.H | 2 +- src/mecat2cns/argument.cpp | 75 -------------------------- src/mecat2cns/argument.h | 70 ------------------------ src/mecat2cns/dw.cpp | 25 +++++---- src/mecat2cns/main.cpp | 2 +- src/mecat2cns/mecat2cns.mk | 1 - src/mecat2cns/mecat_correction.h | 3 +- src/mecat2cns/options.cpp | 38 ++++--------- src/mecat2cns/options.h | 26 ++++----- src/mecat2cns/overlaps_partition.cpp | 6 +-- src/mecat2cns/overlaps_partition.h | 5 +- src/mecat2cns/overlaps_store.h | 6 +-- src/mecat2cns/packed_db.cpp | 2 +- src/mecat2cns/packed_db.h | 11 ++-- src/mecat2cns/reads_correction_aux.h | 18 ++++--- src/mecat2cns/reads_correction_can.cpp | 1 + src/mecat2cns/reads_correction_can.h | 2 +- src/mecat2cns/reads_correction_m4.h | 5 +- 19 files changed, 65 insertions(+), 238 deletions(-) delete mode 100644 src/mecat2cns/argument.cpp delete mode 100644 src/mecat2cns/argument.h diff --git a/src/common/alignment.h b/src/common/alignment.h index e9a2fa4..12595ec 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -226,9 +226,4 @@ m4_to_candidate(const M4Record& m4, ExtensionCandidate& ec) ec.score = m4vscore(m4); } -struct CnsResult { - idx_t id, range[2]; - std::string seq; -}; - #endif // ALIGNMENT_H diff --git a/src/mecat2cns/MECAT_AlnGraphBoost.H b/src/mecat2cns/MECAT_AlnGraphBoost.H index 1f75839..916f449 100644 --- a/src/mecat2cns/MECAT_AlnGraphBoost.H +++ b/src/mecat2cns/MECAT_AlnGraphBoost.H @@ -39,7 +39,7 @@ #include -#include "../common/alignment.h" +#include "reads_correction_aux.h" // CnsResult namespace ns_meap_cns { diff --git a/src/mecat2cns/argument.cpp b/src/mecat2cns/argument.cpp deleted file mode 100644 index 94b9f35..0000000 --- a/src/mecat2cns/argument.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "argument.h" - -#include - -#define error_and_exit(msg) { std::cerr << msg << "\n"; abort(); } - -#define no_argument \ - do { \ - std::ostringstream err_msg; \ - err_msg << "no argument is supplied to option \'" << (*arg_name) << "\'"; \ - error_and_exit(err_msg.str()); \ - } while(0); - -#define argument_process_error \ - do { \ - std::ostringstream err_msg; \ - err_msg << "argument \'" << argv[0] << "\' for option \'" << (*arg_name) << "\' processed failed."; \ - error_and_exit(err_msg.str()); \ - } while(0); - -int IntegerArgument::ProcessArgument(int argc, char** argv) -{ - if (!argc) no_argument; - std::istringstream ins(argv[0]); - ins >> val; - if (!ins) argument_process_error; - return 1; -} - -int DoubleArgument::ProcessArgument(int argc, char** argv) -{ - if (!argc) no_argument; - std::istringstream ins(argv[0]); - ins >> val; - if (!ins) argument_process_error; - return 1; -} - -int BooleanArgument::ProcessArgument(int argc, char** argv) -{ - if (!need_arg) - { - val = true; - return 0; - } - else - { - if (!argc) no_argument; - - if (argv[0][0] == '0') - { - val = false; - return 1; - } - else if (argv[0][0] == '1') - { - val = true; - return 1; - } - else - { - std::ostringstream err_msg; - err_msg << "argument to option \'" << (*arg_name) << "\' must be \'0\' or \'1\'"; - error_and_exit(err_msg.str()); - } - } -} - -int StringArgument::ProcessArgument(int argc, char** argv) -{ - if (!argc) no_argument; - - val = argv[0]; - return 1; -} diff --git a/src/mecat2cns/argument.h b/src/mecat2cns/argument.h deleted file mode 100644 index a81c086..0000000 --- a/src/mecat2cns/argument.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef ARGUMENT_H -#define ARGUMENT_H - -#include - -#include "../common/defs.h" - -class Argument -{ -public: - Argument(const std::string* an, const std::string* ad) : arg_name(an), arg_desc(ad) {} - virtual ~Argument() {} - virtual int ProcessArgument(int argc, char** argv) = 0; - -protected: - const std::string* arg_name; - const std::string* arg_desc; -}; - -class IntegerArgument : public Argument -{ -public: - IntegerArgument(const std::string* an, const std::string* ad, const idx_t v) : Argument(an, ad), val(v) {} - virtual ~IntegerArgument() {} - virtual int ProcessArgument(int argc, char** argv); - idx_t value() { return val; } - -private: - idx_t val; -}; - -class DoubleArgument : public Argument -{ -public: - DoubleArgument(const std::string* an, const std::string* ad, double v) : Argument(an, ad), val(v) {} - virtual ~DoubleArgument() {} - virtual int ProcessArgument(int argc, char** argv); - double value() { return val; } - -private: - double val; -}; - -class BooleanArgument : public Argument -{ -public: - BooleanArgument(const std::string* an, const std::string* ad, const bool v, const bool na) - : Argument(an, ad), val(v), need_arg(na) {} - virtual ~BooleanArgument() {} - virtual int ProcessArgument(int argc, char** argv); - bool value() { return val; } - -private: - bool val; - bool need_arg; -}; - -class StringArgument : public Argument -{ -public: - StringArgument(const std::string* an, const std::string* ad, const char* v) : Argument(an, ad), val(v) {} - ~StringArgument() {} - virtual int ProcessArgument(int argc, char** argv); - const char* value() { return val; } - -private: - const char* val; -}; - -#endif // ARGUMENT_H diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 49b7716..b372494 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -81,9 +81,9 @@ static int Align(const int extend_size, const std::string& query, const int q_of const int k_offset(extend_size * 4 * error_rate); const int band_tolerance(extend_size * 3 / 10); const int max_band_size(band_tolerance * 2 + 1); - int d_path_idx(0), best_combined_match_length(0), min_k(0), max_k(0); + int d_path_idx(0), best_combined_match_length(0), best(-1), best_score(-k_offset); q_extent[k_offset + 1] = 0; // initialize starting point - for (int d(0); d != k_offset && max_k - min_k < max_band_size; ++d) { + for (int d(0), min_k(0), max_k(0); d != k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries d_path_index[d].set(d_path_idx, min_k); for (int k(min_k); k <= max_k; k += 2) { @@ -105,18 +105,23 @@ static int Align(const int extend_size, const std::string& query, const int q_of for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } } d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); - // see if we got as much as we can - if (q_pos == extend_size || t_pos == extend_size) { - fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[d_path_idx], d_path_index, d, aln_path, extend_forward); - return 1; + const int score(q_pos + t_pos); + // see if we reached the end + if ((q_pos == extend_size || t_pos == extend_size) && best_score < score) { + best_score = score; + best = d_path_idx; } ++d_path_idx; q_extent[k_offset + k] = q_pos; - combined_match_length[k_offset + k] = q_pos + t_pos; - if (best_combined_match_length < q_pos + t_pos) { - best_combined_match_length = q_pos + t_pos; + combined_match_length[k_offset + k] = score; + if (best_combined_match_length < score) { + best_combined_match_length = score; } } + if (best != -1) { // finished alignment + fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); + return 1; + } // shift ends to one outside "good" band const int cutoff(best_combined_match_length - band_tolerance); for (; combined_match_length[k_offset + min_k] < cutoff; min_k += 2) { } @@ -124,7 +129,7 @@ static int Align(const int extend_size, const std::string& query, const int q_of for (; combined_match_length[k_offset + max_k] < cutoff; max_k -= 2) { } ++max_k; } - return 0; + return 0; // couldn't complete alignemnt } static void dw_in_one_direction(const std::string& query, const int q_offset, const std::string& target, const int t_offset, std::vector& U, std::vector& V, Alignment& align, std::vector& d_path, std::vector& d_path_index, std::vector& aln_path, const int segment_size, OutputStore& result, const int extend_forward, const double error_rate) { diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index db96b4e..30b9101 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -1,7 +1,7 @@ #include "reads_correction_can.h" #include "reads_correction_m4.h" #include "overlaps_partition.h" -#include "options.h" +#include "options.h" // ReadsCorrectionOptions #include "packed_db.h" // PackedDB #include // S_IRUSR, S_IXUSR diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index ef2b942..9837648 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -7,7 +7,6 @@ endif TARGET := mecat2cns SOURCES := main.cpp \ - argument.cpp \ dw.cpp \ MECAT_AlnGraphBoost.C \ mecat_correction.cpp \ diff --git a/src/mecat2cns/mecat_correction.h b/src/mecat2cns/mecat_correction.h index 09bb8c5..4d1daf3 100644 --- a/src/mecat2cns/mecat_correction.h +++ b/src/mecat2cns/mecat_correction.h @@ -1,7 +1,8 @@ #ifndef MEAP_CORRECTION_H #define MEAP_CORRECTION_H -#include "reads_correction_aux.h" +#include "reads_correction_aux.h" // ConsensusPerThreadData, ConsensusThreadData +#include "../common/defs.h" // idx_t void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index d003aed..fd15c94 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -7,6 +7,8 @@ #include #include +#include "../common/defs.h" // TECH_NANOPORE, TECH_PACBIO + static int input_type_pacbio = 1; static int num_threads_pacbio = 1; static double mapping_ratio_pacbio = 0.6; @@ -30,7 +32,7 @@ static double error_rate_nanopore = .2; static int default_tech = TECH_PACBIO; static int num_partition_files = 0; static int full_reads = 0; -static idx_t read_buffer_size = 0; +static size_t read_buffer_size = 0; static idx_t batch_size = idx_t(1) << 33; // 8 GB static const char input_type_n = 'i'; @@ -52,9 +54,9 @@ static const char full_reads_n = 'F'; static const char read_buffer_size_n = 'b'; // convert number with potential suffix (k, m, g) -static idx_t convert_integer(const std::string& s) { +static size_t convert_integer(const std::string& s) { std::istringstream x(s); - idx_t value; + size_t value; x >> value; const size_t i(s.find_first_not_of("0123456789")); if (i != std::string::npos) { @@ -95,7 +97,7 @@ void print_nanopore_default_options() // given options, recreate arguments from the command line std::string -make_options(const ConsensusOptions& options) +make_options(const ReadsCorrectionOptions& options) { std::ostringstream cmd; cmd << " -" << input_type_n << " " << (options.input_type == INPUT_TYPE_CAN ? 0 : 1); @@ -176,8 +178,8 @@ void print_usage(const char* prog) { print_nanopore_default_options(); } -ConsensusOptions init_consensus_options(const int tech) { - ConsensusOptions t; +ReadsCorrectionOptions init_consensus_options(const int tech) { + ReadsCorrectionOptions t; t.m4 = NULL; t.reads = NULL; t.corrected_reads = NULL; @@ -238,7 +240,7 @@ int detect_tech(int argc, char* argv[]) { return t; } -int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { +int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { bool parse_success(true); const int tech(detect_tech(argc, argv)); if (tech == -1) { @@ -316,10 +318,6 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { break; } } - if (t.num_threads < 0) { - std::cerr << "cpu threads must be greater than 0\n"; - parse_success = false; - } if (t.batch_size == 0) { std::cerr << "batch size must be greater than 0\n"; parse_success = false; @@ -328,26 +326,10 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { std::cerr << "mapping ratio must be >= 0.0\n"; parse_success = false; } - if (t.min_cov < 0) { - std::cerr << "coverage must be >= 0\n"; - parse_success = false; - } if (t.min_size < 0) { std::cerr << "sequence size must be >= 0\n"; parse_success = false; } - if (t.reads_to_correct < 0) { - std::cerr << "number of reads must be >= 0\n"; - parse_success = false; - } - if (t.grid_start_delay < 0) { - std::cerr << "grid start delay must be >= 0\n"; - parse_success = false; - } - if (t.read_buffer_size < 0) { - std::cerr << "read buffer size must be greater than 0\n"; - parse_success = false; - } if (argc - optind < 3) { return 1; } @@ -358,7 +340,7 @@ int parse_arguments(int argc, char* argv[], ConsensusOptions& t) { } void -print_options(ConsensusOptions& t) +print_options(ReadsCorrectionOptions& t) { std::cout << "input_type:\t" << t.input_type << "\n"; if (t.m4) std::cout << "reads\t" << t.m4 << "\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index e593bf7..e8abacc 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -1,14 +1,12 @@ #ifndef OPTIONS_H #define OPTIONS_H -#include "../common/defs.h" -#include +#include // string #define INPUT_TYPE_CAN 0 #define INPUT_TYPE_M4 1 -struct ConsensusOptions -{ +struct ReadsCorrectionOptions { int input_type; const char* m4; const char* reads; @@ -16,11 +14,11 @@ struct ConsensusOptions const char* grid_options; const char* grid_options_split; int num_threads; - idx_t batch_size; + int batch_size; double min_mapping_ratio; int min_align_size; int min_cov; - idx_t min_size; + int min_size; bool print_usage_info; int tech; int num_partition_files; @@ -28,22 +26,16 @@ struct ConsensusOptions int reads_to_correct; int grid_start_delay; int full_reads; - idx_t read_buffer_size; + size_t read_buffer_size; double error_rate; // .15 for pacbio, .2 for nanopore }; -void -print_usage(const char* prog); +void print_usage(const char* prog); -int -parse_arguments(int argc, char* argv[], ConsensusOptions& t); +int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t); -void -print_options(ConsensusOptions& t); +void print_options(ReadsCorrectionOptions& t); -std::string -make_options(const ConsensusOptions& t); - -typedef ConsensusOptions ReadsCorrectionOptions; +std::string make_options(const ReadsCorrectionOptions& t); #endif // OPTIONS_H diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index da50e07..568346d 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -88,7 +88,7 @@ void generate_partition_index_file_name(const std::string& input_file_name, std: ret = input_file_name + ".partition_files"; } -void generate_partition_file_name(const std::string& input_file_name, const idx_t part, std::string& ret) { +static void generate_partition_file_name(const std::string& input_file_name, const int part, std::string& ret) { std::ostringstream os; os << part; ret = input_file_name + ".part" + os.str(); @@ -110,7 +110,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi std::vector read_sizes; PackedDB::read_sizes(pac_prefix, read_sizes); PartitionResultsWriter prw(num_files); - idx_t i(0); + int i(0); off_t input_pos; int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); std::string idx_file_name; @@ -195,7 +195,7 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra M4Record m4, nm4; ExtensionCandidate ec; PartitionResultsWriter prw(num_files); - for (idx_t i(0); i < num_batches; i += prw.kNumFiles) { + for (int i(0); i < num_batches; i += prw.kNumFiles) { const idx_t sfid(i); const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index b05ebc9..07b9325 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -2,15 +2,12 @@ #define OVERLAPS_PARTITION_H #include // string -#include // pair<> #include // vector<> -#include "../common/alignment.h" +#include "../common/defs.h" // idx_t void generate_partition_index_file_name(const std::string& input_file_name, std::string& ret); -void generate_partition_file_name(const std::string& input_file_name, const idx_t part, std::string& ret); - void partition_m4records(const char* m4_file_name, double min_cov_ratio, size_t batch_size, int min_read_size, int num_files); void partition_candidates(const std::string& input, const std::string& pac_prefix, size_t batch_size, int num_files, idx_t num_reads); diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index db4cc35..2116e6a 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -13,7 +13,7 @@ template class PartitionResultsWriter { public: - typedef void (*file_name_generator)(const std::string& prefix, idx_t id, std::string& name); + typedef void (*file_name_generator)(const std::string& prefix, int id, std::string& name); public: const int kNumFiles; // effective open file limit int kStoreSize; @@ -88,7 +88,7 @@ template class PartitionResultsWriter { } return 0; } - int restart(const std::string& prefix, file_name_generator fng, const std::string& done_file, idx_t& sfid, off_t& input_pos) { + int restart(const std::string& prefix, file_name_generator fng, const std::string& done_file, int& sfid, off_t& input_pos) { CloseFiles(); done_file_ = done_file; ckpt_file_ = done_file_ + ".ckpt"; @@ -156,7 +156,7 @@ template class PartitionResultsWriter { return total; } private: - idx_t batch_start_ ; + int batch_start_ ; time_t next_checkpoint_time_; std::string done_file_; std::string ckpt_file_; diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index d4dbebe..d868221 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -10,7 +10,7 @@ #include "../common/defs.h" #include "../common/fasta_reader.h" -#include "../common/alignment.h" +#include "../common/alignment.h" // ExtensionCandidateCompressed void PackedDB::add_one_seq(const Sequence& seq) { seq_idx.push_back(SeqIndex(-1, db_size, seq.size())); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index d60a7c9..f76fd5f 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -2,12 +2,9 @@ #define PACKED_DB_H #include // ifstream -#include // set<> #include // string -#include // pair<> #include // vector<> -#include "../common/defs.h" #include "../common/sequence.h" // Sequence #include "../common/alignment.h" // ExtensionCandidateCompressed @@ -58,14 +55,14 @@ class PackedDB { return max_read_size_; } private: - static void set_char(std::vector& p, const idx_t idx, const u1_t c) { + static void set_char(std::vector& p, const idx_t idx, const uint1 c) { p[idx >> 2] |= c << ((~idx & 3) << 1); } - void set_char(const idx_t idx, const u1_t c) { + void set_char(const idx_t idx, const uint1 c) { // use ~x instead of 3 - x for speed, since we have to & 3 anyway pac[idx >> 2] |= c << ((~idx & 3) << 1); } - u1_t get_char(const idx_t idx) const { + uint1 get_char(const idx_t idx) const { return pac[idx >> 2] >> ((~idx & 3) << 1) & 3; } void add_one_seq(const Sequence& seq); @@ -75,7 +72,7 @@ class PackedDB { max_db_size = db_size = 0; } private: - u1_t* pac; + uint1* pac; idx_t db_size, max_db_size, max_read_size_; std::vector seq_idx; std::ifstream pstream; diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 0f33b06..cb8d18c 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -9,16 +9,26 @@ #include "dw.h" // DiffRunningData, M5Record #include "packed_db.h" #include "options.h" +#include "../common/alignment.h" // ExtensionCandidate, ExtensionCandidateCompressed #include "../common/defs.h" // GAP +#define MAX_CNS_OVLPS 100 + +// 1k seems to work a bit better than 10k - perhaps less time waiting for +// another thread to finish writing? +#define MAX_CNS_RESULTS 1000 + +struct CnsResult { + idx_t id, range[2]; + std::string seq; +}; + struct CnsTableItem { char base; uint1 mat_cnt, ins_cnt, del_cnt; explicit CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } }; -#define MAX_CNS_OVLPS 100 - struct MappingRange { int start, end; explicit MappingRange(const int s, const int e) : start(s), end(e) { } @@ -88,10 +98,6 @@ class CnsAlns { std::vector cns_alns_; }; -// 1k seems to work a bit better than 10k - perhaps less time waiting for -// another thread to finish writing? -#define MAX_CNS_RESULTS 1000 - struct CmpExtensionCandidateBySidAndScore { bool operator()(const ExtensionCandidate& a, const ExtensionCandidate& b) { if (a.sid != b.sid) { // primary sort diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index fe6c8cc..1fb0d21 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -9,6 +9,7 @@ #include "mecat_correction.h" #include "overlaps_partition.h" #include "overlaps_store.h" +#include "options.h" // ReadsCorrectionOptions static void* reads_correction_func_can(void* const arg) { ConsensusThreadData& data(*(static_cast(arg))); diff --git a/src/mecat2cns/reads_correction_can.h b/src/mecat2cns/reads_correction_can.h index ff311ea..cca1caf 100644 --- a/src/mecat2cns/reads_correction_can.h +++ b/src/mecat2cns/reads_correction_can.h @@ -1,7 +1,7 @@ #ifndef _READS_CORRECTION_CAN_H #define _READS_CORRECTION_CAN_H -#include "options.h" +#include "options.h" // ReadsCorrectionOptions int reads_correction_can(ReadsCorrectionOptions& rco); diff --git a/src/mecat2cns/reads_correction_m4.h b/src/mecat2cns/reads_correction_m4.h index e81fc0c..4189a43 100644 --- a/src/mecat2cns/reads_correction_m4.h +++ b/src/mecat2cns/reads_correction_m4.h @@ -1,10 +1,7 @@ #ifndef READS_CORRECTION_H #define READS_CORRECTION_H -#include -#include - -#include "reads_correction_aux.h" +#include "options.h" // ReadsCorrectionOptions int reads_correction_m4(ReadsCorrectionOptions& rco); From b0fe7413b560f6d04cdb23fd723165154e7113ac Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Tue, 21 May 2019 10:57:40 -0500 Subject: [PATCH 50/56] more cleanup of header file includes --- src/mecat2cns/overlaps_store.h | 24 +++++++++++------------- src/mecat2cns/packed_db.cpp | 14 +++++++------- src/mecat2cns/packed_db.h | 16 +++++++++------- src/mecat2cns/reads_correction_aux.h | 21 +++++++++++---------- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 2116e6a..2cef270 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -1,15 +1,14 @@ #ifndef _OVERLAPS_STORE_H #define _OVERLAPS_STORE_H -#include -#include -#include -#include -#include -#include +#include // ifstream, ofstream, streampos, streamsize +#include // string +#include // time(), time_t +#include // _SC_OPEN_MAX, F_OK, access(), off_t, rename(), sysconf(), unlink() +#include // vector<> -#include "../common/defs.h" -#include "../common/pod_darr.h" +#include "../common/defs.h" // ERROR(), LOG(), close_fstream(), idx_t, open_fstream() +#include "../common/pod_darr.h" // PODArray<> template class PartitionResultsWriter { public: @@ -45,7 +44,7 @@ template class PartitionResultsWriter { if (num_open_files == 0) { return; } - for (int i(0); i < num_open_files; ++i) { // flush buffers + for (int i(0); i != num_open_files; ++i) { // flush buffers if (results[i].size()) { write_buffer_to_disk(i); } @@ -102,7 +101,7 @@ template class PartitionResultsWriter { ERROR("Read error while restoring checkpoint from %s", ckpt_file_.c_str()); } allocate_data(prefix, fng, 1); - for (int i(0); i < num_open_files; ++i) { + for (int i(0); i != num_open_files; ++i) { off_t file_pos; in >> file_pos >> counts[i]; if (!in) { @@ -128,7 +127,7 @@ template class PartitionResultsWriter { return; } // flush buffers - for (int i(0); i < num_open_files; ++i) { + for (int i(0); i != num_open_files; ++i) { if (results[i].size()) { write_buffer_to_disk(i); results[i].clear(); @@ -169,7 +168,7 @@ template class PartitionResultsWriter { file_names.assign(num_open_files, ""); counts.assign(num_open_files, 0); files = new std::ofstream[num_open_files]; - for (int i(0); i < num_open_files; ++i) { + for (int i(0); i != num_open_files; ++i) { fng(prefix, i + batch_start_, file_names[i]); const std::string tmp_file(file_names[i] + ".tmp"); if (is_restart) { @@ -210,7 +209,6 @@ template T* load_partition_data(const char* const path, idx_t& num_res in.seekg(0, std::ios::beg); num_results = fs / sizeof(T); T* const arr(new T[num_results]); -std::cerr << "load_partition_data: allocating for " << num_results << " objects of size " << sizeof(T) << "\n"; in.read((char*)arr, fs); // can't use static_cast<> if (!in) { ERROR("Error reading partition data: %s", path); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index d868221..15fbbf5 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -22,9 +22,9 @@ void PackedDB::add_one_seq(const Sequence& seq) { idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } uint1* const new_pac(new uint1[(new_size + 3) / 4]); - memcpy(new_pac, pac, (db_size + 3) / 4); - delete[] pac; - pac = new_pac; + memcpy(new_pac, pac_, (db_size + 3) / 4); + delete[] pac_; + pac_ = new_pac; max_db_size = new_size; } const Sequence::str_t& org_seq(seq.sequence()); @@ -180,7 +180,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { const idx_t file_size(pstream.tellg()); max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { - pac = new uint1[max_db_size]; + pac_ = new uint1[max_db_size]; } size_t read_count; if (!pstream.read((char*)&read_count, sizeof(size_t))) { @@ -204,7 +204,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { } } close_fstream(index); - if (!pstream.read((char*)pac, max_db_size)) { + if (!pstream.read((char*)pac_, max_db_size)) { ERROR("Error reading fasta database\n"); } close_fstream(pstream); @@ -266,7 +266,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co LOG(stderr, "using %ld bytes for %lu reads, %ld aligns (out of %ld)", total_size, read_ids.size(), i, nec); if (max_db_size == 0) { max_db_size = total_size; - pac = new uint1[max_db_size]; + pac_ = new uint1[max_db_size]; } // now read in the reads std::set::const_iterator a(read_ids.begin()); @@ -280,7 +280,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co ERROR("Error seeking on fasta db"); } const idx_t bytes((si.size + 3) / 4); - if (!pstream.read((char*)pac + pos, bytes)) { + if (!pstream.read((char*)pac_ + pos, bytes)) { ERROR("Error reading fasta db"); } si.memory_offset = pos * 4; diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index f76fd5f..011b390 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -3,10 +3,12 @@ #include // ifstream #include // string +#include // off_t #include // vector<> -#include "../common/sequence.h" // Sequence #include "../common/alignment.h" // ExtensionCandidateCompressed +#include "../common/defs.h" // idx_t +#include "../common/sequence.h" // Sequence class PackedDB { private: @@ -17,9 +19,9 @@ class PackedDB { explicit SeqIndex(const off_t i, const idx_t j, const idx_t k) : file_offset(i), memory_offset(j), size(k) { } }; public: - explicit PackedDB() : pac(0), db_size(0), max_db_size(0), max_read_size_(0) { } + explicit PackedDB() : pac_(0), db_size(0), max_db_size(0), max_read_size_(0) { } ~PackedDB() { - delete[] pac; + delete[] pac_; } // returns number of reads static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); @@ -60,19 +62,19 @@ class PackedDB { } void set_char(const idx_t idx, const uint1 c) { // use ~x instead of 3 - x for speed, since we have to & 3 anyway - pac[idx >> 2] |= c << ((~idx & 3) << 1); + pac_[idx >> 2] |= c << ((~idx & 3) << 1); } uint1 get_char(const idx_t idx) const { - return pac[idx >> 2] >> ((~idx & 3) << 1) & 3; + return pac_[idx >> 2] >> ((~idx & 3) << 1) & 3; } void add_one_seq(const Sequence& seq); void destroy() { - delete[] pac; + delete[] pac_; seq_idx.clear(); max_db_size = db_size = 0; } private: - uint1* pac; + uint1* pac_; idx_t db_size, max_db_size, max_read_size_; std::vector seq_idx; std::ifstream pstream; diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index cb8d18c..7354524 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -1,16 +1,18 @@ #ifndef _READS_CORRECTION_AUX_H #define _READS_CORRECTION_AUX_H +#include // ifstream, ofstream +#include // ostream #include // lock_guard<>, mutex #include // string -#include +#include // F_OK, access(), rename() #include // vector<> -#include "dw.h" // DiffRunningData, M5Record -#include "packed_db.h" -#include "options.h" #include "../common/alignment.h" // ExtensionCandidate, ExtensionCandidateCompressed -#include "../common/defs.h" // GAP +#include "../common/defs.h" // GAP, idx_t, r_assert() +#include "dw.h" // DiffRunningData, ERROR(), LOG(), M5Record +#include "options.h" // ReadsCorrectionOptions +#include "packed_db.h" // PackedDB #define MAX_CNS_OVLPS 100 @@ -44,7 +46,7 @@ class CnsAln : public MappingRange { return 0; } sb_out = std::max(start, sb); - while (start < sb && aln_idx_ < aln_size) { + while (start < sb && aln_idx_ != aln_size) { if (saln_[++aln_idx_] != GAP) { ++start; } @@ -52,7 +54,7 @@ class CnsAln : public MappingRange { // should we test for start < sb, and return 0 if so, // rather than returning the last basepair of the alignment? const int aln_start(aln_idx_); - while (start < se && aln_idx_ < aln_size) { + while (start < se && aln_idx_ != aln_size) { if (saln_[++aln_idx_] != GAP) { ++start; } @@ -216,8 +218,7 @@ class ConsensusThreadData { ~ConsensusThreadData() { } int get_thread_id() { std::lock_guard lock(id_lock_); - const int tid(++last_thread_id_); - return tid; + return ++last_thread_id_; } void reset_threads() { last_thread_id_ = -1; @@ -229,7 +230,7 @@ class ConsensusThreadData { } void write_buffer(const int tid, const idx_t i) { ConsensusPerThreadData& pdata(data[tid]); - { + { // scoping for lock_guard std::vector::const_iterator a(pdata.cns_results.begin()); const std::vector::const_iterator end_a(pdata.cns_results.end()); std::lock_guard lock(out_lock); From e9f3a9f091514c0085c806a339922d427886fd40 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 24 May 2019 11:59:17 -0500 Subject: [PATCH 51/56] added timing to output --- src/mecat2cns/dw.cpp | 5 +++-- src/mecat2cns/main.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index b372494..c78298a 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -81,7 +81,8 @@ static int Align(const int extend_size, const std::string& query, const int q_of const int k_offset(extend_size * 4 * error_rate); const int band_tolerance(extend_size * 3 / 10); const int max_band_size(band_tolerance * 2 + 1); - int d_path_idx(0), best_combined_match_length(0), best(-1), best_score(-k_offset); + int d_path_idx(0), best_combined_match_length(0); + int best(-1), best_score(-k_offset); q_extent[k_offset + 1] = 0; // initialize starting point for (int d(0), min_k(0), max_k(0); d != k_offset && max_k - min_k < max_band_size; ++d) { // starting point of each "d" set of entries @@ -105,8 +106,8 @@ static int Align(const int extend_size, const std::string& query, const int q_of for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } } d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); - const int score(q_pos + t_pos); // see if we reached the end + const int score(q_pos + t_pos); if ((q_pos == extend_size || t_pos == extend_size) && best_score < score) { best_score = score; best = d_path_idx; diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 30b9101..f5f1296 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -35,7 +35,7 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt unlink(script_file.c_str()); std::ofstream out; open_fstream(out, script_file.c_str(), std::ios::out); - out << "#!/bin/bash\nset -e\ntrap 'touch " << exit_file << "' EXIT\nulimit -c 0\n" << prog << make_options(new_options) << "\n"; + out << "#!/bin/bash\nset -e\ntrap 'touch " << exit_file << "' EXIT\nulimit -c 0\ntime " << prog << make_options(new_options) << "\n"; if (!out) { std::cerr << "Error writing to " << script_file << "\n"; exit(1); From 33c64a3d36ca5a371cbe6d1254323d914936197d Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 14 Jun 2019 13:00:38 -0500 Subject: [PATCH 52/56] Minor cleanup, added lto, moved if out of inner Align loop The lto additions might not be portable, though (particularly the change to src/mecat2cns/main.mk, as I had to specify the plugin location for ar) --- src/main.mk | 4 +- src/mecat2cns/dw.cpp | 125 ++++++++++++++++++++--------- src/mecat2cns/dw.h | 2 + src/mecat2cns/main.cpp | 6 +- src/mecat2cns/mecat2cns.mk | 4 +- src/mecat2cns/mecat_correction.cpp | 46 +++-------- src/mecat2cns/mecat_correction.h | 12 +-- src/mecat2cns/options.cpp | 31 +++---- src/mecat2cns/options.h | 6 +- src/mecat2cns/packed_db.h | 2 +- src/mecat2pw/pw.mk | 7 +- 11 files changed, 131 insertions(+), 114 deletions(-) diff --git a/src/main.mk b/src/main.mk index fb23300..ac447a9 100644 --- a/src/main.mk +++ b/src/main.mk @@ -25,4 +25,6 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -march=native -flto -fno-fat-lto-objects -fno-builtin +TGT_LDFLAGS := ${TGT_CXXFLAGS} +ARFLAGS += --plugin /usr/lib/gcc/x86_64-linux-gnu/5/liblto_plugin.so diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index c78298a..1cfca5e 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -84,51 +84,96 @@ static int Align(const int extend_size, const std::string& query, const int q_of int d_path_idx(0), best_combined_match_length(0); int best(-1), best_score(-k_offset); q_extent[k_offset + 1] = 0; // initialize starting point - for (int d(0), min_k(0), max_k(0); d != k_offset && max_k - min_k < max_band_size; ++d) { - // starting point of each "d" set of entries - d_path_index[d].set(d_path_idx, min_k); - for (int k(min_k); k <= max_k; k += 2) { - int q_pos, pre_k; - if (k == min_k || (k != max_k && q_extent[k_offset + k - 1] < q_extent[k_offset + k + 1])) { - pre_k = k + 1; - q_pos = q_extent[k_offset + k + 1]; - } else { - pre_k = k - 1; - q_pos = q_extent[k_offset + k - 1] + 1; - } - int t_pos(q_pos - k); - // start of exact match - const int q_start(q_pos), t_start(t_pos); - // find the other end of exact match - if (extend_forward) { + // extend_forward only affects innermost loop, but let's move it + // outside as a loop invariant + if (extend_forward) { + for (int d(0), min_k(0), max_k(0); d != k_offset && max_k - min_k < max_band_size; ++d) { + // starting point of each "d" set of entries + d_path_index[d].set(d_path_idx, min_k); + for (int k(min_k); k <= max_k; k += 2) { + int pre_k, q_pos; + if (k == min_k || (k != max_k && q_extent[k_offset + k - 1] < q_extent[k_offset + k + 1])) { + pre_k = k + 1; + q_pos = q_extent[k_offset + k + 1]; + } else { + pre_k = k - 1; + q_pos = q_extent[k_offset + k - 1] + 1; + } + int t_pos(q_pos - k); + // start of exact match + const int q_start(q_pos), t_start(t_pos); + // find the other end of exact match + // XXX - this loop could in theory be vectorized for (; q_pos < extend_size && t_pos < extend_size && query[q_offset + q_pos] == target[t_offset + t_pos]; ++q_pos, ++t_pos) { } - } else { - for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } - } - d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); - // see if we reached the end - const int score(q_pos + t_pos); - if ((q_pos == extend_size || t_pos == extend_size) && best_score < score) { - best_score = score; - best = d_path_idx; + d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); + // see if we reached the end + const int score(q_pos + t_pos); + if ((q_pos == extend_size || t_pos == extend_size) && best_score < score) { + best_score = score; + best = d_path_idx; + } + ++d_path_idx; + q_extent[k_offset + k] = q_pos; + combined_match_length[k_offset + k] = score; + if (best_combined_match_length < score) { + best_combined_match_length = score; + } } - ++d_path_idx; - q_extent[k_offset + k] = q_pos; - combined_match_length[k_offset + k] = score; - if (best_combined_match_length < score) { - best_combined_match_length = score; + if (best != -1) { // finished alignment + fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); + return 1; } + // shift ends to one outside "good" band + const int cutoff(best_combined_match_length - band_tolerance); + for (; combined_match_length[k_offset + min_k] < cutoff; min_k += 2) { } + --min_k; + for (; combined_match_length[k_offset + max_k] < cutoff; max_k -= 2) { } + ++max_k; } - if (best != -1) { // finished alignment - fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); - return 1; + } else { + for (int d(0), min_k(0), max_k(0); d != k_offset && max_k - min_k < max_band_size; ++d) { + // starting point of each "d" set of entries + d_path_index[d].set(d_path_idx, min_k); + for (int k(min_k); k <= max_k; k += 2) { + int pre_k, q_pos; + if (k == min_k || (k != max_k && q_extent[k_offset + k - 1] < q_extent[k_offset + k + 1])) { + pre_k = k + 1; + q_pos = q_extent[k_offset + k + 1]; + } else { + pre_k = k - 1; + q_pos = q_extent[k_offset + k - 1] + 1; + } + int t_pos(q_pos - k); + // start of exact match + const int q_start(q_pos), t_start(t_pos); + // find the other end of exact match + // XXX - this loop could in theory be vectorized + for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } + d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); + // see if we reached the end + const int score(q_pos + t_pos); + if ((q_pos == extend_size || t_pos == extend_size) && best_score < score) { + best_score = score; + best = d_path_idx; + } + ++d_path_idx; + q_extent[k_offset + k] = q_pos; + combined_match_length[k_offset + k] = score; + if (best_combined_match_length < score) { + best_combined_match_length = score; + } + } + if (best != -1) { // finished alignment + fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); + return 1; + } + // shift ends to one outside "good" band + const int cutoff(best_combined_match_length - band_tolerance); + for (; combined_match_length[k_offset + min_k] < cutoff; min_k += 2) { } + --min_k; + for (; combined_match_length[k_offset + max_k] < cutoff; max_k -= 2) { } + ++max_k; } - // shift ends to one outside "good" band - const int cutoff(best_combined_match_length - band_tolerance); - for (; combined_match_length[k_offset + min_k] < cutoff; min_k += 2) { } - --min_k; - for (; combined_match_length[k_offset + max_k] < cutoff; max_k -= 2) { } - ++max_k; } return 0; // couldn't complete alignemnt } diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 67f9590..389fe99 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -86,6 +86,8 @@ class DiffRunningData { std::vector aln_path; public: explicit DiffRunningData(const double error_rate, const idx_t max_read_size) : + // if the definitions of k_offset, band_tolerance, max_band_size + // in dw.cpp are changed, you'll need to update these to reflect them align((segment_size + SEGMENT_BORDER) * 2), result(max_read_size * 2), DynQ(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index f5f1296..b1bfdf3 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -1,7 +1,7 @@ #include "reads_correction_can.h" #include "reads_correction_m4.h" #include "overlaps_partition.h" -#include "options.h" // ReadsCorrectionOptions +#include "options.h" // ReadsCorrectionOptions, make_options() #include "packed_db.h" // PackedDB #include // S_IRUSR, S_IXUSR @@ -35,7 +35,9 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt unlink(script_file.c_str()); std::ofstream out; open_fstream(out, script_file.c_str(), std::ios::out); - out << "#!/bin/bash\nset -e\ntrap 'touch " << exit_file << "' EXIT\nulimit -c 0\ntime " << prog << make_options(new_options) << "\n"; + std::string new_options_str; + make_options(new_options, new_options_str); + out << "#!/bin/bash\nset -e\ntrap 'touch " << exit_file << "' EXIT\nulimit -c 0\ntime " << prog << new_options_str << "\n"; if (!out) { std::cerr << "Error writing to " << script_file << "\n"; exit(1); diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index 9837648..8bd1a4c 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -21,8 +21,8 @@ SRC_INCDIRS := . libboost # make sure large files are okay (and off_t is 8 bytes); # requires c++11 or higher for headers -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -TGT_LDFLAGS := -L${TARGET_DIR} +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -march=native -flto -fno-fat-lto-objects -fno-builtin +TGT_LDFLAGS := -L${TARGET_DIR} ${TGT_CXXFLAGS} TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index c39e862..2ee7b24 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -165,13 +165,7 @@ static void get_effective_ranges(std::vector& mranges, std::vector // breaks output sequence into chunks of no more than MaxSeqSize (if needed); // split chunks will have an OvlpSize overlap -static void -output_cns_result(std::vector& cns_results, - CnsResult& cr, - const idx_t beg, - const idx_t end, - const std::string& cns_seq) -{ +static void output_cns_result(std::vector& cns_results, CnsResult& cr, const idx_t beg, const idx_t end, const std::string& cns_seq) { const size_t MaxSeqSize = 60000; const size_t OvlpSize = 10000; // BlkSize must be >= OvlpSize @@ -200,16 +194,8 @@ output_cns_result(std::vector& cns_results, } } -static inline bool -check_ovlp_mapping_range(const int qb, const int qe, const int qs, - const int sb, const int se, const int ss, - double ratio) -{ - const int oq = qe - qb; - const int qqs = qs * ratio; - const int os = se - sb; - const int qss = ss * ratio; - return oq >= qqs || os >= qss; +static inline bool check_ovlp_mapping_range(const int qb, const int qe, const int qs, const int sb, const int se, const int ss, double ratio) { + return qe - qb >= qs * ratio || se - sb >= ss * ratio; } // look for areas of high coverage of about min_size or more, @@ -327,8 +313,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa Overlap& ovlp(overlaps[i]); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); - const idx_t sext(ovlp.sext); - const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size)); + const int r(GetAlignment(qstr, qext, tstr, ovlp.sext, drd, m5, error_rate, min_align_size)); if (r) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); @@ -341,9 +326,7 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } -void -consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) -{ +void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { PackedDB& reads = ctd.reads; ExtensionCandidate* overlaps = (ExtensionCandidate*)pctd.candidates; DiffRunningData& drd = pctd.drd; @@ -376,16 +359,12 @@ consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData std::vector& cns_table = pctd.cns_table; cns_table.assign(read_size, CnsTableItem()); // reset table cns_vec.clear(); - for (idx_t i = L; i < R; ++i) - { - Overlap& ovlp = overlaps[i]; + for (idx_t i(L); i < R; ++i) { + Overlap& ovlp(overlaps[i]); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); - idx_t qext = ovlp.qext; - idx_t sext = ovlp.sext; - if (ovlp.qdir == REV) qext = ovlp.qsize - 1 - qext; - bool r = GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size); - if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, ovlp.qsize, m5.soff, m5.send, ovlp.ssize, min_mapping_ratio)) - { + const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - qext); + const int r(GetAlignment(qstr, qext, tstr, ovlp.sext, drd, m5, error_rate, min_align_size)); + if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, ovlp.qsize, m5.soff, m5.send, ovlp.ssize, min_mapping_ratio)) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); meap_add_one_aln(nqstr, ntstr, m5.soff, cns_table); cns_vec.add_aln(m5.soff, m5.send, nqstr, ntstr); @@ -456,7 +435,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); + const int r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; @@ -510,9 +489,8 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea } const idx_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); - const idx_t sext(ec.sext); const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); - const bool r(GetAlignment(qstr, qext, tstr, sext, drd, m5, error_rate, min_align_size)); + const int r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { ++num_added; diff --git a/src/mecat2cns/mecat_correction.h b/src/mecat2cns/mecat_correction.h index 4d1daf3..966a443 100644 --- a/src/mecat2cns/mecat_correction.h +++ b/src/mecat2cns/mecat_correction.h @@ -4,16 +4,12 @@ #include "reads_correction_aux.h" // ConsensusPerThreadData, ConsensusThreadData #include "../common/defs.h" // idx_t -void -consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); -void -consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); -void -consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); -void -consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); #endif // MEAP_CORRECTION_H diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index fd15c94..f689f6f 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -15,7 +15,7 @@ static double mapping_ratio_pacbio = 0.6; static int align_size_pacbio = 1000; static int cov_pacbio = 4; static int min_size_pacbio = 2000; -static bool print_usage_pacbio = false; +static int print_usage_pacbio = 0; static int tech_pacbio = TECH_PACBIO; static double error_rate_pacbio = .15; @@ -25,7 +25,7 @@ static double mapping_ratio_nanopore = 0.4; static int align_size_nanopore = 400; static int cov_nanopore = 6; static int min_size_nanopore = 2000; -static bool print_usage_nanopore = false; +static int print_usage_nanopore = 0; static int tech_nanopore = TECH_NANOPORE; static double error_rate_nanopore = .2; @@ -72,9 +72,7 @@ static size_t convert_integer(const std::string& s) { return value; } -void -print_pacbio_default_options() -{ +static void print_pacbio_default_options() { std::cerr << "-" << input_type_n << " " << input_type_pacbio << " -" << num_threads_n << " " << num_threads_pacbio << " -" << mapping_ratio_n << " " << mapping_ratio_pacbio @@ -84,8 +82,7 @@ print_pacbio_default_options() << "\n"; } -void print_nanopore_default_options() -{ +static void print_nanopore_default_options() { std::cerr << "-" << input_type_n << " " << input_type_nanopore << " -" << num_threads_n << " " << num_threads_nanopore << " -" << mapping_ratio_n << " " << mapping_ratio_nanopore @@ -96,9 +93,7 @@ void print_nanopore_default_options() } // given options, recreate arguments from the command line -std::string -make_options(const ReadsCorrectionOptions& options) -{ +void make_options(const ReadsCorrectionOptions& options, std::string& new_options) { std::ostringstream cmd; cmd << " -" << input_type_n << " " << (options.input_type == INPUT_TYPE_CAN ? 0 : 1); if (options.num_threads > -1) { @@ -146,7 +141,7 @@ make_options(const ReadsCorrectionOptions& options) cmd << " " << options.m4; cmd << " " << options.reads; cmd << " " << options.corrected_reads; - return cmd.str(); + new_options = cmd.str(); } void print_usage(const char* prog) { @@ -241,7 +236,7 @@ int detect_tech(int argc, char* argv[]) { } int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { - bool parse_success(true); + int parse_success(1); const int tech(detect_tech(argc, argv)); if (tech == -1) { return 1; @@ -320,15 +315,15 @@ int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { } if (t.batch_size == 0) { std::cerr << "batch size must be greater than 0\n"; - parse_success = false; + parse_success = 0; } if (t.min_mapping_ratio < 0.0) { std::cerr << "mapping ratio must be >= 0.0\n"; - parse_success = false; + parse_success = 0; } if (t.min_size < 0) { std::cerr << "sequence size must be >= 0\n"; - parse_success = false; + parse_success = 0; } if (argc - optind < 3) { return 1; @@ -336,12 +331,10 @@ int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { t.m4 = argv[argc - 3]; t.reads = argv[argc - 2]; t.corrected_reads = argv[argc - 1]; - return parse_success ? 0 : 1; + return parse_success; } -void -print_options(ReadsCorrectionOptions& t) -{ +static void print_options(ReadsCorrectionOptions& t) { std::cout << "input_type:\t" << t.input_type << "\n"; if (t.m4) std::cout << "reads\t" << t.m4 << "\n"; if (t.reads) std::cout << "output\t" << t.reads << "\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index e8abacc..f64cf6e 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -19,7 +19,7 @@ struct ReadsCorrectionOptions { int min_align_size; int min_cov; int min_size; - bool print_usage_info; + int print_usage_info; int tech; int num_partition_files; int job_index; @@ -34,8 +34,6 @@ void print_usage(const char* prog); int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t); -void print_options(ReadsCorrectionOptions& t); - -std::string make_options(const ReadsCorrectionOptions& t); +void make_options(const ReadsCorrectionOptions& t, std::string& new_options_str); #endif // OPTIONS_H diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index 011b390..a8eac73 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -32,7 +32,7 @@ class PackedDB { void open_db(const std::string& filename, idx_t memory_footprint); // returns number of candidates that can be processed idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); - void GetSequence(const idx_t id, const bool forward, std::string& seq) const { + void GetSequence(const idx_t id, const int forward, std::string& seq) const { const SeqIndex &si(seq_idx[id]); seq.resize(si.size); if (forward) { diff --git a/src/mecat2pw/pw.mk b/src/mecat2pw/pw.mk index 29890b7..916d5bb 100644 --- a/src/mecat2pw/pw.mk +++ b/src/mecat2pw/pw.mk @@ -10,8 +10,9 @@ SOURCES := pw.cpp pw_impl.cpp pw_options.cpp SRC_INCDIRS := ../common . -TGT_LDFLAGS := -L${TARGET_DIR} -TGT_LDLIBS := -lmecat -TGT_PREREQS := libmecat.a +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -march=native -flto -fno-fat-lto-objects -fno-builtin +TGT_LDFLAGS := -L${TARGET_DIR} ${TGT_CXXFLAGS} +TGT_LDLIBS := -lmecat +TGT_PREREQS := libmecat.a SUBMAKEFILES := From c96a93be2e7b78f50b97a8ab446ab7ea763b61ee Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Fri, 14 Jun 2019 14:46:06 -0500 Subject: [PATCH 53/56] type cleanup, header file cleanup Got rid of non-standard basic type definitions in mecat2cns, changed all asserts to assert() --- src/mecat2cns/dw.cpp | 50 +++++++++++++------------- src/mecat2cns/dw.h | 10 +++--- src/mecat2cns/main.cpp | 3 +- src/mecat2cns/mecat_correction.cpp | 37 ++++++++++--------- src/mecat2cns/mecat_correction.h | 11 +++--- src/mecat2cns/options.cpp | 5 ++- src/mecat2cns/overlaps_partition.cpp | 13 +++---- src/mecat2cns/overlaps_partition.h | 5 ++- src/mecat2cns/overlaps_store.h | 25 ++++++------- src/mecat2cns/packed_db.cpp | 20 +++++------ src/mecat2cns/packed_db.h | 40 ++++++++++----------- src/mecat2cns/reads_correction_aux.cpp | 33 ++++++++--------- src/mecat2cns/reads_correction_aux.h | 32 +++++++++-------- src/mecat2cns/reads_correction_can.cpp | 6 ++-- src/mecat2cns/reads_correction_m4.cpp | 6 ++-- 15 files changed, 157 insertions(+), 139 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index 1cfca5e..b2dd6b9 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,8 +1,10 @@ #include "dw.h" + #include // copy(), fill() #include // vector<> +#include // uint8_t -#define GAP_ALN 4 +#define GAP_VAL 4 static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { align.aln_q_e = d_path_aux->x2; @@ -35,10 +37,10 @@ static void fill_align(const std::string& query, const int q_offset, const std:: align.current_size += dx; } else if (dx) { std::copy(query_p + current_x, query_p + new_x, &align.q_aln_str[align.current_size]); - std::fill(&align.t_aln_str[align.current_size], &align.t_aln_str[align.current_size] + dx, GAP_ALN); + std::fill(&align.t_aln_str[align.current_size], &align.t_aln_str[align.current_size] + dx, GAP_VAL); align.current_size += dx; } else if (dy) { - std::fill(&align.q_aln_str[align.current_size], &align.q_aln_str[align.current_size] + dy, GAP_ALN); + std::fill(&align.q_aln_str[align.current_size], &align.q_aln_str[align.current_size] + dy, GAP_VAL); std::copy(target_p + current_y, target_p + new_y, &align.t_aln_str[align.current_size]); align.current_size += dy; } @@ -62,11 +64,11 @@ static void fill_align(const std::string& query, const int q_offset, const std:: align.current_size += dx; const int offset(align.q_aln_str.size() - align.current_size); std::copy(query_p - new_x, query_p - current_x, &align.q_aln_str[offset]); - std::fill(&align.t_aln_str[offset], &align.t_aln_str[offset] + dx, GAP_ALN); + std::fill(&align.t_aln_str[offset], &align.t_aln_str[offset] + dx, GAP_VAL); } else if (dy) { align.current_size += dy; const int offset(align.q_aln_str.size() - align.current_size); - std::fill(&align.q_aln_str[offset], &align.q_aln_str[offset] + dy, GAP_ALN); + std::fill(&align.q_aln_str[offset], &align.q_aln_str[offset] + dy, GAP_VAL); std::copy(target_p - new_y, target_p - current_y, &align.t_aln_str[offset]); } current_x = new_x; @@ -199,13 +201,13 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co int q_bps(0), t_bps(0), num_matches(0); if (extend_forward) { for (k = align.current_size - 1; k != -1; --k) { - const uint1 qc(align.q_aln_str[k]); - const uint1 tc(align.t_aln_str[k]); + const uint8_t qc(align.q_aln_str[k]); + const uint8_t tc(align.t_aln_str[k]); if (qc != tc) { num_matches = 0; - if (qc == GAP_ALN) { + if (qc == GAP_VAL) { --q_bps; - } else if (tc == GAP_ALN) { + } else if (tc == GAP_VAL) { --t_bps; } } else if (++num_matches == 4) { @@ -217,13 +219,13 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } else { const int offset(align.q_aln_str.size() - align.current_size); for (k = offset; k != static_cast(align.q_aln_str.size()); ++k) { - const uint1 qc(align.q_aln_str[k]); - const uint1 tc(align.t_aln_str[k]); + const uint8_t qc(align.q_aln_str[k]); + const uint8_t tc(align.t_aln_str[k]); if (qc != tc) { num_matches = 0; - if (qc == GAP_ALN) { + if (qc == GAP_VAL) { --q_bps; - } else if (tc == GAP_ALN) { + } else if (tc == GAP_VAL) { --t_bps; } } else if (++num_matches == 4) { @@ -262,10 +264,10 @@ static void dw_in_one_direction(const std::string& query, const int q_offset, co } while (not_at_end); } -static int gap_count(const std::vector& buffer, int i, const int end_i) { +static int gap_count(const std::vector& buffer, int i, const int end_i) { int j(0); for (; i != end_i; ++i) { - if (buffer[i] == GAP_ALN) { + if (buffer[i] == GAP_VAL) { ++j; } } @@ -290,7 +292,7 @@ static int dw(const std::string& query, const int query_start, const std::string return 1; } -static void decode_sequence(std::string& out_seq, const std::vector& in_seq, const size_t offset, const size_t size) { +static void decode_sequence(std::string& out_seq, const std::vector& in_seq, const size_t offset, const size_t size) { out_seq.resize(size); for (size_t i(0); i != size; ++i) { out_seq[i] = "ACGT-"[in_seq[offset + i]]; @@ -312,14 +314,14 @@ int GetAlignment(const std::string& query, const int query_start, const std::str const int end_k(result.buffer_start + result.right_size); int k(start_k); for (; k != end_k; ++k) { - const uint1 qc(result.q_buffer[k]); - const uint1 tc(result.t_buffer[k]); + const uint8_t qc(result.q_buffer[k]); + const uint8_t tc(result.t_buffer[k]); if (qc != tc) { eit = 0; // we don't count gaps - if (qc == GAP_ALN) { + if (qc == GAP_VAL) { --qrb; - } else if (tc == GAP_ALN) { + } else if (tc == GAP_VAL) { --trb; } } else if (++eit == consecutive_match_region_size) { @@ -338,14 +340,14 @@ int GetAlignment(const std::string& query, const int query_start, const std::str int tre(0); // t ending basepair offset to good sequence eit = 0; // still matching run length for (k = end_k - 1;; --k) { - const uint1 qc(result.q_buffer[k]); - const uint1 tc(result.t_buffer[k]); + const uint8_t qc(result.q_buffer[k]); + const uint8_t tc(result.t_buffer[k]); if (qc != tc) { eit = 0; // we don't count gaps - if (qc == GAP_ALN) { + if (qc == GAP_VAL) { --qre; - } else if (tc == GAP_ALN) { + } else if (tc == GAP_VAL) { --tre; } } else if (++eit == consecutive_match_region_size) { diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index 389fe99..c9c6ddd 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -1,16 +1,16 @@ #ifndef DW_H #define DW_H -#include "../common/defs.h" // idx_t #include // ceil() #include // string +#include // int64_t, uint8_t #include // vector<> class Alignment { public: // current_size tracks actual buffer use int aln_q_e, aln_t_e, current_size; - std::vector q_aln_str, t_aln_str; + std::vector q_aln_str, t_aln_str; public: explicit Alignment(const size_t max_size) : q_aln_str(max_size), t_aln_str(max_size) { } ~Alignment() { } @@ -29,7 +29,7 @@ class OutputStore { int buffer_start, left_size, right_size; int query_start, query_end; int target_start, target_end; - std::vector q_buffer, t_buffer; + std::vector q_buffer, t_buffer; public: explicit OutputStore(const size_t max_size) : q_buffer(max_size), t_buffer(max_size) { } ~OutputStore() { } @@ -85,7 +85,7 @@ class DiffRunningData { std::vector d_path_index; std::vector aln_path; public: - explicit DiffRunningData(const double error_rate, const idx_t max_read_size) : + explicit DiffRunningData(const double error_rate, const int64_t max_read_size) : // if the definitions of k_offset, band_tolerance, max_band_size // in dw.cpp are changed, you'll need to update these to reflect them align((segment_size + SEGMENT_BORDER) * 2), @@ -102,7 +102,7 @@ class DiffRunningData { class M5Record { public: - idx_t qoff, qend, soff, send; + int64_t qoff, qend, soff, send; std::string qaln, saln; explicit M5Record() { } ~M5Record() { } diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index b1bfdf3..677e637 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -4,6 +4,7 @@ #include "options.h" // ReadsCorrectionOptions, make_options() #include "packed_db.h" // PackedDB +#include // assert() #include // S_IRUSR, S_IXUSR #include // list<> #include // ostringstream @@ -46,7 +47,7 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt chmod(script_file.c_str(), S_IRUSR | S_IXUSR); std::string cmd(i == -1 && options.grid_options_split ? options.grid_options_split : options.grid_options); cmd += " " + name + " " + script_file; - assert(system(cmd.c_str()) == 0); + assert(system(cmd.c_str())); } // exit files get modified during loop diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index 2ee7b24..d8fa134 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -1,8 +1,11 @@ #include "mecat_correction.h" -#include "MECAT_AlnGraphBoost.H" - +#include // assert() #include // numeric_limits::max() +#include // uint8_t + +#include "MECAT_AlnGraphBoost.H" +#include "reads_correction_aux.h" // GAP_CHAR #define FMAT 1 #define FDEL 2 @@ -11,9 +14,9 @@ // returns type of coverage present -static inline uint1 identify_one_consensus_item(const CnsTableItem& cns_item) { +static inline uint8_t identify_one_consensus_item(const CnsTableItem& cns_item) { const int cov((cns_item.mat_cnt + cns_item.ins_cnt) * 0.8); - uint1 ident; + uint8_t ident; if (cns_item.mat_cnt >= cov) { // coverage is 80% or more matches ident = FMAT; } else if (cns_item.ins_cnt >= cov) { // coverage is 80% or more inserts @@ -36,25 +39,25 @@ struct CompareOverlapByOverlapSize { }; static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, std::vector& cns_table) { - r_assert(qaln.size() == saln.size()); + assert(qaln.size() == saln.size()); const idx_t aln_size(qaln.size()); for (idx_t i(0); i < aln_size;) { const char q(qaln[i]); const char s(saln[i]); - if (q == '-' && s == '-') { // skip + if (q == GAP_CHAR && s == GAP_CHAR) { // skip ++i; } else if (q == s) { // match ++cns_table[start_soff].mat_cnt; cns_table[start_soff].base = s; ++start_soff; ++i; - } else if (q == '-') { // insert + } else if (q == GAP_CHAR) { // insert ++cns_table[start_soff].ins_cnt; ++start_soff; ++i; } else { // delete - r_assert(s == '-'); - for (++i; i < aln_size && saln[i] == '-'; ++i) { } + assert(s == GAP_CHAR); + for (++i; i < aln_size && saln[i] == GAP_CHAR; ++i) { } ++cns_table[start_soff - 1].del_cnt; } } @@ -72,7 +75,7 @@ static void meap_cns_one_indel(const int sb, const int se, CnsAlns& cns_vec, con ag.consensus(min_cov * 0.4, cns); } -static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target) { +static void meap_consensus_one_segment(const std::vector& cns_list, const int cns_list_size, std::vector& cns_id_vec, const int start_soff, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, std::string& target) { if (cns_id_vec.size() < static_cast(cns_list_size)) { cns_id_vec.resize(cns_list_size); } @@ -82,7 +85,7 @@ static void meap_consensus_one_segment(const std::vector& cns_list } std::string cns; target.clear(); - const uint1 unds_or_fdel(UNDS | FDEL); // questionable coverage types + const uint8_t unds_or_fdel(UNDS | FDEL); // questionable coverage types int i(0); // advance to matching coverage for (; i < cns_list_size && !(cns_id_vec[i] & FMAT); ++i) { } @@ -201,7 +204,7 @@ static inline bool check_ovlp_mapping_range(const int qb, const int qe, const in // look for areas of high coverage of about min_size or more, // improve them and stick on the results pile -static void consensus_worker(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { +static void consensus_worker(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(ceil(0.95 * min_size)); @@ -236,7 +239,7 @@ static void decode_and_append_sequence(std::string& s, const std::string& seq, i // same as consensus_worker, but produces entire read as one entry; // uncorrected sections are just copied as is; -static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::string& tstr, std::vector& cns_results) { +static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::string& tstr, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); @@ -380,7 +383,7 @@ void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThread // limit coverage - if there's not yet sufficient coverage, add coverage and return true; // once coverage gets high enough, stop adding coverage and return false -static inline int check_cov_stats(std::vector& cov_stats, const int soff, const int send) { +static inline int check_cov_stats(std::vector& cov_stats, const int soff, const int send) { if (cov_stats.size() < static_cast(send)) { cov_stats.resize(send, 0); } @@ -393,7 +396,7 @@ static inline int check_cov_stats(std::vector& cov_stats, const int soff, if (send - soff >= n + 200) { for (int i(soff); i < send; ++i) { // don't let small redundant region cause overflow - if (cov_stats[i] != std::numeric_limits::max()) { + if (cov_stats[i] != std::numeric_limits::max()) { ++cov_stats[i]; } } @@ -425,7 +428,7 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD cns_table.assign(ssize, CnsTableItem()); // reset table cns_vec.clear(); std::set used_ids; - std::vector& id_list(pctd.id_list); // used to be called cov_stats + std::vector& id_list(pctd.id_list); // used to be called cov_stats id_list.clear(); for (idx_t i(sid); i < eid && num_added < max_added; ++i) { const ExtensionCandidateCompressed& ec(candidates[i]); @@ -479,7 +482,7 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea cns_table.assign(ssize, CnsTableItem()); // reset table cns_vec.clear(); std::set used_ids; - std::vector& id_list(pctd.id_list); // used to be called cov_stats + std::vector& id_list(pctd.id_list); // used to be called cov_stats id_list.clear(); for (idx_t i(sid); i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { ++num_ext; diff --git a/src/mecat2cns/mecat_correction.h b/src/mecat2cns/mecat_correction.h index 966a443..176fcea 100644 --- a/src/mecat2cns/mecat_correction.h +++ b/src/mecat2cns/mecat_correction.h @@ -1,15 +1,16 @@ #ifndef MEAP_CORRECTION_H #define MEAP_CORRECTION_H +#include // int64_t + #include "reads_correction_aux.h" // ConsensusPerThreadData, ConsensusThreadData -#include "../common/defs.h" // idx_t -void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const int64_t read_id, const int64_t sid, const int64_t eid); -void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const int64_t read_id, const int64_t sid, const int64_t eid); -void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const int64_t read_id, const int64_t sid, const int64_t eid); -void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const idx_t read_id, const idx_t sid, const idx_t eid); +void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &cptd, const int64_t read_id, const int64_t sid, const int64_t eid); #endif // MEAP_CORRECTION_H diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index f689f6f..fd01dbe 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -213,7 +213,7 @@ ReadsCorrectionOptions init_consensus_options(const int tech) { int detect_tech(int argc, char* argv[]) { int t = default_tech; - char tech_nstr[64]; tech_nstr[0] = '-'; tech_nstr[1] = tech_n; tech_nstr[2] = '\0'; + const char tech_nstr[] = {'-', tech_n, 0}; for (int i = 0; i < argc; ++i) { if (strcmp(tech_nstr, argv[i]) == 0) { if (i + 1 == argc) { @@ -334,6 +334,8 @@ int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { return parse_success; } +// not in use at the moment +#if 0 static void print_options(ReadsCorrectionOptions& t) { std::cout << "input_type:\t" << t.input_type << "\n"; if (t.m4) std::cout << "reads\t" << t.m4 << "\n"; @@ -352,3 +354,4 @@ static void print_options(ReadsCorrectionOptions& t) { std::cout << "partition files:\t" << t.num_partition_files << "\n"; std::cout << "tech:\t" << t.tech << "\n"; } +#endif diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 568346d..19fdb47 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -1,5 +1,6 @@ #include "overlaps_partition.h" +#include // assert() #include #include #include @@ -143,15 +144,15 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi continue; } // make sure values match before tossing the values - r_assert(ec.ssize == read_sizes[ec.sid] && ec.qsize == read_sizes[ec.qid]); + assert(ec.ssize == read_sizes[ec.sid] && ec.qsize == read_sizes[ec.qid]); // as variable sizes may be different, make sure the ones // we read in can be safely stored - r_assert(ec.sid <= ExtensionCandidateCompressed::max_value); - r_assert(ec.qid <= ExtensionCandidateCompressed::max_value); - r_assert(ec.sext <= ExtensionCandidateCompressed::max_value); + assert(ec.sid <= ExtensionCandidateCompressed::max_value); + assert(ec.qid <= ExtensionCandidateCompressed::max_value); + assert(ec.sext <= ExtensionCandidateCompressed::max_value); // qext is one bit smaller than the others - r_assert(ec.qext <= ExtensionCandidateCompressed::max_qext); - r_assert(ec.score <= ExtensionCandidateCompressed::max_value); + assert(ec.qext <= ExtensionCandidateCompressed::max_qext); + assert(ec.score <= ExtensionCandidateCompressed::max_value); if (L <= ec.sid && ec.sid < R) { nec.set(ec); if (prw.WriteOneResult((nec.sid - L) / reads_per_batch, nec.sid, nec)) { diff --git a/src/mecat2cns/overlaps_partition.h b/src/mecat2cns/overlaps_partition.h index 07b9325..f8f7cc3 100644 --- a/src/mecat2cns/overlaps_partition.h +++ b/src/mecat2cns/overlaps_partition.h @@ -3,14 +3,13 @@ #include // string #include // vector<> - -#include "../common/defs.h" // idx_t +#include // int64_t void generate_partition_index_file_name(const std::string& input_file_name, std::string& ret); void partition_m4records(const char* m4_file_name, double min_cov_ratio, size_t batch_size, int min_read_size, int num_files); -void partition_candidates(const std::string& input, const std::string& pac_prefix, size_t batch_size, int num_files, idx_t num_reads); +void partition_candidates(const std::string& input, const std::string& pac_prefix, size_t batch_size, int num_files, int64_t num_reads); void load_partition_files_info(const char* idx_file_name, std::vector& file_info_vec); diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index 2cef270..ebff986 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -3,11 +3,12 @@ #include // ifstream, ofstream, streampos, streamsize #include // string +#include // int64_t #include // time(), time_t #include // _SC_OPEN_MAX, F_OK, access(), off_t, rename(), sysconf(), unlink() #include // vector<> -#include "../common/defs.h" // ERROR(), LOG(), close_fstream(), idx_t, open_fstream() +#include "../common/defs.h" // ERROR(), LOG() #include "../common/pod_darr.h" // PODArray<> template class PartitionResultsWriter { @@ -20,7 +21,7 @@ template class PartitionResultsWriter { PODArray* results; // can't use vector<>, causes memory corruption std::ofstream* files; // can't use vector<>, non-copyable std::vector file_names; - std::vector counts; + std::vector counts; public: // can't make kNumFiles static, as sysconf() is run-time only; // leave room for stdin, stdout, stderr, a few others @@ -48,7 +49,7 @@ template class PartitionResultsWriter { if (results[i].size()) { write_buffer_to_disk(i); } - close_fstream(files[i]); + files[i].close(); // don't attempt to re-finish finished files if (!file_names[i].empty()) { const std::string tmp_file(file_names[i] + ".tmp"); @@ -75,7 +76,7 @@ template class PartitionResultsWriter { // remove checkpoint file unlink(std::string(done_file_ + ".ckpt").c_str()); } - int WriteOneResult(const int i, const idx_t seq_id, const T& r) { + int WriteOneResult(const int i, const int64_t seq_id, const T& r) { ++counts[i]; results[i].push_back(r); if (results[i].size() == kStoreSize) { @@ -145,10 +146,10 @@ template class PartitionResultsWriter { } next_checkpoint_time_ = time(0) + 300; } - idx_t total_count() const { - idx_t total(0); - std::vector::const_iterator a(counts.begin()); - const std::vector::const_iterator end_a(counts.end()); + int64_t total_count() const { + int64_t total(0); + std::vector::const_iterator a(counts.begin()); + const std::vector::const_iterator end_a(counts.end()); for (; a != end_a; ++a) { total += *a; } @@ -176,13 +177,13 @@ template class PartitionResultsWriter { // mark as already finished file_names[i].clear(); // use /dev/null to prevent write errors - open_fstream(files[i], "/dev/null", std::ios::binary); + files[i].open("/dev/null", std::ios::binary); } else { // don't truncate on restart - open_fstream(files[i], tmp_file.c_str(), std::ios::binary | std::ios::in); + files[i].open(tmp_file.c_str(), std::ios::binary | std::ios::in); } } else { - open_fstream(files[i], tmp_file.c_str(), std::ios::binary); + files[i].open(tmp_file.c_str(), std::ios::binary); } if (!files[i]) { ERROR("Open failed on %s", tmp_file.c_str()); @@ -201,7 +202,7 @@ template class PartitionResultsWriter { } }; -template T* load_partition_data(const char* const path, idx_t& num_results) { +template T* load_partition_data(const char* const path, int64_t& num_results) { std::ifstream in; open_fstream(in, path, std::ios::binary); in.seekg(0, std::ios::end); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index 15fbbf5..cdfd950 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -2,14 +2,14 @@ #include // ifstream, ofstream #include // set<> +#include // uint8_t #include // rename() #include #include // bzero() #include // stat(), struct stat #include // F_OK, unlink() -#include "../common/defs.h" -#include "../common/fasta_reader.h" +#include "../common/fasta_reader.h" // FastaReader #include "../common/alignment.h" // ExtensionCandidateCompressed void PackedDB::add_one_seq(const Sequence& seq) { @@ -21,17 +21,17 @@ void PackedDB::add_one_seq(const Sequence& seq) { if (max_db_size < needed_size) { idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } - uint1* const new_pac(new uint1[(new_size + 3) / 4]); + uint8_t* const new_pac(new uint8_t[(new_size + 3) / 4]); memcpy(new_pac, pac_, (db_size + 3) / 4); delete[] pac_; pac_ = new_pac; max_db_size = new_size; } const Sequence::str_t& org_seq(seq.sequence()); - const uint1* const table(get_dna_encode_table()); + const uint8_t* const table(get_dna_encode_table()); unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion for (idx_t i(0); i < seq.size(); ++i, ++db_size) { - const uint1 c(table[static_cast(org_seq[i])]); + const uint8_t c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); } } @@ -93,8 +93,8 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string return read_count; } DynamicTimer dtimer(__func__); - std::vector buffer; - const uint1* const et(get_dna_encode_table()); + std::vector buffer; + const uint8_t* const et(get_dna_encode_table()); FastaReader fr(fasta.c_str()); const std::string pac_name_tmp(pac_name + ".tmp"); const std::string index_name_tmp(index_name + ".tmp"); @@ -137,7 +137,7 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string // set_char uses | to set bits, so clear first buffer.assign(rbytes, 0); for (idx_t i(0); i < rsize; ++i) { - const uint1 c(et[static_cast(s[i])]); + const uint8_t c(et[static_cast(s[i])]); set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } if (!pout.write((char*)&buffer[0], rbytes)) { @@ -180,7 +180,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { const idx_t file_size(pstream.tellg()); max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { - pac_ = new uint1[max_db_size]; + pac_ = new uint8_t[max_db_size]; } size_t read_count; if (!pstream.read((char*)&read_count, sizeof(size_t))) { @@ -266,7 +266,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co LOG(stderr, "using %ld bytes for %lu reads, %ld aligns (out of %ld)", total_size, read_ids.size(), i, nec); if (max_db_size == 0) { max_db_size = total_size; - pac_ = new uint1[max_db_size]; + pac_ = new uint8_t[max_db_size]; } // now read in the reads std::set::const_iterator a(read_ids.begin()); diff --git a/src/mecat2cns/packed_db.h b/src/mecat2cns/packed_db.h index a8eac73..012130b 100644 --- a/src/mecat2cns/packed_db.h +++ b/src/mecat2cns/packed_db.h @@ -3,20 +3,20 @@ #include // ifstream #include // string +#include // uint8_t #include // off_t #include // vector<> #include "../common/alignment.h" // ExtensionCandidateCompressed -#include "../common/defs.h" // idx_t #include "../common/sequence.h" // Sequence class PackedDB { private: struct SeqIndex { off_t file_offset; - idx_t memory_offset, size; + int64_t memory_offset, size; explicit SeqIndex() { } - explicit SeqIndex(const off_t i, const idx_t j, const idx_t k) : file_offset(i), memory_offset(j), size(k) { } + explicit SeqIndex(const off_t i, const int64_t j, const int64_t k) : file_offset(i), memory_offset(j), size(k) { } }; public: explicit PackedDB() : pac_(0), db_size(0), max_db_size(0), max_read_size_(0) { } @@ -24,47 +24,47 @@ class PackedDB { delete[] pac_; } // returns number of reads - static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, idx_t min_size); - static void read_sizes(const std::string& output_prefix, std::vector& sizes); + static size_t convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, int64_t min_size); + static void read_sizes(const std::string& output_prefix, std::vector& sizes); // only call one of load_fasta_db and open_db exactly once void load_fasta_db(const char* fasta); // opens data file, reads in index file - void open_db(const std::string& filename, idx_t memory_footprint); + void open_db(const std::string& filename, int64_t memory_footprint); // returns number of candidates that can be processed - idx_t load_reads(const ExtensionCandidateCompressed* ec_list, idx_t nec); - void GetSequence(const idx_t id, const int forward, std::string& seq) const { + int64_t load_reads(const ExtensionCandidateCompressed* ec_list, int64_t nec); + void GetSequence(const int64_t id, const int forward, std::string& seq) const { const SeqIndex &si(seq_idx[id]); seq.resize(si.size); if (forward) { - const idx_t offset(si.memory_offset); - for (idx_t i(0); i < si.size; ++i) { + const int64_t offset(si.memory_offset); + for (int64_t i(0); i < si.size; ++i) { seq[i] = get_char(offset + i); } } else { - const idx_t offset(si.memory_offset + si.size - 1); - for (idx_t i(0); i < si.size; ++i) { + const int64_t offset(si.memory_offset + si.size - 1); + for (int64_t i(0); i < si.size; ++i) { seq[i] = 3 - get_char(offset - i); } } } - idx_t num_reads() const { + int64_t num_reads() const { return seq_idx.size(); } - idx_t read_size(const idx_t read_id) const { + int64_t read_size(const int64_t read_id) const { return seq_idx[read_id].size; } - idx_t max_read_size() const { + int64_t max_read_size() const { return max_read_size_; } private: - static void set_char(std::vector& p, const idx_t idx, const uint1 c) { + static void set_char(std::vector& p, const int64_t idx, const uint8_t c) { p[idx >> 2] |= c << ((~idx & 3) << 1); } - void set_char(const idx_t idx, const uint1 c) { + void set_char(const int64_t idx, const uint8_t c) { // use ~x instead of 3 - x for speed, since we have to & 3 anyway pac_[idx >> 2] |= c << ((~idx & 3) << 1); } - uint1 get_char(const idx_t idx) const { + uint8_t get_char(const int64_t idx) const { return pac_[idx >> 2] >> ((~idx & 3) << 1) & 3; } void add_one_seq(const Sequence& seq); @@ -74,8 +74,8 @@ class PackedDB { max_db_size = db_size = 0; } private: - uint1* pac_; - idx_t db_size, max_db_size, max_read_size_; + uint8_t* pac_; + int64_t db_size, max_db_size, max_read_size_; std::vector seq_idx; std::ifstream pstream; }; diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index af0d4bf..741bdfa 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -1,11 +1,12 @@ #include "reads_correction_aux.h" + +#include // assert() #include // string void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, const int push) { qnorm.clear(); tnorm.clear(); - const char kGap = '-'; #ifndef NDEBUG int qcnt = 0, tcnt = 0; @@ -13,8 +14,8 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin { const char qc = qstr[i]; const char tc = tstr[i]; - if (qc != kGap) ++qcnt; - if (tc != kGap) ++tcnt; + if (qc != GAP_CHAR) ++qcnt; + if (tc != GAP_CHAR) ++tcnt; } #endif @@ -23,8 +24,8 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin { const char qc = qstr[i]; const char tc = tstr[i]; - if (qc != tc && qc != kGap && tc != kGap) - { qnorm += kGap; qnorm += qc; tnorm += tc; tnorm += kGap; } + if (qc != tc && qc != GAP_CHAR && tc != GAP_CHAR) + { qnorm += GAP_CHAR; qnorm += qc; tnorm += tc; tnorm += GAP_CHAR; } else { qnorm += qc; tnorm += tc; } } @@ -37,45 +38,45 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin for (idx_t i = 0; i < qlen - 1; ++i) { // push target gaps - if (tnorm[i] == kGap) + if (tnorm[i] == GAP_CHAR) { idx_t j = i; while (1) { const char c = tnorm[++j]; - if (c != kGap || j > qlen - 1) + if (c != GAP_CHAR || j > qlen - 1) { - if (c == qnorm[i]) { tnorm[i] = c; tnorm[j] = kGap; } + if (c == qnorm[i]) { tnorm[i] = c; tnorm[j] = GAP_CHAR; } break; } } } // push query gaps - if (qnorm[i] == kGap) + if (qnorm[i] == GAP_CHAR) { idx_t j = i; while (1) { const char c = qnorm[++j]; - if (c != kGap || j > tlen - 1) + if (c != GAP_CHAR || j > tlen - 1) { - if (c == tnorm[i]) { qnorm[i] = c; qnorm[j] = kGap; } + if (c == tnorm[i]) { qnorm[i] = c; qnorm[j] = GAP_CHAR; } break; } } } } } - r_assert(qnorm.size() == tnorm.size()); + assert(qnorm.size() == tnorm.size()); #ifndef NDEBUG int qcnt2 = 0, tcnt2 = 0; for (std::string::const_iterator citer = qnorm.begin(); citer != qnorm.end(); ++citer) - if ((*citer) != kGap) ++qcnt2; + if ((*citer) != GAP_CHAR) ++qcnt2; for (std::string::const_iterator citer = tnorm.begin(); citer != tnorm.end(); ++citer) - if ((*citer) != kGap) ++tcnt2; - d_assert(qcnt == qcnt2); - d_assert(tcnt == tcnt2); + if ((*citer) != GAP_CHAR) ++tcnt2; + assert(qcnt == qcnt2); + assert(tcnt == tcnt2); #endif } diff --git a/src/mecat2cns/reads_correction_aux.h b/src/mecat2cns/reads_correction_aux.h index 7354524..65e5106 100644 --- a/src/mecat2cns/reads_correction_aux.h +++ b/src/mecat2cns/reads_correction_aux.h @@ -1,33 +1,35 @@ #ifndef _READS_CORRECTION_AUX_H #define _READS_CORRECTION_AUX_H +#include // assert() #include // ifstream, ofstream #include // ostream #include // lock_guard<>, mutex #include // string +#include // int64_t, uint8_t #include // F_OK, access(), rename() #include // vector<> #include "../common/alignment.h" // ExtensionCandidate, ExtensionCandidateCompressed -#include "../common/defs.h" // GAP, idx_t, r_assert() #include "dw.h" // DiffRunningData, ERROR(), LOG(), M5Record #include "options.h" // ReadsCorrectionOptions #include "packed_db.h" // PackedDB #define MAX_CNS_OVLPS 100 +#define GAP_CHAR '-' // 1k seems to work a bit better than 10k - perhaps less time waiting for // another thread to finish writing? #define MAX_CNS_RESULTS 1000 struct CnsResult { - idx_t id, range[2]; + int64_t id, range[2]; std::string seq; }; struct CnsTableItem { char base; - uint1 mat_cnt, ins_cnt, del_cnt; + uint8_t mat_cnt, ins_cnt, del_cnt; explicit CnsTableItem() : base('N'), mat_cnt(0), ins_cnt(0), del_cnt(0) { } }; @@ -47,7 +49,7 @@ class CnsAln : public MappingRange { } sb_out = std::max(start, sb); while (start < sb && aln_idx_ != aln_size) { - if (saln_[++aln_idx_] != GAP) { + if (saln_[++aln_idx_] != GAP_CHAR) { ++start; } } @@ -55,7 +57,7 @@ class CnsAln : public MappingRange { // rather than returning the last basepair of the alignment? const int aln_start(aln_idx_); while (start < se && aln_idx_ != aln_size) { - if (saln_[++aln_idx_] != GAP) { + if (saln_[++aln_idx_] != GAP_CHAR) { ++start; } } @@ -90,7 +92,7 @@ class CnsAlns { return cns_alns_.end(); } void add_aln(const int soff, const int send, const std::string& qstr, const std::string& tstr) { - r_assert(qstr.size() == tstr.size()); + assert(qstr.size() == tstr.size()); cns_alns_.push_back(CnsAln(soff, send, qstr, tstr)); } void get_mapping_ranges(std::vector& ranges) const { @@ -150,7 +152,7 @@ struct CmpExtensionCandidateCompressedBySidAndScore { class CmpExtensionCandidateCompressedNewOrder { public: - explicit CmpExtensionCandidateCompressedNewOrder(const std::vector& order) : order_(order) { } + explicit CmpExtensionCandidateCompressedNewOrder(const std::vector& order) : order_(order) { } bool operator()(const ExtensionCandidateCompressed& a, const ExtensionCandidateCompressed& b) { if (a.sid != b.sid) { // primary sort // for splitting up in allocate_ecs() @@ -168,7 +170,7 @@ class CmpExtensionCandidateCompressedNewOrder { } } private: - const std::vector& order_; // [read_id] = new_order + const std::vector& order_; // [read_id] = new_order }; class ConsensusPerThreadData { @@ -178,16 +180,16 @@ class ConsensusPerThreadData { void* candidates; // num_candidates, candidates initialized by allocate_ecs() // next_candidate initialized by ConsensusThreadData::restart() - idx_t num_candidates, next_candidate; + int64_t num_candidates, next_candidate; DiffRunningData drd; M5Record m5; CnsAlns cns_alns; std::vector cns_table; - std::vector id_list; + std::vector id_list; std::vector cns_results; std::string query, target, qaln, saln; public: - explicit ConsensusPerThreadData(const double error_rate, const idx_t max_read_size) : drd(error_rate, max_read_size) { + explicit ConsensusPerThreadData(const double error_rate, const int64_t max_read_size) : drd(error_rate, max_read_size) { // we'll definitely be seeing at least this much use, // so might as well preallocate cns_results.reserve(MAX_CNS_RESULTS); @@ -201,7 +203,7 @@ class ConsensusThreadData { PackedDB& reads; std::ostream& out; std::mutex out_lock; - idx_t ec_offset; + int64_t ec_offset; std::vector data; public: ConsensusThreadData(ReadsCorrectionOptions& prco, PackedDB& r, std::ostream& output, const std::string& input_file_name) : @@ -228,7 +230,7 @@ class ConsensusThreadData { a->next_candidate = 0; } } - void write_buffer(const int tid, const idx_t i) { + void write_buffer(const int tid, const int64_t i) { ConsensusPerThreadData& pdata(data[tid]); { // scoping for lock_guard std::vector::const_iterator a(pdata.cns_results.begin()); @@ -308,7 +310,7 @@ class ConsensusThreadData { void normalize_gaps(const std::string& qstr, const std::string& tstr, std::string& qnorm, std::string& tnorm, int push); -void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, idx_t nec); -void allocate_ecs(ConsensusThreadData &data, ExtensionCandidateCompressed* ec_list, idx_t nec); +void allocate_ecs(ConsensusThreadData &data, ExtensionCandidate* ec_list, int64_t nec); +void allocate_ecs(ConsensusThreadData &data, ExtensionCandidateCompressed* ec_list, int64_t nec); #endif // _READS_CORRECTION_AUX_H diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 1fb0d21..647ca88 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -1,10 +1,12 @@ #include "reads_correction_can.h" +#include // assert() #include #include // numeric_limits<> #include #include // string +#include "../common/defs.h" // TECH_PACBIO #include "MECAT_AlnGraphBoost.H" #include "mecat_correction.h" #include "overlaps_partition.h" @@ -185,7 +187,7 @@ static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector // sort() -#include // string +#include // assert() #include +#include // string +#include "../common/defs.h" // TECH_PACBIO #include "mecat_correction.h" #include "overlaps_partition.h" #include "overlaps_store.h" @@ -88,7 +90,7 @@ static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector Date: Wed, 19 Jun 2019 15:32:48 -0500 Subject: [PATCH 54/56] vectorization of mecat2cns inner loop, some cleanup vectorized using SSE2 commands and a touch of assembly more conversion of idx_t to int64_t --- mecat2canu/src/AS_UTL/stddev.H | 4 +- .../src/utgcns/libcns/unitigConsensus.C | 4 +- src/main.mk | 6 +- src/mecat2cns/dw.cpp | 126 ++++++++++++++++-- src/mecat2cns/dw.h | 23 ++-- src/mecat2cns/main.cpp | 6 +- src/mecat2cns/mecat2cns.mk | 4 +- src/mecat2cns/mecat_correction.cpp | 72 +++++----- src/mecat2cns/options.cpp | 64 ++++----- src/mecat2cns/options.h | 2 +- src/mecat2cns/overlaps_partition.cpp | 44 +++--- src/mecat2cns/packed_db.cpp | 48 +++---- src/mecat2cns/reads_correction_aux.cpp | 24 ++-- src/mecat2cns/reads_correction_can.cpp | 48 +++---- src/mecat2cns/reads_correction_m4.cpp | 10 +- 15 files changed, 292 insertions(+), 193 deletions(-) diff --git a/mecat2canu/src/AS_UTL/stddev.H b/mecat2canu/src/AS_UTL/stddev.H index 88ff91b..88b55b2 100644 --- a/mecat2canu/src/AS_UTL/stddev.H +++ b/mecat2canu/src/AS_UTL/stddev.H @@ -246,12 +246,12 @@ public: return(_mad); }; - vector &histogram(void) { // Returns pointer to private histogram data + vector *histogram(void) { // Returns pointer to private histogram data finalizeData(); return(&_histogram); }; - vector &Nstatistics(void) { // Returns pointer to private N data + vector *Nstatistics(void) { // Returns pointer to private N data finalizeData(); return(&_Nstatistics); }; diff --git a/mecat2canu/src/utgcns/libcns/unitigConsensus.C b/mecat2canu/src/utgcns/libcns/unitigConsensus.C index 5789a21..a6d6f17 100644 --- a/mecat2canu/src/utgcns/libcns/unitigConsensus.C +++ b/mecat2canu/src/utgcns/libcns/unitigConsensus.C @@ -726,7 +726,7 @@ unitigConsensus::computePositionFromAlignment(void) { if (foundAlign == false) { - if (oaPartial == false) + if (oaPartial == NULL) oaPartial = new NDalign(pedLocal, errorRate, 17); // partial allowed! oaPartial->initialize(0, abacus->bases(), abacus->numberOfColumns(), 0, abacus->numberOfColumns(), @@ -999,7 +999,7 @@ unitigConsensus::alignFragment(bool forceAlignment) { // Create new aligner object. 'Global' in this case just means to not stop early, not a true global alignment. - if (oaFull == false) + if (oaFull == NULL) oaFull = new NDalign(pedGlobal, errorRate, 17); oaFull->initialize(0, aseq, cnsEnd - cnsBgn, 0, cnsEnd - cnsBgn, diff --git a/src/main.mk b/src/main.mk index ac447a9..1eb8186 100644 --- a/src/main.mk +++ b/src/main.mk @@ -25,6 +25,8 @@ SUBMAKEFILES := mecat2pw/pw.mk \ mecat2cns/mecat2cns.mk \ filter_reads/filter_reads.mk -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -march=native -flto -fno-fat-lto-objects -fno-builtin +# Note: -O2 performed about 1% worse than -O3 + +TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -mfpmath=sse -march=native -flto -fno-fat-lto-objects -fno-builtin -mmmx -msse -msse2 -mssse3 -msse4.1 -msse4.2 -mavx -maes -mpopcnt -mfxsr -mxsave -mxsaveopt TGT_LDFLAGS := ${TGT_CXXFLAGS} -ARFLAGS += --plugin /usr/lib/gcc/x86_64-linux-gnu/5/liblto_plugin.so +ARFLAGS += --plugin /mnt/local/gnu/libexec/gcc/x86_64-pc-linux-gnu/9.1.0/liblto_plugin.so.0 diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index b2dd6b9..f904b56 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -1,23 +1,67 @@ #include "dw.h" +#ifdef __SSE2__ +#include // _mm_loadu_si128(), _mm_xor_si128() +#include // _mm_test_all_zeros() +#endif #include // copy(), fill() #include // vector<> #include // uint8_t #define GAP_VAL 4 -static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, const DPathData* d_path_aux, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { - align.aln_q_e = d_path_aux->x2; - align.aln_t_e = d_path_aux->y2; +#ifdef __SSE2__ + +// bit_scan_forward definitions are taken from vectori128.h, part of +// Agner Fog's vector class package version 1.28, and are copyright +// by him (2017) (https://www.agner.org/optimize/#vectorclass) + +// Define bit-scan-forward function. Gives index to lowest set bit +#if defined (__GNUC__) || defined(__clang__) +static inline uint32_t bit_scan_forward(uint32_t a) __attribute__ ((pure)); +static inline uint32_t bit_scan_forward(uint32_t a) { + uint32_t r; + __asm("bsfl %1, %0" : "=r"(r) : "r"(a) : ); + return r; +} +#else +static inline uint32_t bit_scan_forward (uint32_t a) { + unsigned long r; + _BitScanForward(&r, a); // defined in intrin.h for MS and Intel compilers + return r; +} +#endif + +// Define bit-scan-reverse function. Gives index to highest set bit. +// Make sure to mask unused high bits +#if defined (__GNUC__) || defined(__clang__) +static inline uint32_t bit_scan_reverse(uint32_t a) __attribute__ ((pure)); +static inline uint32_t bit_scan_reverse(uint32_t a) { + uint32_t r; + __asm("bsrl %1, %0" : "=r"(r) : "r"(a) : ); + return r; +} +#else +static inline uint32_t bit_scan_reverse (uint32_t a) { + unsigned long r; + _BitScanReverse(&r, a); // defined in intrin.h for MS and Intel compilers + return r; +} +#endif +#endif // __SSEE2__ + +static void fill_align(const std::string& query, const int q_offset, const std::string& target, const int t_offset, Alignment& align, const std::vector& d_path, int i, const std::vector& d_path_index, int d, std::vector& aln_path, const int extend_forward) { + align.aln_q_e = d_path[i].x2; + align.aln_t_e = d_path[i].y2; // get align path int aln_idx(-1); for (;;) { - aln_path[++aln_idx].set(d_path_aux->x2, d_path_aux->y2); - aln_path[++aln_idx].set(d_path_aux->x1, d_path_aux->y1); + aln_path[++aln_idx].set(d_path[i].x2, d_path[i].y2); + aln_path[++aln_idx].set(d_path[i].x1, d_path[i].y1); if (--d == -1) { break; } - d_path_aux = &d_path[d_path_index[d].d_offset + (d_path_aux->pre_k - d_path_index[d].min_k) / 2]; + i = d_path_index[d].d_offset + (d_path[i].pre_k - d_path_index[d].min_k) / 2; } // walk backwards along align path to fill in sequence with gaps align.clear(); @@ -83,6 +127,8 @@ static int Align(const int extend_size, const std::string& query, const int q_of const int k_offset(extend_size * 4 * error_rate); const int band_tolerance(extend_size * 3 / 10); const int max_band_size(band_tolerance * 2 + 1); + const char* const q_ptr_start(query.data() + q_offset); + const char* const t_ptr_start(target.data() + t_offset); int d_path_idx(0), best_combined_match_length(0); int best(-1), best_score(-k_offset); q_extent[k_offset + 1] = 0; // initialize starting point @@ -105,8 +151,37 @@ static int Align(const int extend_size, const std::string& query, const int q_of // start of exact match const int q_start(q_pos), t_start(t_pos); // find the other end of exact match - // XXX - this loop could in theory be vectorized +#ifdef __SSE2__ + // vectorization of loop using gnu intrinsics + __m128i a16, b16, c16; + const char* q_ptr(q_ptr_start + q_pos); + const char* t_ptr(t_ptr_start + t_pos); + int i(0); + // round down to nearest multiple of 16 + int end_i((extend_size - std::max(q_pos, t_pos)) & (~0xf)); + for (; i != end_i; i += 16, q_ptr += 16, t_ptr += 16) { + a16 = _mm_loadu_si128((const __m128i*)q_ptr); + b16 = _mm_loadu_si128((const __m128i*)t_ptr); + c16 = _mm_cmpeq_epi8(a16, b16); + const uint32_t x(_mm_movemask_epi8(c16)); + if (x != 0xffff) { + const int b(bit_scan_forward(~x)); + q_ptr += b; + t_ptr += b; + // no need to increment i here, it's + // sufficient that it doesn't equal end_i + break; + } + } + if (i == end_i) { // deal with remainder + end_i += (extend_size - std::max(q_pos, t_pos)) & 0xf; + for (; i != end_i && *q_ptr == *t_ptr; ++i, ++q_ptr, ++t_ptr) { } + } + q_pos = q_ptr - q_ptr_start; + t_pos = t_ptr - t_ptr_start; +#else for (; q_pos < extend_size && t_pos < extend_size && query[q_offset + q_pos] == target[t_offset + t_pos]; ++q_pos, ++t_pos) { } +#endif // __SSE2__ d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); // see if we reached the end const int score(q_pos + t_pos); @@ -122,7 +197,7 @@ static int Align(const int extend_size, const std::string& query, const int q_of } } if (best != -1) { // finished alignment - fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); + fill_align(query, q_offset, target, t_offset, align, d_path, best, d_path_index, d, aln_path, extend_forward); return 1; } // shift ends to one outside "good" band @@ -149,8 +224,39 @@ static int Align(const int extend_size, const std::string& query, const int q_of // start of exact match const int q_start(q_pos), t_start(t_pos); // find the other end of exact match - // XXX - this loop could in theory be vectorized +#ifdef __SSE2__ + // vectorization of above loop + __m128i a16, b16, c16; + const char* q_ptr(q_ptr_start - q_pos - 15); + const char* t_ptr(t_ptr_start - t_pos - 15); + int i(0); + // round down to nearest multiple of 16 + int end_i((extend_size - std::max(q_pos, t_pos)) & (~0xf)); + for (; i != end_i; i += 16, q_ptr -= 16, t_ptr -= 16) { + a16 = _mm_loadu_si128((const __m128i*)q_ptr); + b16 = _mm_loadu_si128((const __m128i*)t_ptr); + c16 = _mm_cmpeq_epi8(a16, b16); + const uint32_t x(_mm_movemask_epi8(c16)); + if (x != 0xffff) { + // have to mask high bits + const int b(bit_scan_reverse(0xffff & (~x))); + // technically, += 15 - (15 - b) + q_ptr += b; + t_ptr += b; + // no need to increment i here, it's + // sufficient that it doesn't equal end_i + break; + } + } + if (i == end_i) { // deal with remainder + end_i += (extend_size - std::max(q_pos, t_pos)) & 0xf; + for (q_ptr += 15, t_ptr += 15; i != end_i && *q_ptr == *t_ptr; ++i, --q_ptr, --t_ptr) { } + } + q_pos = q_ptr_start - q_ptr; + t_pos = t_ptr_start - t_ptr; +#else for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } +#endif // __SSE2__ d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); // see if we reached the end const int score(q_pos + t_pos); @@ -166,7 +272,7 @@ static int Align(const int extend_size, const std::string& query, const int q_of } } if (best != -1) { // finished alignment - fill_align(query, q_offset, target, t_offset, align, d_path, &d_path[best], d_path_index, d, aln_path, extend_forward); + fill_align(query, q_offset, target, t_offset, align, d_path, best, d_path_index, d, aln_path, extend_forward); return 1; } // shift ends to one outside "good" band diff --git a/src/mecat2cns/dw.h b/src/mecat2cns/dw.h index c9c6ddd..33dd77b 100644 --- a/src/mecat2cns/dw.h +++ b/src/mecat2cns/dw.h @@ -2,15 +2,15 @@ #define DW_H #include // ceil() -#include // string #include // int64_t, uint8_t +#include // string #include // vector<> class Alignment { public: // current_size tracks actual buffer use - int aln_q_e, aln_t_e, current_size; std::vector q_aln_str, t_aln_str; + int aln_q_e, aln_t_e, current_size; public: explicit Alignment(const size_t max_size) : q_aln_str(max_size), t_aln_str(max_size) { } ~Alignment() { } @@ -26,10 +26,10 @@ class OutputStore { public: // these track actual buffer use - buffer use starts at // buffer_start, with left going down, and right going up + std::vector q_buffer, t_buffer; int buffer_start, left_size, right_size; int query_start, query_end; int target_start, target_end; - std::vector q_buffer, t_buffer; public: explicit OutputStore(const size_t max_size) : q_buffer(max_size), t_buffer(max_size) { } ~OutputStore() { } @@ -43,7 +43,8 @@ class OutputStore { }; struct DPathData { - int x1, y1, x2, y2, pre_k; + // using shorts to reduce size of largest buffer + short int x1, y1, x2, y2, pre_k; void set(const int i, const int j, const int k, const int l, const int m) { x1 = i; y1 = j; @@ -78,25 +79,25 @@ struct PathPoint { class DiffRunningData { public: static const int segment_size = 500; // 500 is "small", 1000 is "large" - Alignment align; - OutputStore result; std::vector DynQ, DynT; - std::vector d_path; std::vector d_path_index; std::vector aln_path; + std::vector d_path; + Alignment align; + OutputStore result; public: explicit DiffRunningData(const double error_rate, const int64_t max_read_size) : // if the definitions of k_offset, band_tolerance, max_band_size // in dw.cpp are changed, you'll need to update these to reflect them - align((segment_size + SEGMENT_BORDER) * 2), - result(max_read_size * 2), DynQ(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), DynT(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 2), // effectively a right triangle on a rectangle, // as it's bounded geometric growth (4 * error_rate limited to .3) - d_path(4 * error_rate < .3 ? ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * .3 + 2) / 2 + ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * (4 * error_rate - .3)) : ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), d_path_index(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate)), - aln_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 4) { } + aln_path(ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * 4), + d_path(4 * error_rate < .3 ? ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * .3 + 2) / 2 + ceil((segment_size + SEGMENT_BORDER) * .3 + 1) * ceil((segment_size + SEGMENT_BORDER) * (4 * error_rate - .3)) : ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate) * ceil((segment_size + SEGMENT_BORDER) * 4 * error_rate + 1) / 2), + align((segment_size + SEGMENT_BORDER) * 2), + result(max_read_size * 2) { } ~DiffRunningData() { } }; diff --git a/src/mecat2cns/main.cpp b/src/mecat2cns/main.cpp index 677e637..2163458 100644 --- a/src/mecat2cns/main.cpp +++ b/src/mecat2cns/main.cpp @@ -47,7 +47,7 @@ static void grid_start(const char* const prog, const ReadsCorrectionOptions &opt chmod(script_file.c_str(), S_IRUSR | S_IXUSR); std::string cmd(i == -1 && options.grid_options_split ? options.grid_options_split : options.grid_options); cmd += " " + name + " " + script_file; - assert(system(cmd.c_str())); + assert(system(cmd.c_str()) == 0); } // exit files get modified during loop @@ -98,7 +98,7 @@ static void merge_results(const char* const output, const std::list int main(int argc, char** argv) { ReadsCorrectionOptions rco; - if (parse_arguments(argc, argv, rco)) { + if (!parse_arguments(argc, argv, rco)) { print_usage(argv[0]); exit(1); } else if (rco.print_usage_info) { @@ -118,7 +118,7 @@ int main(int argc, char** argv) { } else { if (rco.input_type == INPUT_TYPE_CAN) { // this speeds up candidate starts - const idx_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); + const int64_t n_reads(PackedDB::convert_fasta_to_db(rco.reads, "fasta.db", rco.min_size)); if (rco.reads_to_correct <= 0 || n_reads < rco.reads_to_correct) { rco.reads_to_correct = n_reads; } diff --git a/src/mecat2cns/mecat2cns.mk b/src/mecat2cns/mecat2cns.mk index 8bd1a4c..abb9110 100644 --- a/src/mecat2cns/mecat2cns.mk +++ b/src/mecat2cns/mecat2cns.mk @@ -21,8 +21,8 @@ SRC_INCDIRS := . libboost # make sure large files are okay (and off_t is 8 bytes); # requires c++11 or higher for headers -TGT_CXXFLAGS := -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -march=native -flto -fno-fat-lto-objects -fno-builtin -TGT_LDFLAGS := -L${TARGET_DIR} ${TGT_CXXFLAGS} +TGT_CXXFLAGS := -static -D _FILE_OFFSET_BITS=64 -std=c++11 -O3 -mfpmath=sse -march=native -flto -fno-fat-lto-objects -fno-builtin -mmmx -msse -msse2 -mssse3 -msse4.1 -msse4.2 -mavx -maes -mpopcnt -mfxsr -mxsave -mxsaveopt +TGT_LDFLAGS := -static -L${TARGET_DIR} ${TGT_CXXFLAGS} TGT_LDLIBS := -lmecat TGT_PREREQS := libmecat.a diff --git a/src/mecat2cns/mecat_correction.cpp b/src/mecat2cns/mecat_correction.cpp index d8fa134..3fc1389 100644 --- a/src/mecat2cns/mecat_correction.cpp +++ b/src/mecat2cns/mecat_correction.cpp @@ -32,16 +32,16 @@ static inline uint8_t identify_one_consensus_item(const CnsTableItem& cns_item) struct CompareOverlapByOverlapSize { bool operator()(const Overlap& a, const Overlap& b) { - const idx_t ovlp_a(std::max(a.qend - a.qoff, a.send - a.soff)); - const idx_t ovlp_b(std::max(b.qend - b.qoff, b.send - b.soff)); + const int64_t ovlp_a(std::max(a.qend - a.qoff, a.send - a.soff)); + const int64_t ovlp_b(std::max(b.qend - b.qoff, b.send - b.soff)); return ovlp_b < ovlp_a; } }; -static void meap_add_one_aln(const std::string& qaln, const std::string& saln, idx_t start_soff, std::vector& cns_table) { +static void meap_add_one_aln(const std::string& qaln, const std::string& saln, int64_t start_soff, std::vector& cns_table) { assert(qaln.size() == saln.size()); - const idx_t aln_size(qaln.size()); - for (idx_t i(0); i < aln_size;) { + const int64_t aln_size(qaln.size()); + for (int64_t i(0); i < aln_size;) { const char q(qaln[i]); const char s(saln[i]); if (q == GAP_CHAR && s == GAP_CHAR) { // skip @@ -168,7 +168,7 @@ static void get_effective_ranges(std::vector& mranges, std::vector // breaks output sequence into chunks of no more than MaxSeqSize (if needed); // split chunks will have an OvlpSize overlap -static void output_cns_result(std::vector& cns_results, CnsResult& cr, const idx_t beg, const idx_t end, const std::string& cns_seq) { +static void output_cns_result(std::vector& cns_results, CnsResult& cr, const int64_t beg, const int64_t end, const std::string& cns_seq) { const size_t MaxSeqSize = 60000; const size_t OvlpSize = 10000; // BlkSize must be >= OvlpSize @@ -207,16 +207,16 @@ static inline bool check_ovlp_mapping_range(const int qb, const int qe, const in static void consensus_worker(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; - const idx_t min_size_95(ceil(0.95 * min_size)); + const int64_t min_size_95(ceil(0.95 * min_size)); std::string cns_seq; std::vector::const_iterator a(eranges.begin()); const std::vector::const_iterator end_a(eranges.end()); for (; a != end_a; ++a) { const int end_i(a->end); - for (idx_t i(a->start); i < end_i;) { + for (int64_t i(a->start); i < end_i;) { // find start of next high coverage area for (; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } - const idx_t start(i); + const int64_t start(i); // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start >= min_size_95) { @@ -229,7 +229,7 @@ static void consensus_worker(const std::vector& cns_table, std::ve } } -static void decode_and_append_sequence(std::string& s, const std::string& seq, idx_t i, const idx_t end_i) { +static void decode_and_append_sequence(std::string& s, const std::string& seq, int64_t i, const int64_t end_i) { s.reserve(s.size() + end_i - i); for (; i < end_i; ++i) { s += "ACGT"[static_cast(seq[i])]; @@ -242,7 +242,7 @@ static void decode_and_append_sequence(std::string& s, const std::string& seq, i static void consensus_worker_one_read(const std::vector& cns_table, std::vector& id_list, CnsAlns& cns_vec, std::string& aux_qstr, std::string& aux_tstr, const std::vector& eranges, const int min_cov, const int min_size, const int read_id, const std::string& tstr, std::vector& cns_results) { CnsResult cns_result; cns_result.id = read_id; - const idx_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); + const int64_t min_size_95(std::max(ceil(0.95 * min_size), 1) - 1); std::string cns_seq; std::vector::const_iterator a(eranges.begin()); const std::vector::const_iterator end_a(eranges.end()); @@ -255,16 +255,16 @@ static void consensus_worker_one_read(const std::vector& cns_table } const int begin_i(a->start - 1); const int end_i(a->end); - for (idx_t i(begin_i); i < end_i;) { + for (int64_t i(begin_i); i < end_i;) { // find start of next high coverage area - const idx_t last_end(i != begin_i ? i : a->start); + const int64_t last_end(i != begin_i ? i : a->start); for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt < min_cov; ++i) { } // add low coverage area as-is decode_and_append_sequence(cns_result.seq, tstr, last_end, i); if (i == end_i) { break; } - const idx_t start(i); + const int64_t start(i); // find end of high coverage area for (++i; i < end_i && cns_table[i].mat_cnt + cns_table[i].ins_cnt >= min_cov; ++i) { } if (i - start > min_size_95) { @@ -288,14 +288,14 @@ static void consensus_worker_one_read(const std::vector& cns_table cns_results.push_back(cns_result); } -void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { +void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const int64_t read_id, const int64_t sid, const int64_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidate* overlaps((ExtensionCandidate*)pctd.candidates); DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t read_size(overlaps[read_id].ssize); + const int64_t read_size(overlaps[read_id].ssize); std::string& qstr(pctd.query); std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); @@ -307,15 +307,15 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa std::vector& cns_table(pctd.cns_table); cns_table.assign(read_size, CnsTableItem()); // reset table cns_vec.clear(); - const idx_t L(sid); - const idx_t R(eid - sid <= max_added ? eid : L + max_added); + const int64_t L(sid); + const int64_t R(eid - sid <= max_added ? eid : L + max_added); if (eid - sid > max_added) { // only use largest max_added overlaps std::sort(overlaps + sid, overlaps + eid, CompareOverlapByOverlapSize()); } - for (idx_t i(L); i < R; ++i) { + for (int64_t i(L); i < R; ++i) { Overlap& ovlp(overlaps[i]); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); - const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); + const int64_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - ovlp.qext); const int r(GetAlignment(qstr, qext, tstr, ovlp.sext, drd, m5, error_rate, min_align_size)); if (r) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); @@ -329,14 +329,14 @@ void consensus_one_read_m4_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadDa consensus_worker(cns_table, pctd.id_list, cns_vec, nqstr, ntstr, eranges, ctd.rco.min_cov, ctd.rco.min_size, read_id, cns_results); } -void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { +void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const int64_t read_id, const int64_t sid, const int64_t eid) { PackedDB& reads = ctd.reads; ExtensionCandidate* overlaps = (ExtensionCandidate*)pctd.candidates; DiffRunningData& drd = pctd.drd; M5Record& m5 = pctd.m5; CnsAlns& cns_vec = pctd.cns_alns; std::vector& cns_results = pctd.cns_results; - const idx_t read_size = overlaps[read_id].ssize; + const int64_t read_size = overlaps[read_id].ssize; std::string& qstr = pctd.query; std::string& tstr = pctd.target; reads.GetSequence(read_id, 1, tstr); @@ -346,7 +346,7 @@ void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThread const double error_rate(ctd.rco.error_rate); const double min_mapping_ratio = ctd.rco.min_mapping_ratio - 0.02; - idx_t L, R; + int64_t L, R; if (eid - sid <= MAX_CNS_OVLPS) { L = sid; @@ -362,10 +362,10 @@ void consensus_one_read_m4_nanopore(ConsensusThreadData& ctd, ConsensusPerThread std::vector& cns_table = pctd.cns_table; cns_table.assign(read_size, CnsTableItem()); // reset table cns_vec.clear(); - for (idx_t i(L); i < R; ++i) { + for (int64_t i(L); i < R; ++i) { Overlap& ovlp(overlaps[i]); reads.GetSequence(ovlp.qid, ovlp.qdir == FWD, qstr); - const idx_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - qext); + const int64_t qext(ovlp.qdir == FWD ? ovlp.qext : ovlp.qsize - 1 - qext); const int r(GetAlignment(qstr, qext, tstr, ovlp.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, ovlp.qsize, m5.soff, m5.send, ovlp.ssize, min_mapping_ratio)) { normalize_gaps(m5.qaln, m5.saln, nqstr, ntstr, 1); @@ -405,14 +405,14 @@ static inline int check_cov_stats(std::vector& cov_stats, const int sof return 0; } -void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const idx_t read_id, const idx_t sid, idx_t eid) { +void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadData& pctd, const int64_t read_id, const int64_t sid, int64_t eid) { const PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t ssize(reads.read_size(read_id)); + const int64_t ssize(reads.read_size(read_id)); std::string& qstr(pctd.query); std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); @@ -427,17 +427,17 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD std::vector& cns_table(pctd.cns_table); cns_table.assign(ssize, CnsTableItem()); // reset table cns_vec.clear(); - std::set used_ids; + std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats id_list.clear(); - for (idx_t i(sid); i < eid && num_added < max_added; ++i) { + for (int64_t i(sid); i < eid && num_added < max_added; ++i) { const ExtensionCandidateCompressed& ec(candidates[i]); if (used_ids.find(ec.qid) != used_ids.end()) { continue; } - const idx_t qsize(reads.read_size(ec.qid)); + const int64_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); - const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); + const int64_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); const int r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { @@ -459,14 +459,14 @@ void consensus_one_read_can_pacbio(ConsensusThreadData& ctd, ConsensusPerThreadD } } -void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const idx_t read_id, const idx_t sid, const idx_t eid) { +void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThreadData &pctd, const int64_t read_id, const int64_t sid, const int64_t eid) { PackedDB& reads(ctd.reads); ExtensionCandidateCompressed* candidates((ExtensionCandidateCompressed*)pctd.candidates); DiffRunningData& drd(pctd.drd); M5Record& m5(pctd.m5); CnsAlns& cns_vec(pctd.cns_alns); std::vector& cns_results(pctd.cns_results); - const idx_t ssize(reads.read_size(read_id)); + const int64_t ssize(reads.read_size(read_id)); std::string& qstr(pctd.query); std::string& tstr(pctd.target); reads.GetSequence(read_id, 1, tstr); @@ -484,15 +484,15 @@ void consensus_one_read_can_nanopore(ConsensusThreadData& ctd, ConsensusPerThrea std::set used_ids; std::vector& id_list(pctd.id_list); // used to be called cov_stats id_list.clear(); - for (idx_t i(sid); i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { + for (int64_t i(sid); i < eid && num_added < MAX_CNS_OVLPS && num_ext < max_ext; ++i) { ++num_ext; const ExtensionCandidateCompressed& ec(candidates[i]); if (used_ids.find(ec.qid) != used_ids.end()) { continue; } - const idx_t qsize(reads.read_size(ec.qid)); + const int64_t qsize(reads.read_size(ec.qid)); reads.GetSequence(ec.qid, ec.qdir() == FWD, qstr); - const idx_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); + const int64_t qext(ec.qdir() == FWD ? ec.qext() : qsize - 1 - ec.qext()); const int r(GetAlignment(qstr, qext, tstr, ec.sext, drd, m5, error_rate, min_align_size)); if (r && check_ovlp_mapping_range(m5.qoff, m5.qend, qsize, m5.soff, m5.send, ssize, min_mapping_ratio)) { if (check_cov_stats(id_list, m5.soff, m5.send)) { diff --git a/src/mecat2cns/options.cpp b/src/mecat2cns/options.cpp index fd01dbe..e7da475 100644 --- a/src/mecat2cns/options.cpp +++ b/src/mecat2cns/options.cpp @@ -9,31 +9,28 @@ #include "../common/defs.h" // TECH_NANOPORE, TECH_PACBIO -static int input_type_pacbio = 1; -static int num_threads_pacbio = 1; -static double mapping_ratio_pacbio = 0.6; -static int align_size_pacbio = 1000; -static int cov_pacbio = 4; -static int min_size_pacbio = 2000; -static int print_usage_pacbio = 0; -static int tech_pacbio = TECH_PACBIO; -static double error_rate_pacbio = .15; +static const double mapping_ratio_pacbio = 0.6; +static const int align_size_pacbio = 1000; +static const int cov_pacbio = 4; +static const int min_size_pacbio = 2000; +static const int tech_pacbio = TECH_PACBIO; +static const double error_rate_pacbio = .15; -static int input_type_nanopore = 1; -static int num_threads_nanopore = 1; -static double mapping_ratio_nanopore = 0.4; -static int align_size_nanopore = 400; -static int cov_nanopore = 6; -static int min_size_nanopore = 2000; -static int print_usage_nanopore = 0; -static int tech_nanopore = TECH_NANOPORE; -static double error_rate_nanopore = .2; +static const double mapping_ratio_nanopore = 0.4; +static const int align_size_nanopore = 400; +static const int cov_nanopore = 6; +static const int min_size_nanopore = 2000; +static const int tech_nanopore = TECH_NANOPORE; +static const double error_rate_nanopore = .2; -static int default_tech = TECH_PACBIO; -static int num_partition_files = 0; -static int full_reads = 0; -static size_t read_buffer_size = 0; -static idx_t batch_size = idx_t(1) << 33; // 8 GB +static const int default_tech = TECH_PACBIO; +static const int num_partition_files = 0; +static const int full_reads = 0; +static const size_t read_buffer_size = 0; +static const size_t batch_size = size_t(1) << 33; // 8 GB +static const int print_usage_info = 0; +static const int input_type = 1; +static const int num_threads = 1; static const char input_type_n = 'i'; static const char num_threads_n = 't'; @@ -73,9 +70,7 @@ static size_t convert_integer(const std::string& s) { } static void print_pacbio_default_options() { - std::cerr << "-" << input_type_n << " " << input_type_pacbio - << " -" << num_threads_n << " " << num_threads_pacbio - << " -" << mapping_ratio_n << " " << mapping_ratio_pacbio + std::cerr << " -" << mapping_ratio_n << " " << mapping_ratio_pacbio << " -" << align_size_n << " " << align_size_pacbio << " -" << cov_n << " " << cov_pacbio << " " << " -" << min_size_n << " " << min_size_pacbio @@ -83,9 +78,7 @@ static void print_pacbio_default_options() { } static void print_nanopore_default_options() { - std::cerr << "-" << input_type_n << " " << input_type_nanopore - << " -" << num_threads_n << " " << num_threads_nanopore - << " -" << mapping_ratio_n << " " << mapping_ratio_nanopore + std::cerr << " -" << mapping_ratio_n << " " << mapping_ratio_nanopore << " -" << align_size_n << " " << align_size_nanopore << " -" << cov_n << " " << cov_nanopore << " -" << min_size_n << " " << min_size_nanopore @@ -187,24 +180,21 @@ ReadsCorrectionOptions init_consensus_options(const int tech) { t.full_reads = full_reads; t.read_buffer_size = read_buffer_size; t.batch_size = batch_size; + t.print_usage_info = print_usage_info; + t.input_type = input_type; + t.num_threads = num_threads; if (tech == TECH_PACBIO) { - t.input_type = input_type_pacbio; - t.num_threads = num_threads_pacbio; t.min_mapping_ratio = mapping_ratio_pacbio; t.min_align_size = align_size_pacbio; t.min_cov = cov_pacbio; t.min_size = min_size_pacbio; - t.print_usage_info = print_usage_pacbio; t.tech = tech_pacbio; t.error_rate = error_rate_pacbio; } else { - t.input_type = input_type_nanopore; - t.num_threads = num_threads_nanopore; t.min_mapping_ratio = mapping_ratio_nanopore; t.min_align_size = align_size_nanopore; t.min_cov = cov_nanopore; t.min_size = min_size_nanopore; - t.print_usage_info = print_usage_nanopore; t.tech = tech_nanopore; t.error_rate = error_rate_nanopore; } @@ -275,7 +265,7 @@ int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { t.min_size = convert_integer(optarg); break; case usage_n: - t.print_usage_info = true; + t.print_usage_info = 1; break; case grid_options_n: t.grid_options = optarg; @@ -334,7 +324,7 @@ int parse_arguments(int argc, char* argv[], ReadsCorrectionOptions& t) { return parse_success; } -// not in use at the moment +// not currently used #if 0 static void print_options(ReadsCorrectionOptions& t) { std::cout << "input_type:\t" << t.input_type << "\n"; diff --git a/src/mecat2cns/options.h b/src/mecat2cns/options.h index f64cf6e..489b91b 100644 --- a/src/mecat2cns/options.h +++ b/src/mecat2cns/options.h @@ -14,7 +14,7 @@ struct ReadsCorrectionOptions { const char* grid_options; const char* grid_options_split; int num_threads; - int batch_size; + size_t batch_size; double min_mapping_ratio; int min_align_size; int min_cov; diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index 19fdb47..ad17c61 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -25,10 +25,10 @@ inline static bool check_m4record_mapping_range(const M4Record& m4, const double return query_is_contained(m4, min_cov_ratio) || subject_is_contained(m4, min_cov_ratio); } -static idx_t get_qualified_m4record_counts(const char* const m4_file_name, const double min_cov_ratio) { +static int64_t get_qualified_m4record_counts(const char* const m4_file_name, const double min_cov_ratio) { std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); - idx_t num_reads(-1), num_records(0), num_qualified_records(0); + int64_t num_reads(-1), num_records(0), num_qualified_records(0); M4Record m4; m4qext(m4) = m4sext(m4) = INVALID_IDX; while (in >> m4) { @@ -49,7 +49,7 @@ static idx_t get_qualified_m4record_counts(const char* const m4_file_name, const // not in use at the moment #if 0 -static void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const idx_t num_reads, std::set& repeat_reads) { +static void get_repeat_reads(const char* const m4_file_name, const double min_cov_ratio, const int64_t num_reads, std::set& repeat_reads) { const int max_contained(100); // used to increment to a max of max_contained char count_table[max_contained + 1]; @@ -64,18 +64,18 @@ static void get_repeat_reads(const char* const m4_file_name, const double min_co m4qext(m4) = m4sext(m4) = INVALID_IDX; while (in >> m4) { if (query_is_contained(m4, min_cov_ratio)) { - const idx_t qid = m4qid(m4); + const int64_t qid = m4qid(m4); // increments count up to max_contained counts[qid] = count_table[static_cast(counts[qid])]; } if (subject_is_contained(m4, min_cov_ratio)) { - const idx_t sid = m4sid(m4); + const int64_t sid = m4sid(m4); // increments count up to max_contained counts[sid] = count_table[static_cast(counts[sid])]; } } close_fstream(in); - for (idx_t i(0); i < num_reads; ++i) { + for (int64_t i(0); i < num_reads; ++i) { if (counts[i] == max_contained) { std::cerr << "repeat read " << i << "\n"; repeat_reads.insert(i); @@ -95,7 +95,7 @@ static void generate_partition_file_name(const std::string& input_file_name, con ret = input_file_name + ".part" + os.str(); } -void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t batch_size, const int num_files, const idx_t num_reads) { +void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t batch_size, const int num_files, const int64_t num_reads) { DynamicTimer dtimer(__func__); struct stat buf; if (stat(input.c_str(), &buf) == -1) { @@ -103,12 +103,12 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi } // each candidate line takes on average 44 characters, but go with 32; // each one produces two candidates (forward and reverse) - const idx_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size)); + const int64_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size)); // separate them by read id on this pass, as we don't know how // many candidates each read is part of; we're assuming an even // distribution on average - const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); - std::vector read_sizes; + const int64_t reads_per_batch((num_reads + num_batches - 1) / num_batches); + std::vector read_sizes; PackedDB::read_sizes(pac_prefix, read_sizes); PartitionResultsWriter prw(num_files); int i(0); @@ -123,11 +123,11 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi // and here we go through the input file num_batches times, // being limited by the number of open output files we can have for (; i < num_batches; i += prw.kNumFiles) { - const idx_t sfid(i); - const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + const int64_t sfid(i); + const int64_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); - const idx_t L(sfid * reads_per_batch); - const idx_t R(std::min(efid * reads_per_batch, num_reads)); + const int64_t L(sfid * reads_per_batch); + const int64_t R(std::min(efid * reads_per_batch, num_reads)); std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); if (is_restart) { @@ -180,15 +180,15 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const size_t batch_size, const int min_read_size, const int num_files) { DynamicTimer dtimer(__func__); - idx_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); - std::set repeat_reads; + int64_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); + std::set repeat_reads; //get_repeat_reads(m4_file_name, min_cov_ratio, num_reads, repeat_reads); struct stat buf; if (stat(m4_file_name, &buf) == -1) { ERROR("Could not get file size: %s", m4_file_name); } - const idx_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidate) / batch_size)); - const idx_t reads_per_batch((num_reads + num_batches - 1) / num_batches); + const int64_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidate) / batch_size)); + const int64_t reads_per_batch((num_reads + num_batches - 1) / num_batches); std::string idx_file_name; generate_partition_index_file_name(m4_file_name, idx_file_name); std::ofstream idx_file; @@ -197,11 +197,11 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra ExtensionCandidate ec; PartitionResultsWriter prw(num_files); for (int i(0); i < num_batches; i += prw.kNumFiles) { - const idx_t sfid(i); - const idx_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + const int64_t sfid(i); + const int64_t efid(std::min(sfid + prw.kNumFiles, num_batches)); const int nf(efid - sfid); - const idx_t L(reads_per_batch * sfid); - const idx_t R(efid < num_batches ? reads_per_batch * efid : num_reads); + const int64_t L(reads_per_batch * sfid); + const int64_t R(efid < num_batches ? reads_per_batch * efid : num_reads); std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name, "partition.done"); diff --git a/src/mecat2cns/packed_db.cpp b/src/mecat2cns/packed_db.cpp index cdfd950..b4682db 100644 --- a/src/mecat2cns/packed_db.cpp +++ b/src/mecat2cns/packed_db.cpp @@ -17,9 +17,9 @@ void PackedDB::add_one_seq(const Sequence& seq) { if (max_read_size_ < seq.size()) { max_read_size_ = seq.size(); } - const idx_t needed_size(db_size + seq.size()); + const int64_t needed_size(db_size + seq.size()); if (max_db_size < needed_size) { - idx_t new_size(max_db_size > 1024 ? max_db_size : 1024); + int64_t new_size(max_db_size > 1024 ? max_db_size : 1024); for (; new_size < needed_size; new_size *= 2) { } uint8_t* const new_pac(new uint8_t[(new_size + 3) / 4]); memcpy(new_pac, pac_, (db_size + 3) / 4); @@ -30,7 +30,7 @@ void PackedDB::add_one_seq(const Sequence& seq) { const Sequence::str_t& org_seq(seq.sequence()); const uint8_t* const table(get_dna_encode_table()); unsigned int rand_char(0); // spread out unknown sequence in a repeatable fashion - for (idx_t i(0); i < seq.size(); ++i, ++db_size) { + for (int64_t i(0); i < seq.size(); ++i, ++db_size) { const uint8_t c(table[static_cast(org_seq[i])]); set_char(db_size, c < 4 ? c : ++rand_char & 3); } @@ -74,7 +74,7 @@ static void checkpoint_conversion(const std::string& ckpt_file, const std::strin } } -size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const idx_t min_size) { +size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string& output_prefix, const int64_t min_size) { const std::string pac_name(output_prefix + ".pac"); const std::string index_name(output_prefix + ".idx"); size_t read_count(0); @@ -117,7 +117,7 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string Sequence read; time_t next_checkpoint_time(time(0) + 300); for (;;) { - const idx_t rsize(fr.read_one_seq(read)); + const int64_t rsize(fr.read_one_seq(read)); if (rsize == -1) { break; } @@ -133,10 +133,10 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string continue; } Sequence::str_t& s(read.sequence()); - const idx_t rbytes((rsize + 3) / 4); + const int64_t rbytes((rsize + 3) / 4); // set_char uses | to set bits, so clear first buffer.assign(rbytes, 0); - for (idx_t i(0); i < rsize; ++i) { + for (int64_t i(0); i < rsize; ++i) { const uint8_t c(et[static_cast(s[i])]); set_char(buffer, i, c < 4 ? c : ++rand_char & 3); } @@ -169,7 +169,7 @@ size_t PackedDB::convert_fasta_to_db(const std::string& fasta, const std::string return read_count; } -void PackedDB::open_db(const std::string& path, const idx_t size) { +void PackedDB::open_db(const std::string& path, const int64_t size) { destroy(); const std::string pac_name(path + ".pac"); open_fstream(pstream, pac_name.c_str(), std::ios::in); @@ -177,7 +177,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { if (!pstream.seekg(-sizeof(size_t), std::ios_base::end)) { ERROR("Could not seek to end of fasta db to get size\n"); } - const idx_t file_size(pstream.tellg()); + const int64_t file_size(pstream.tellg()); max_db_size = size ? std::min(file_size, size) : file_size; if (max_db_size) { pac_ = new uint8_t[max_db_size]; @@ -220,7 +220,7 @@ void PackedDB::open_db(const std::string& path, const idx_t size) { } } -idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, const idx_t nec) { +int64_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, const int64_t nec) { if (!pstream.is_open()) { // all in memory already return nec; } @@ -233,23 +233,23 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co // // get set of reads to read in (most that will fit in memory); // use sorted set to speed reading (below) - std::set read_ids; - idx_t i(0), total_size(0); + std::set read_ids; + int64_t i(0), total_size(0); while (i != nec) { // find all alignments for a given read // (use set to handle duplicate qids) - const idx_t sid(ec_list[i].sid); - const idx_t start(i); - std::set my_ids; + const int64_t sid(ec_list[i].sid); + const int64_t start(i); + std::set my_ids; my_ids.insert(sid); my_ids.insert(ec_list[i].qid); for (++i; i != nec && ec_list[i].sid == sid; ++i) { my_ids.insert(ec_list[i].qid); } // find size of new read additions - idx_t size(0); - std::set::const_iterator a(my_ids.begin()); - const std::set::const_iterator end_a(my_ids.end()); + int64_t size(0); + std::set::const_iterator a(my_ids.begin()); + const std::set::const_iterator end_a(my_ids.end()); for (; a != end_a; ++a) { if (read_ids.find(*a) == read_ids.end()) { size += (seq_idx[*a].size + 3) / 4; @@ -269,9 +269,9 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co pac_ = new uint8_t[max_db_size]; } // now read in the reads - std::set::const_iterator a(read_ids.begin()); - const std::set::const_iterator end_a(read_ids.end()); - idx_t pos(0); + std::set::const_iterator a(read_ids.begin()); + const std::set::const_iterator end_a(read_ids.end()); + int64_t pos(0); off_t offset(-1); for (; a != end_a; ++a) { SeqIndex& si(seq_idx[*a]); @@ -279,7 +279,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co if (offset != si.file_offset && !pstream.seekg(si.file_offset)) { ERROR("Error seeking on fasta db"); } - const idx_t bytes((si.size + 3) / 4); + const int64_t bytes((si.size + 3) / 4); if (!pstream.read((char*)pac_ + pos, bytes)) { ERROR("Error reading fasta db"); } @@ -293,7 +293,7 @@ idx_t PackedDB::load_reads(const ExtensionCandidateCompressed* const ec_list, co return i; } -void PackedDB::read_sizes(const std::string& output_prefix, std::vector& sizes) { +void PackedDB::read_sizes(const std::string& output_prefix, std::vector& sizes) { const std::string pac_name(output_prefix + ".pac"); std::ifstream in(pac_name.c_str()); // pre-allocate index if possible @@ -311,7 +311,7 @@ void PackedDB::read_sizes(const std::string& output_prefix, std::vector& } const std::string index_name(output_prefix + ".idx"); open_fstream(in, index_name.c_str(), std::ios::in); - idx_t i, j; + int64_t i, j; while (in >> i >> j) { sizes.push_back(j); } diff --git a/src/mecat2cns/reads_correction_aux.cpp b/src/mecat2cns/reads_correction_aux.cpp index 741bdfa..3dab7cd 100644 --- a/src/mecat2cns/reads_correction_aux.cpp +++ b/src/mecat2cns/reads_correction_aux.cpp @@ -33,14 +33,14 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin // push gaps to the right, but not pass the end if (push) { - idx_t qlen = qnorm.size(); - idx_t tlen = tnorm.size(); - for (idx_t i = 0; i < qlen - 1; ++i) + int64_t qlen = qnorm.size(); + int64_t tlen = tnorm.size(); + for (int64_t i = 0; i < qlen - 1; ++i) { // push target gaps if (tnorm[i] == GAP_CHAR) { - idx_t j = i; + int64_t j = i; while (1) { const char c = tnorm[++j]; @@ -54,7 +54,7 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin // push query gaps if (qnorm[i] == GAP_CHAR) { - idx_t j = i; + int64_t j = i; while (1) { const char c = qnorm[++j]; @@ -80,12 +80,12 @@ void normalize_gaps(const std::string& qstr, const std::string& tstr, std::strin #endif } -void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, const idx_t nec) { +void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, const int64_t nec) { const int n(data.rco.num_threads); // split by number of ec's, rather than reads, since reads ids // are not contiguous and we could get empty lists - for (idx_t i(0), k(0); k != n; ++k) { - const idx_t start(i); + for (int64_t i(0), k(0); k != n; ++k) { + const int64_t start(i); // drop fractions here, as we'll likely add a few more ec's below i += (nec - i) / (n - k); if (i != nec) { // include all ec's for the last read @@ -97,12 +97,12 @@ void allocate_ecs(ConsensusThreadData& data, ExtensionCandidate* const ec_list, } } -void allocate_ecs(ConsensusThreadData& data, ExtensionCandidateCompressed* const ec_list, const idx_t nec) { - const idx_t n(data.rco.num_threads); +void allocate_ecs(ConsensusThreadData& data, ExtensionCandidateCompressed* const ec_list, const int64_t nec) { + const int64_t n(data.rco.num_threads); // split by number of ec's, rather than reads, since reads ids // are not contiguous and we could get empty lists - for (idx_t i(0), k(0); k != n; ++k) { - const idx_t start(i); + for (int64_t i(0), k(0); k != n; ++k) { + const int64_t start(i); // drop fractions here, as we'll likely add a few more ec's below i += (nec - i) / (n - k); if (i != nec) { // include all ec's for the last read diff --git a/src/mecat2cns/reads_correction_can.cpp b/src/mecat2cns/reads_correction_can.cpp index 647ca88..2abd8b8 100644 --- a/src/mecat2cns/reads_correction_can.cpp +++ b/src/mecat2cns/reads_correction_can.cpp @@ -18,11 +18,11 @@ static void* reads_correction_func_can(void* const arg) { const int tid(data.get_thread_id()); ConsensusPerThreadData& pdata(data.data[tid]); const ExtensionCandidateCompressed* const candidates((const ExtensionCandidateCompressed*)pdata.candidates); - idx_t i(pdata.next_candidate); + int64_t i(pdata.next_candidate); if (data.rco.tech == TECH_PACBIO) { while (i != pdata.num_candidates) { - const idx_t start(i); - const idx_t sid(candidates[start].sid); + const int64_t start(i); + const int64_t sid(candidates[start].sid); for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } // still have to check this here, as it's not always checked earlier if (i - start < data.rco.min_cov) { @@ -35,8 +35,8 @@ static void* reads_correction_func_can(void* const arg) { } } else { while (i != pdata.num_candidates) { - const idx_t start(i); - const idx_t sid(candidates[start].sid); + const int64_t start(i); + const int64_t sid(candidates[start].sid); for (++i; i != pdata.num_candidates && candidates[i].sid == sid; ++i) { } if (i - start < data.rco.min_cov) { continue; @@ -59,9 +59,9 @@ struct CmpExtensionCandidateCompressedBySid { class EC_Index { // offset into ec_list (and number of ecs) for each read id public: - idx_t offset, count; + int64_t offset, count; explicit EC_Index() : offset(0), count(0) { } - explicit EC_Index(idx_t i, idx_t j) : offset(i), count(j) { } + explicit EC_Index(int64_t i, int64_t j) : offset(i), count(j) { } ~EC_Index() { } }; @@ -70,20 +70,20 @@ class EC_Index { // offset into ec_list (and number of ecs) for each read id // candidates of reads with low coverage; we filter out alignments of // low-coverage reads, so nec may be reduced -static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_t& nec, const idx_t reads_to_correct, const idx_t num_reads, const int min_cov) { +static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, int64_t& nec, const int64_t reads_to_correct, const int64_t num_reads, const int min_cov) { // allow us to easily access a given sid's aligns std::sort(ec_list, ec_list + nec, CmpExtensionCandidateCompressedBySid()); - idx_t total_ec(0); + int64_t total_ec(0); // count will be zero for any read id beyond reads_to_correct std::vector index(reads_to_correct); // index existing list by sid - for (idx_t i(0); i != nec;) { + for (int64_t i(0); i != nec;) { // we are guaranteed sid is < reads_to_correct by partitioning - const idx_t sid(ec_list[i].sid); - const idx_t start(i); + const int64_t sid(ec_list[i].sid); + const int64_t start(i); for (++i; i != nec && ec_list[i].sid == sid; ++i) { } // make sure we have enough coverage - const idx_t count(i - start); + const int64_t count(i - start); if (count >= min_cov) { index[sid] = EC_Index(start, count); total_ec += count; @@ -91,9 +91,9 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ } // generate the new read order std::vector used(num_reads, 0); - std::vector new_order; + std::vector new_order; new_order.reserve(num_reads); // possible overestimate, but whatever - idx_t next_unused(0); + int64_t next_unused(0); size_t next_search(0); for (;;) { // skip over used reads, reads with no alignments @@ -105,14 +105,14 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ new_order.push_back(next_unused); // add all reads aligned to, and aligned to those, and so on for (; next_search != new_order.size(); ++next_search) { - const idx_t sid(new_order[next_search]); + const int64_t sid(new_order[next_search]); // make sure read has index entry if (sid < reads_to_correct) { const EC_Index& a(index[sid]); - idx_t i(a.offset); - const idx_t end_i(i + a.count); + int64_t i(a.offset); + const int64_t end_i(i + a.count); for (; i != end_i; ++i) { - const idx_t qid(ec_list[i].qid); + const int64_t qid(ec_list[i].qid); if (!used[qid]) { used[qid] = 1; new_order.push_back(qid); @@ -124,7 +124,7 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ // re-sort ec_list with new order (note that coverage-excluded // ec's will sort last, so we also change nec to exclude them); // sort reads not used (no new_order entry) at end - std::vector rid_to_order(num_reads, std::numeric_limits::max()); + std::vector rid_to_order(num_reads, std::numeric_limits::max()); for (size_t i(0); i != new_order.size(); ++i) { rid_to_order[new_order[i]] = i; } @@ -135,7 +135,7 @@ static void reorder_candidates(ExtensionCandidateCompressed* const ec_list, idx_ // load and sort partition data, assign to threads, start threads static void consensus_one_partition_can(const char* const m4_file_name, ConsensusThreadData& data) { - idx_t nec; + int64_t nec; ExtensionCandidateCompressed* ec_list(load_partition_data(m4_file_name, nec)); // if we're memory limited spend some cpu time to speed up passes // (~16s for ~300s speedup in test case) @@ -149,7 +149,7 @@ static void consensus_one_partition_can(const char* const m4_file_name, Consensu // see how many candidates we can run, given // how much read sequence we can load into memory // (unless we're not limited, in which case load 'em all) - const idx_t ecs(data.rco.read_buffer_size ? data.reads.load_reads(ec_list + data.ec_offset, nec - data.ec_offset) : nec); + const int64_t ecs(data.rco.read_buffer_size ? data.reads.load_reads(ec_list + data.ec_offset, nec - data.ec_offset) : nec); allocate_ecs(data, ec_list + data.ec_offset, ecs); for (int i(0); i != data.rco.num_threads; ++i) { pthread_create(&thread_ids[i], NULL, reads_correction_func_can, static_cast(&data)); @@ -187,7 +187,7 @@ static int reads_correction_can_p(ReadsCorrectionOptions& rco, std::vector(m4_file_name, nec); std::sort(ec_list, ec_list + nec, CmpExtensionCandidateBySidAndScore()); ConsensusThreadData data(rco, reads, out, m4_file_name); @@ -90,7 +90,7 @@ static int reads_correction_m4_p(ReadsCorrectionOptions& rco, std::vector Date: Thu, 20 Jun 2019 19:32:59 -0500 Subject: [PATCH 55/56] improved inner loop improved both the vectorized and non-vectorized string comparison in the inner loop; vectorization relies on sse2 gnu intrinsics and the bsfl/bsrl assembly commands vectorized version is roughly 10% faster --- src/mecat2cns/dw.cpp | 75 ++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/mecat2cns/dw.cpp b/src/mecat2cns/dw.cpp index f904b56..eed9a9d 100644 --- a/src/mecat2cns/dw.cpp +++ b/src/mecat2cns/dw.cpp @@ -7,6 +7,7 @@ #include // copy(), fill() #include // vector<> #include // uint8_t +#include #define GAP_VAL 4 @@ -18,31 +19,31 @@ // Define bit-scan-forward function. Gives index to lowest set bit #if defined (__GNUC__) || defined(__clang__) -static inline uint32_t bit_scan_forward(uint32_t a) __attribute__ ((pure)); -static inline uint32_t bit_scan_forward(uint32_t a) { +static inline uint32_t bit_scan_forward(const uint32_t a) __attribute__ ((pure)); +static inline uint32_t bit_scan_forward(const uint32_t a) { uint32_t r; __asm("bsfl %1, %0" : "=r"(r) : "r"(a) : ); return r; } #else -static inline uint32_t bit_scan_forward (uint32_t a) { +static inline uint32_t bit_scan_forward (const uint32_t a) { unsigned long r; _BitScanForward(&r, a); // defined in intrin.h for MS and Intel compilers return r; } #endif -// Define bit-scan-reverse function. Gives index to highest set bit. -// Make sure to mask unused high bits +// Define bit-scan-reverse function. Gives index to highest set bit +// (make sure to zero out unused high bits) #if defined (__GNUC__) || defined(__clang__) -static inline uint32_t bit_scan_reverse(uint32_t a) __attribute__ ((pure)); -static inline uint32_t bit_scan_reverse(uint32_t a) { +static inline uint32_t bit_scan_reverse(const uint32_t a) __attribute__ ((pure)); +static inline uint32_t bit_scan_reverse(const uint32_t a) { uint32_t r; __asm("bsrl %1, %0" : "=r"(r) : "r"(a) : ); return r; } #else -static inline uint32_t bit_scan_reverse (uint32_t a) { +static inline uint32_t bit_scan_reverse (const uint32_t a) { unsigned long r; _BitScanReverse(&r, a); // defined in intrin.h for MS and Intel compilers return r; @@ -151,37 +152,38 @@ static int Align(const int extend_size, const std::string& query, const int q_of // start of exact match const int q_start(q_pos), t_start(t_pos); // find the other end of exact match + const char* q_ptr(q_ptr_start + q_pos); + const char* t_ptr(t_ptr_start + t_pos); #ifdef __SSE2__ // vectorization of loop using gnu intrinsics __m128i a16, b16, c16; - const char* q_ptr(q_ptr_start + q_pos); - const char* t_ptr(t_ptr_start + t_pos); - int i(0); // round down to nearest multiple of 16 - int end_i((extend_size - std::max(q_pos, t_pos)) & (~0xf)); - for (; i != end_i; i += 16, q_ptr += 16, t_ptr += 16) { + const char* end_q(q_ptr + ((extend_size - std::max(q_pos, t_pos)) & (~0xf))); + for (; q_ptr != end_q; q_ptr += 16, t_ptr += 16) { + // load vector registers a16 = _mm_loadu_si128((const __m128i*)q_ptr); b16 = _mm_loadu_si128((const __m128i*)t_ptr); - c16 = _mm_cmpeq_epi8(a16, b16); + c16 = _mm_cmpeq_epi8(a16, b16); // compare registers + // create mask with 1 == true for each position const uint32_t x(_mm_movemask_epi8(c16)); + // if not all true, find first false if (x != 0xffff) { const int b(bit_scan_forward(~x)); q_ptr += b; t_ptr += b; - // no need to increment i here, it's - // sufficient that it doesn't equal end_i break; } } - if (i == end_i) { // deal with remainder - end_i += (extend_size - std::max(q_pos, t_pos)) & 0xf; - for (; i != end_i && *q_ptr == *t_ptr; ++i, ++q_ptr, ++t_ptr) { } + if (q_ptr == end_q) { // deal with remainder + end_q += (extend_size - std::max(q_pos, t_pos)) & 0xf; + for (; q_ptr != end_q && *q_ptr == *t_ptr; ++q_ptr, ++t_ptr) { } } - q_pos = q_ptr - q_ptr_start; - t_pos = t_ptr - t_ptr_start; #else - for (; q_pos < extend_size && t_pos < extend_size && query[q_offset + q_pos] == target[t_offset + t_pos]; ++q_pos, ++t_pos) { } + const char* const end_q(q_ptr + extend_size - std::max(q_pos, t_pos)); + for (; q_ptr != end_q && *q_ptr == *t_ptr; ++q_ptr, ++t_ptr) { } #endif // __SSE2__ + q_pos = q_ptr - q_ptr_start; + t_pos = t_ptr - t_ptr_start; d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); // see if we reached the end const int score(q_pos + t_pos); @@ -229,34 +231,39 @@ static int Align(const int extend_size, const std::string& query, const int q_of __m128i a16, b16, c16; const char* q_ptr(q_ptr_start - q_pos - 15); const char* t_ptr(t_ptr_start - t_pos - 15); - int i(0); // round down to nearest multiple of 16 - int end_i((extend_size - std::max(q_pos, t_pos)) & (~0xf)); - for (; i != end_i; i += 16, q_ptr -= 16, t_ptr -= 16) { + const char* end_q(q_ptr - ((extend_size - std::max(q_pos, t_pos)) & (~0xf))); + for (; q_ptr != end_q; q_ptr -= 16, t_ptr -= 16) { + // load vector registers a16 = _mm_loadu_si128((const __m128i*)q_ptr); b16 = _mm_loadu_si128((const __m128i*)t_ptr); - c16 = _mm_cmpeq_epi8(a16, b16); + c16 = _mm_cmpeq_epi8(a16, b16); // compare registers + // create mask with 1 == true for each position const uint32_t x(_mm_movemask_epi8(c16)); + // if not all true, find first false if (x != 0xffff) { // have to mask high bits const int b(bit_scan_reverse(0xffff & (~x))); // technically, += 15 - (15 - b) q_ptr += b; t_ptr += b; - // no need to increment i here, it's - // sufficient that it doesn't equal end_i break; } } - if (i == end_i) { // deal with remainder - end_i += (extend_size - std::max(q_pos, t_pos)) & 0xf; - for (q_ptr += 15, t_ptr += 15; i != end_i && *q_ptr == *t_ptr; ++i, --q_ptr, --t_ptr) { } + if (q_ptr == end_q) { // deal with remainder + q_ptr += 15; + t_ptr += 15; + end_q = q_ptr - ((extend_size - std::max(q_pos, t_pos)) & 0xf); + for (; q_ptr != end_q && *q_ptr == *t_ptr; --q_ptr, --t_ptr) { } } - q_pos = q_ptr_start - q_ptr; - t_pos = t_ptr_start - t_ptr; #else - for (; q_pos < extend_size && t_pos < extend_size && query[q_offset - q_pos] == target[t_offset - t_pos]; ++q_pos, ++t_pos) { } + const char* q_ptr(q_ptr_start - q_pos); + const char* t_ptr(t_ptr_start - t_pos); + const char* const end_q(q_ptr - extend_size + std::max(q_pos, t_pos)); + for (; q_ptr != end_q && *q_ptr == *t_ptr; --q_ptr, --t_ptr) { } #endif // __SSE2__ + q_pos = q_ptr_start - q_ptr; + t_pos = t_ptr_start - t_ptr; d_path[d_path_idx].set(q_start, t_start, q_pos, t_pos, pre_k); // see if we reached the end const int score(q_pos + t_pos); From 8e172fa5da385ae5fb921ba779adf25685d85c53 Mon Sep 17 00:00:00 2001 From: Dave Flowers Date: Mon, 24 Jun 2019 11:53:12 -0500 Subject: [PATCH 56/56] random cleanup some variables renamed to be more expressive, some int64_t changed to int --- src/common/alignment.h | 5 +-- src/mecat2cns/overlaps_partition.cpp | 54 ++++++++++++++-------------- src/mecat2cns/overlaps_store.h | 2 +- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/common/alignment.h b/src/common/alignment.h index 12595ec..bfefaa2 100644 --- a/src/common/alignment.h +++ b/src/common/alignment.h @@ -2,7 +2,6 @@ #define ALIGNMENT_H #include -#include // numeric_limits<> #include "defs.h" @@ -44,7 +43,9 @@ struct ExtensionCandidateCompressed { qext_ = a.sdir == a.qdir ? a.sext : a.sext | MSB_; score = a.score; } - // to check conversions from type int; need to use int64_t in case + // (these two are set in alignment.cpp, as std::numeric_limits<> + // can't be used at compile time) + // used to check conversions from type int; need to use int64_t in case // int is only int32_t in size (which wouldn't hold uint32_t max) static const int64_t max_value; // account for using MSB for qdir diff --git a/src/mecat2cns/overlaps_partition.cpp b/src/mecat2cns/overlaps_partition.cpp index ad17c61..ec560a5 100644 --- a/src/mecat2cns/overlaps_partition.cpp +++ b/src/mecat2cns/overlaps_partition.cpp @@ -95,7 +95,7 @@ static void generate_partition_file_name(const std::string& input_file_name, con ret = input_file_name + ".part" + os.str(); } -void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t batch_size, const int num_files, const int64_t num_reads) { +void partition_candidates(const std::string& input, const std::string& pac_prefix, const size_t file_size, const int max_files_per_batch, const int64_t num_reads) { DynamicTimer dtimer(__func__); struct stat buf; if (stat(input.c_str(), &buf) == -1) { @@ -103,16 +103,16 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi } // each candidate line takes on average 44 characters, but go with 32; // each one produces two candidates (forward and reverse) - const int64_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidateCompressed) / batch_size)); - // separate them by read id on this pass, as we don't know how - // many candidates each read is part of; we're assuming an even - // distribution on average - const int64_t reads_per_batch((num_reads + num_batches - 1) / num_batches); + const int num_files(ceil(double(buf.st_size / 32 * 2) * sizeof(ExtensionCandidateCompressed) / file_size)); + // separate them by read id as we don't know how many candidates each + // read is part of; we're assuming an even distribution on average + const int64_t reads_per_file((num_reads + num_files - 1) / num_files); std::vector read_sizes; PackedDB::read_sizes(pac_prefix, read_sizes); - PartitionResultsWriter prw(num_files); + PartitionResultsWriter prw(max_files_per_batch); int i(0); off_t input_pos; + // if restarting, both i and input_pos can be changed int is_restart(prw.restart(input, generate_partition_file_name, "partition.done", i, input_pos)); std::string idx_file_name; generate_partition_index_file_name(input, idx_file_name); @@ -120,14 +120,14 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); ExtensionCandidate ec; ExtensionCandidateCompressed nec; - // and here we go through the input file num_batches times, + // and here we go through the input file to write num_files, // being limited by the number of open output files we can have - for (; i < num_batches; i += prw.kNumFiles) { - const int64_t sfid(i); - const int64_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + for (; i < num_files; i += prw.kNumFiles) { + const int sfid(i); + const int efid(std::min(sfid + prw.kNumFiles, num_files)); const int nf(efid - sfid); - const int64_t L(sfid * reads_per_batch); - const int64_t R(std::min(efid * reads_per_batch, num_reads)); + const int64_t L(sfid * reads_per_file); + const int64_t R(std::min(efid * reads_per_file, num_reads)); std::ifstream in; open_fstream(in, input.c_str(), std::ios::in); if (is_restart) { @@ -150,18 +150,18 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi assert(ec.sid <= ExtensionCandidateCompressed::max_value); assert(ec.qid <= ExtensionCandidateCompressed::max_value); assert(ec.sext <= ExtensionCandidateCompressed::max_value); + assert(ec.score <= ExtensionCandidateCompressed::max_value); // qext is one bit smaller than the others assert(ec.qext <= ExtensionCandidateCompressed::max_qext); - assert(ec.score <= ExtensionCandidateCompressed::max_value); if (L <= ec.sid && ec.sid < R) { nec.set(ec); - if (prw.WriteOneResult((nec.sid - L) / reads_per_batch, nec.sid, nec)) { + if (prw.WriteOneResult((nec.sid - L) / reads_per_file, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } if (L <= ec.qid && ec.qid < R) { nec.set_swap(ec); - if (prw.WriteOneResult((nec.sid - L) / reads_per_batch, nec.sid, nec)) { + if (prw.WriteOneResult((nec.sid - L) / reads_per_file, nec.sid, nec)) { prw.checkpoint(in.tellg()); } } @@ -178,7 +178,7 @@ void partition_candidates(const std::string& input, const std::string& pac_prefi prw.finalize(); } -void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const size_t batch_size, const int min_read_size, const int num_files) { +void partition_m4records(const char* const m4_file_name, const double min_cov_ratio, const size_t file_size, const int min_read_size, const int max_files_per_batch) { DynamicTimer dtimer(__func__); int64_t num_reads(get_qualified_m4record_counts(m4_file_name, min_cov_ratio)); std::set repeat_reads; @@ -187,21 +187,21 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra if (stat(m4_file_name, &buf) == -1) { ERROR("Could not get file size: %s", m4_file_name); } - const int64_t num_batches(ceil(double(buf.st_size) / 32 * 2 * sizeof(ExtensionCandidate) / batch_size)); - const int64_t reads_per_batch((num_reads + num_batches - 1) / num_batches); + const int num_files(ceil(double(buf.st_size / 32 * 2) * sizeof(ExtensionCandidate) / file_size)); + const int64_t reads_per_file((num_reads + num_files - 1) / num_files); std::string idx_file_name; generate_partition_index_file_name(m4_file_name, idx_file_name); std::ofstream idx_file; open_fstream(idx_file, idx_file_name.c_str(), std::ios::out); M4Record m4, nm4; ExtensionCandidate ec; - PartitionResultsWriter prw(num_files); - for (int i(0); i < num_batches; i += prw.kNumFiles) { - const int64_t sfid(i); - const int64_t efid(std::min(sfid + prw.kNumFiles, num_batches)); + PartitionResultsWriter prw(max_files_per_batch); + for (int i(0); i < num_files; i += prw.kNumFiles) { + const int sfid(i); + const int efid(std::min(sfid + prw.kNumFiles, num_files)); const int nf(efid - sfid); - const int64_t L(reads_per_batch * sfid); - const int64_t R(efid < num_batches ? reads_per_batch * efid : num_reads); + const int64_t L(reads_per_file * sfid); + const int64_t R(efid < num_files ? reads_per_file * efid : num_reads); std::ifstream in; open_fstream(in, m4_file_name, std::ios::in); prw.OpenFiles(sfid, efid, m4_file_name, generate_partition_file_name, "partition.done"); @@ -216,12 +216,12 @@ void partition_m4records(const char* const m4_file_name, const double min_cov_ra if (m4qid(m4) >= L && m4qid(m4) < R) { normalize_m4record(m4, false, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4qid(m4) - L) / reads_per_batch , m4qid(m4), ec); + prw.WriteOneResult((m4qid(m4) - L) / reads_per_file, m4qid(m4), ec); } if (m4sid(m4) >= L && m4sid(m4) < R) { normalize_m4record(m4, true, nm4); m4_to_candidate(nm4, ec); - prw.WriteOneResult((m4sid(m4) - L) / reads_per_batch , m4sid(m4), ec); + prw.WriteOneResult((m4sid(m4) - L) / reads_per_file, m4sid(m4), ec); } } for (int k(0); k < nf; ++k) { diff --git a/src/mecat2cns/overlaps_store.h b/src/mecat2cns/overlaps_store.h index ebff986..e015ae1 100644 --- a/src/mecat2cns/overlaps_store.h +++ b/src/mecat2cns/overlaps_store.h @@ -25,7 +25,7 @@ template class PartitionResultsWriter { public: // can't make kNumFiles static, as sysconf() is run-time only; // leave room for stdin, stdout, stderr, a few others - explicit PartitionResultsWriter(const int num_files) : kNumFiles(num_files > 0 ? num_files : sysconf(_SC_OPEN_MAX) - 10), kStoreSize(0), num_open_files(0), results(0), files(0) { } + explicit PartitionResultsWriter(const int max_files_per_batch) : kNumFiles(max_files_per_batch > 0 ? max_files_per_batch : sysconf(_SC_OPEN_MAX) - 10), kStoreSize(0), num_open_files(0), results(0), files(0) { } ~PartitionResultsWriter() { CloseFiles(); }