From 5fa26361050c7054bffd76875d0aaf7fcaadc0f9 Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Tue, 2 Oct 2018 01:14:19 -0700 Subject: [PATCH 1/2] max batch length argument --- bilm/data.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bilm/data.py b/bilm/data.py index 386f2c6..14f8706 100644 --- a/bilm/data.py +++ b/bilm/data.py @@ -205,14 +205,17 @@ def __init__(self, lm_vocab_file: str, max_token_length: int): ) self._max_token_length = max_token_length - def batch_sentences(self, sentences: List[List[str]]): + def batch_sentences(self, sentences: List[List[str]], max_length=None): ''' Batch the sentences as character ids Each sentence is a list of tokens without or , e.g. [['The', 'first', 'sentence', '.'], ['Second', '.']] ''' n_sentences = len(sentences) - max_length = max(len(sentence) for sentence in sentences) + 2 + if max_length == None: + max_length = max(len(sentence) for sentence in sentences) + 2 + else: + max_length += 2 X_char_ids = np.zeros( (n_sentences, max_length, self._max_token_length), @@ -240,14 +243,17 @@ def __init__(self, lm_vocab_file: str): ''' self._lm_vocab = Vocabulary(lm_vocab_file) - def batch_sentences(self, sentences: List[List[str]]): + def batch_sentences(self, sentences: List[List[str]], max_length=None): ''' Batch the sentences as character ids Each sentence is a list of tokens without or , e.g. [['The', 'first', 'sentence', '.'], ['Second', '.']] ''' n_sentences = len(sentences) - max_length = max(len(sentence) for sentence in sentences) + 2 + if max_length == None: + max_length = max(len(sentence) for sentence in sentences) + 2 + else: + max_length += 2 X_ids = np.zeros((n_sentences, max_length), dtype=np.int64) From fbff27f0c461d17100a5948293eee426e3306b46 Mon Sep 17 00:00:00 2001 From: Vincent Liu <36426273+vliu15@users.noreply.github.com> Date: Wed, 31 Oct 2018 13:40:38 -0700 Subject: [PATCH 2/2] Create tqdm progress bar --- bilm/training.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bilm/training.py b/bilm/training.py index b9c022c..10e126d 100644 --- a/bilm/training.py +++ b/bilm/training.py @@ -11,6 +11,7 @@ import numpy as np from tensorflow.python.ops.init_ops import glorot_uniform_initializer +from tqdm import tqdm from .data import Vocabulary, UnicodeCharsVocabulary, InvalidNumberOfCharacters @@ -835,6 +836,7 @@ def train(options, data, n_gpus, tf_save_dir, tf_log_dir, t1 = time.time() data_gen = data.iter_batches(batch_size * n_gpus, unroll_steps) + pbar = tqdm(total=n_batches_total) for batch_no, batch in enumerate(data_gen, start=1): # slice the input in the batch for the feed_dict @@ -893,7 +895,9 @@ def train(options, data, n_gpus, tf_save_dir, tf_log_dir, if batch_no == n_batches_total: # done training! + pbar.close() break + pbar.update(1) def clip_by_global_norm_summary(t_list, clip_norm, norm_name, variables):