From 51865197416e0ad6a285445f901fe962a54a03d9 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 09:26:53 +0530
Subject: [PATCH 01/67] save alignments as file_id_start_end.json

---
 rename-alignments.py | 76 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 rename-alignments.py

diff --git a/rename-alignments.py b/rename-alignments.py
new file mode 100644
index 0000000..284450e
--- /dev/null
+++ b/rename-alignments.py
@@ -0,0 +1,76 @@
+import os
+import sys
+import re
+import multiprocessing
+import json
+import subprocess
+import hashlib
+import random
+import logging
+from shutil import copyfile
+
+logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
+logger = logging.getLogger("info_logger")
+
+def get_duration(audio_file):
+    """Determine the length of an audio file in seconds"""
+
+    duration = float(subprocess.Popen(
+                            ["soxi","-D","{}".format(audio_file)],
+                            stdout=subprocess.PIPE).stdout.read().strip())
+
+    return duration
+
+if __name__ == '__main__':
+
+    file_id = sys.argv[1]
+
+    # output
+    text_out_dir = "/home/aaron/data/deepspeech_data/stm"
+    wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
+    json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
+
+    # transcript
+    txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
+    mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
+    logger.info("Reading transcript {}...".format(file_id))
+
+    try:
+        with open(txt_file,"r") as tr:
+            transcript = tr.read()
+    except IOError:
+        logger.warning("File {} does not exist.".format(txt_file))
+        sys.exit()
+
+    # split transcript by speaker, and get timestamps (as seconds)
+    # of the boundaries of each paragraph
+    logger.info("Splitting transcript by speaker...")
+    paragraphs = []
+    times = []
+    for paragraph in transcript.split("\n"):
+        catch = re.match("\d:\d+:\d+\.\d",paragraph)
+        if catch:
+            timestamp = catch.group()
+            h,m,s = timestamp.split(":")
+            time = int(h)*60*60 + int(m)*60 + float(s)
+            paragraphs.append(paragraph)
+            times.append(time)
+    file_end = get_duration(mp3)
+    times.append(file_end)
+
+    total_captures,captures_dur = 0,0
+
+    for i,paragraph in enumerate(paragraphs):
+        # unique name of json object to read/write
+        paragraph_hash = hashlib.sha1("{}{}{}{}".format(
+                            file_id,paragraph,
+                            paragraph_start,paragraph_end)).hexdigest()
+        json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
+
+        if not os.path.isfile(json_file):
+            logger.info("JSON file with hash {} not found.".format(paragraph_hash))
+        else:
+            logger.info("Found JSON of paragraph {} -- skipping alignment and transcription by gentle".format(i))
+
+            new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
+            copyfile(json_file, new_json_file)

From b0b79c24f17b42b6c08692bc0671d228c76f7e40 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 09:43:05 +0530
Subject: [PATCH 02/67] paragraph start/end

---
 rename-alignments.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rename-alignments.py b/rename-alignments.py
index 284450e..ea490bb 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -61,6 +61,8 @@ def get_duration(audio_file):
     total_captures,captures_dur = 0,0
 
     for i,paragraph in enumerate(paragraphs):
+        paragraph_start, paragraph_end = times[i], times[i+1]
+
         # unique name of json object to read/write
         paragraph_hash = hashlib.sha1("{}{}{}{}".format(
                             file_id,paragraph,

From 492291aaf7aedc6f0a37af8b2fd9b9cc1a8aed9c Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 13:32:45 +0530
Subject: [PATCH 03/67] tuned guard and gap durations and added min utterance
 check

---
 aligner.py | 60 ++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/aligner.py b/aligner.py
index cb4df54..b63659f 100644
--- a/aligner.py
+++ b/aligner.py
@@ -7,6 +7,7 @@
 import hashlib
 import random
 import logging
+import argparse
 
 import boto3
 import gentle
@@ -14,6 +15,14 @@
 logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
 logger = logging.getLogger("info_logger")
 
+data_dir = '/home/aaron/data'
+records_dir = os.path.join(data_dir, 'records')
+mp3_dir = os.path.join(data_dir, 'mp3s')
+text_out_dir = os.path.join(data_dir, 'deepspeech_data/stm')
+wav_out_dir = os.path.join(data_dir, 'deepspeech_data/wav')
+json_out_dir = os.path.join(data_dir, 'deepspeech_data/alignments')
+use_filename_json = False
+
 def clean(text):
     """Clean transcript of timestamps and speaker trackings, metas,
     punctuation, and extra whitespace.
@@ -80,7 +89,7 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
     logger.info("Processing file id {}...".format(file_id))
 
     # grab audio file from s3
-    mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
+    mp3 = os.path.join(mp3_dir, "{}.mp3".format(file_id))
 
     if not os.path.isfile(mp3):
         bucket = boto3.resource("s3").Bucket("cgws")
@@ -91,14 +100,8 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
             logger.warning("Could not download file {} from S3.".format(file_id))
             return
 
-
-    # output
-    text_out_dir = "/home/aaron/data/deepspeech_data/stm"
-    wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
-    json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
-
     # transcript
-    txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
+    txt_file = os.path.join(records_dir, "{}.txt".format(file_id))
     logger.info("Reading transcript {}...".format(file_id))
     try:
         with open(txt_file,"r") as tr:
@@ -139,13 +142,16 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
             logger.info("Skipping paragraph {} (too few words)...".format(i))
             continue
 
-        temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"./temp")
+        temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
 
         # unique name of json object to read/write
-        paragraph_hash = hashlib.sha1("{}{}{}{}".format(
-                            file_id,paragraph,
-                            paragraph_start,paragraph_end)).hexdigest()
-        json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
+        if use_filename_json is True:
+            json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
+        else:
+            paragraph_hash = hashlib.sha1("{}{}{}{}".format(
+                                file_id,paragraph,
+                                paragraph_start,paragraph_end)).hexdigest()
+            json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
 
         result = None
 
@@ -201,14 +207,13 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
         # first two seconds will be skipped even if it contains a capture
         for catch in aligned["words"]:
-
             # successful capture
-            if catch["case"] == "success" and catch["alignedWord"] != "<unk>":
+            if catch["case"] == "success" and catch["alignedWord"] != "<unk>" and catch['start'] > 5 and catch['end'] - catch['start'] > .07:
 
                 # new capture group
                 if not current:
                     # begin capturing if it has been two seconds since the last word
-                    if catch["start"]-end_time > 2:
+                    if catch["start"]-end_time > 1:
                         current = [catch["alignedWord"]]
                         start_time = catch["start"]
                         end_time = catch["end"]
@@ -217,7 +222,7 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
                 else:
                     # large gap between last capture and this one
                     # likely that something was missing in the transcript
-                    if catch["start"]-end_time > .5:
+                    if catch["start"]-end_time > 1:
                         save_capture(captures,start_time,end_time,current)
                         current = []
 
@@ -274,9 +279,24 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
     # per-file logging
     total_dur = get_duration(mp3)
-    logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}."\
-          .format(total_captures,file_id,captures_dur,total_dur))
+    logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}, ratio {:.2f}."\
+          .format(total_captures,file_id,captures_dur,total_dur,captures_dur/total_dur))
 
     return
 
-
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Generate deepspeech data from Scribie transcripts')
+    parser.add_argument('file_id', type=str, help='file id to process')
+    parser.add_argument('--data_dir', type=str, help='path to data dir', default='/home/rajiv/host/align')
+    parser.add_argument('--use_filename_json', type=bool, help='read alignment json from filename_start_end.json file', default=True)
+    args = parser.parse_args()
+
+    data_dir = args.data_dir
+    records_dir = args.data_dir
+    mp3_dir = args.data_dir
+    text_out_dir = args.data_dir
+    wav_out_dir = args.data_dir
+    json_out_dir = args.data_dir
+    use_filename_json = True
+
+    data_generator(args.file_id)

From ed36dd1e4ca95b4866e7fcb2b2e80d66274bbfec Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 13:54:02 +0530
Subject: [PATCH 04/67] always calc paragraph hash

---
 aligner.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/aligner.py b/aligner.py
index b63659f..019ecfe 100644
--- a/aligner.py
+++ b/aligner.py
@@ -145,12 +145,13 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
         temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
 
         # unique name of json object to read/write
+        paragraph_hash = hashlib.sha1("{}{}{}{}".format(
+                            file_id,paragraph,
+                            paragraph_start,paragraph_end)).hexdigest()
+
         if use_filename_json is True:
             json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         else:
-            paragraph_hash = hashlib.sha1("{}{}{}{}".format(
-                                file_id,paragraph,
-                                paragraph_start,paragraph_end)).hexdigest()
             json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
 
         result = None

From e038cd31e9d59dc5567dec30219334bfd9b7b0ba Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 14:39:17 +0530
Subject: [PATCH 05/67] run gentle alignment if json file is not found

---
 rename-alignments.py | 80 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 74 insertions(+), 6 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index ea490bb..12a5ee1 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -8,10 +8,49 @@
 import random
 import logging
 from shutil import copyfile
+import argparse
+
+import boto3
+import gentle
+
+parser = argparse.ArgumentParser(description='Generate paragraph alignments from Scribie transcripts')
+parser.add_argument('file_id', type=str, help='file id to process')
+args = parser.parse_args()
 
 logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
 logger = logging.getLogger("info_logger")
 
+def clean(text):
+    """Clean transcript of timestamps and speaker trackings, metas,
+    punctuation, and extra whitespace.
+    """
+
+    text = re.sub("\d:\d+:\d+\.\d S(\d+|\?): ","",text)
+    text = re.sub("\[.+?\]","",text)
+    text = re.sub("\-"," ",text)
+    text = re.sub(r"[^a-zA-Z0-9\' ]","",text,re.UNICODE)
+    ### don't worry about converting to ascii for now
+    cleaned = re.sub("\s{2,}"," ",text)
+
+    return cleaned
+
+def trim(base_filename,audio_file,start,end,offset,out_directory):
+    """Write out a segment of an audio file to wav, based on start, end,
+    and offset times in seconds.
+    """
+
+    segment = os.path.join(out_directory,"{}_{}_{}.wav".format(
+                                    base_filename,
+                                   "{:07d}".format(int((offset+start)*100)),
+                                   "{:07d}".format(int((offset+end)*100))))
+
+    duration = end-start
+    subprocess.call(["sox","{}".format(audio_file),"-r","16k",
+                "{}".format(segment),"trim","{}".format(start),
+                "{}".format(duration),"remix","-"])
+
+    return segment
+
 def get_duration(audio_file):
     """Determine the length of an audio file in seconds"""
 
@@ -23,16 +62,19 @@ def get_duration(audio_file):
 
 if __name__ == '__main__':
 
-    file_id = sys.argv[1]
+    file_id = args.file_id
 
     # output
-    text_out_dir = "/home/aaron/data/deepspeech_data/stm"
     wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
     json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
-
-    # transcript
     txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
     mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
+
+    #wav_out_dir = "/home/rajiv/host/align/"
+    #json_out_dir = "/home/rajiv/host/align/"
+    #txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
+    #mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
+
     logger.info("Reading transcript {}...".format(file_id))
 
     try:
@@ -71,8 +113,34 @@ def get_duration(audio_file):
 
         if not os.path.isfile(json_file):
             logger.info("JSON file with hash {} not found.".format(paragraph_hash))
+
+            temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
+
+            try:
+                logger.info("Resampling paragraph {}...".format(i))
+                with gentle.resampled(temp_wav) as wav_file:
+                    resources = gentle.Resources()
+                    cleaned = clean(paragraph)
+                    logger.info("Aligning paragraph {} with gentle...".format(i))
+                    aligner = gentle.ForcedAligner(resources,cleaned,
+                                               nthreads=multiprocessing.cpu_count(),
+                                               disfluency=False,conservative=False,
+                                               disfluencies=set(["uh","um"]))
+                    logger.info("Transcribing audio segment {} with gentle...".format(i))
+                    result = aligner.transcribe(wav_file)
+
+                aligned_words = result.to_json()
+                with open(json_file,"w") as f:
+                    f.write(aligned_words)
+
+            except:
+                print(sys.exc_info())
+                sys.exit()
+                logger.warning("Paragraph {} - {} ".format(i,sys.exc_info()[2]))
+                os.remove(temp_wav)
+                continue
         else:
             logger.info("Found JSON of paragraph {} -- skipping alignment and transcription by gentle".format(i))
 
-            new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
-            copyfile(json_file, new_json_file)
+        new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
+        copyfile(json_file, new_json_file)

From 35c09263647aa716cd97e1572e719d7ed7189fdb Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 14:46:04 +0530
Subject: [PATCH 06/67] added duration check

---
 rename-alignments.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rename-alignments.py b/rename-alignments.py
index 12a5ee1..9664799 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -105,6 +105,9 @@ def get_duration(audio_file):
     for i,paragraph in enumerate(paragraphs):
         paragraph_start, paragraph_end = times[i], times[i+1]
 
+        if paragraph_end - paragraph_start <= 0:
+            continue
+
         # unique name of json object to read/write
         paragraph_hash = hashlib.sha1("{}{}{}{}".format(
                             file_id,paragraph,

From 697b37fdd847ec704daa47929e6ee0c39f204693 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 14:49:13 +0530
Subject: [PATCH 07/67] removed logs

---
 rename-alignments.py | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 9664799..1bcec08 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -6,7 +6,6 @@
 import subprocess
 import hashlib
 import random
-import logging
 from shutil import copyfile
 import argparse
 
@@ -17,9 +16,6 @@
 parser.add_argument('file_id', type=str, help='file id to process')
 args = parser.parse_args()
 
-logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
-logger = logging.getLogger("info_logger")
-
 def clean(text):
     """Clean transcript of timestamps and speaker trackings, metas,
     punctuation, and extra whitespace.
@@ -75,18 +71,15 @@ def get_duration(audio_file):
     #txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
     #mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
 
-    logger.info("Reading transcript {}...".format(file_id))
-
     try:
         with open(txt_file,"r") as tr:
             transcript = tr.read()
     except IOError:
-        logger.warning("File {} does not exist.".format(txt_file))
+        print("File {} does not exist.".format(txt_file))
         sys.exit()
 
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph
-    logger.info("Splitting transcript by speaker...")
     paragraphs = []
     times = []
     for paragraph in transcript.split("\n"):
@@ -115,21 +108,17 @@ def get_duration(audio_file):
         json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
 
         if not os.path.isfile(json_file):
-            logger.info("JSON file with hash {} not found.".format(paragraph_hash))
 
             temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
 
             try:
-                logger.info("Resampling paragraph {}...".format(i))
                 with gentle.resampled(temp_wav) as wav_file:
                     resources = gentle.Resources()
                     cleaned = clean(paragraph)
-                    logger.info("Aligning paragraph {} with gentle...".format(i))
                     aligner = gentle.ForcedAligner(resources,cleaned,
                                                nthreads=multiprocessing.cpu_count(),
                                                disfluency=False,conservative=False,
                                                disfluencies=set(["uh","um"]))
-                    logger.info("Transcribing audio segment {} with gentle...".format(i))
                     result = aligner.transcribe(wav_file)
 
                 aligned_words = result.to_json()
@@ -138,12 +127,8 @@ def get_duration(audio_file):
 
             except:
                 print(sys.exc_info())
-                sys.exit()
-                logger.warning("Paragraph {} - {} ".format(i,sys.exc_info()[2]))
                 os.remove(temp_wav)
                 continue
-        else:
-            logger.info("Found JSON of paragraph {} -- skipping alignment and transcription by gentle".format(i))
 
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         copyfile(json_file, new_json_file)

From 045756cf13eed109994ea6c86c3cd5575b43615e Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 15:08:37 +0530
Subject: [PATCH 08/67] remove sox output

---
 rename-alignments.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 1bcec08..0db7a3b 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -43,7 +43,7 @@ def trim(base_filename,audio_file,start,end,offset,out_directory):
     duration = end-start
     subprocess.call(["sox","{}".format(audio_file),"-r","16k",
                 "{}".format(segment),"trim","{}".format(start),
-                "{}".format(duration),"remix","-"])
+                "{}".format(duration),"remix","-", "1>/dev/null", "2>&1"])
 
     return segment
 
@@ -132,3 +132,5 @@ def get_duration(audio_file):
 
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         copyfile(json_file, new_json_file)
+
+        print("processed " + file_id)

From 025c233331a696b20bface4e04c50581d40e6fd7 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 15:14:02 +0530
Subject: [PATCH 09/67] redo last commit

---
 rename-alignments.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 0db7a3b..307b421 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -41,9 +41,10 @@ def trim(base_filename,audio_file,start,end,offset,out_directory):
                                    "{:07d}".format(int((offset+end)*100))))
 
     duration = end-start
+    FNULL = open(os.devnull, 'w')
     subprocess.call(["sox","{}".format(audio_file),"-r","16k",
                 "{}".format(segment),"trim","{}".format(start),
-                "{}".format(duration),"remix","-", "1>/dev/null", "2>&1"])
+                "{}".format(duration),"remix","-"], stdout=FNULL, stderr=FNULL)
 
     return segment
 
@@ -66,10 +67,12 @@ def get_duration(audio_file):
     txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
     mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
 
-    #wav_out_dir = "/home/rajiv/host/align/"
-    #json_out_dir = "/home/rajiv/host/align/"
-    #txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
-    #mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
+    '''
+    wav_out_dir = "/home/rajiv/host/align/"
+    json_out_dir = "/home/rajiv/host/align/"
+    txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
+    mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
+    '''
 
     try:
         with open(txt_file,"r") as tr:
@@ -133,4 +136,4 @@ def get_duration(audio_file):
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         copyfile(json_file, new_json_file)
 
-        print("processed " + file_id)
+    print("processed " + file_id)

From 0a5e701424c6f047d815fe61976b6320ec62fe15 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 16:17:51 +0530
Subject: [PATCH 10/67] download file if it does not exist

---
 rename-alignments.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/rename-alignments.py b/rename-alignments.py
index 307b421..737fcf5 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -81,6 +81,15 @@ def get_duration(audio_file):
         print("File {} does not exist.".format(txt_file))
         sys.exit()
 
+    if not os.path.isfile(mp3):
+        bucket = boto3.resource("s3").Bucket("cgws")
+        logger.info("Downloading file {} from S3...".format(file_id))
+        try:
+            bucket.download_file("{}.mp3".format(file_id),mp3)
+        except:
+            logger.warning("Could not download file {} from S3.".format(file_id))
+            return
+
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph
     paragraphs = []

From c6205d7532dc9870992869057bf1d19b15788bb1 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 16:19:42 +0530
Subject: [PATCH 11/67] removed logger

---
 rename-alignments.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 737fcf5..0dd8f54 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -87,8 +87,8 @@ def get_duration(audio_file):
         try:
             bucket.download_file("{}.mp3".format(file_id),mp3)
         except:
-            logger.warning("Could not download file {} from S3.".format(file_id))
-            return
+            print("Could not download file {} from S3.".format(file_id))
+            sys.exit()
 
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph

From 032f6ed0689a65f2d2b8c049f6848bf0b6319cd3 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 16:41:01 +0530
Subject: [PATCH 12/67] added tqdm

---
 rename-alignments.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 0dd8f54..19ce70f 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -8,6 +8,7 @@
 import random
 from shutil import copyfile
 import argparse
+from tqdm import tqdm
 
 import boto3
 import gentle
@@ -67,12 +68,10 @@ def get_duration(audio_file):
     txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
     mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
 
-    '''
     wav_out_dir = "/home/rajiv/host/align/"
     json_out_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
     mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
-    '''
 
     try:
         with open(txt_file,"r") as tr:
@@ -107,6 +106,7 @@ def get_duration(audio_file):
 
     total_captures,captures_dur = 0,0
 
+    pbar = tqdm(total=len(paragraphs))
     for i,paragraph in enumerate(paragraphs):
         paragraph_start, paragraph_end = times[i], times[i+1]
 
@@ -145,4 +145,7 @@ def get_duration(audio_file):
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         copyfile(json_file, new_json_file)
 
+        pbar.update(i)
+
     print("processed " + file_id)
+    pbar.close()

From 3accab250ac73a59ab383017f86a890f8a97fa1e Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 16:47:52 +0530
Subject: [PATCH 13/67] removed logger

---
 rename-alignments.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 19ce70f..d5cc0e2 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -82,7 +82,6 @@ def get_duration(audio_file):
 
     if not os.path.isfile(mp3):
         bucket = boto3.resource("s3").Bucket("cgws")
-        logger.info("Downloading file {} from S3...".format(file_id))
         try:
             bucket.download_file("{}.mp3".format(file_id),mp3)
         except:

From 6bb7b7c68e578392b0e2cd1ba16ecf346521915a Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 20:19:28 +0530
Subject: [PATCH 14/67] fixed tqdm

---
 rename-alignments.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index d5cc0e2..49ed9e2 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -68,10 +68,12 @@ def get_duration(audio_file):
     txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
     mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
 
+    '''
     wav_out_dir = "/home/rajiv/host/align/"
     json_out_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
     mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
+    '''
 
     try:
         with open(txt_file,"r") as tr:
@@ -103,10 +105,8 @@ def get_duration(audio_file):
     file_end = get_duration(mp3)
     times.append(file_end)
 
-    total_captures,captures_dur = 0,0
-
-    pbar = tqdm(total=len(paragraphs))
-    for i,paragraph in enumerate(paragraphs):
+    for i in tqdm(range(len(paragraphs)), desc=file_id, ncols=100):
+        paragraph = paragraphs[i]
         paragraph_start, paragraph_end = times[i], times[i+1]
 
         if paragraph_end - paragraph_start <= 0:
@@ -122,6 +122,9 @@ def get_duration(audio_file):
 
             temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
 
+            if not os.path.isfile(temp_wav):
+                continue
+
             try:
                 with gentle.resampled(temp_wav) as wav_file:
                     resources = gentle.Resources()
@@ -143,8 +146,3 @@ def get_duration(audio_file):
 
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         copyfile(json_file, new_json_file)
-
-        pbar.update(i)
-
-    print("processed " + file_id)
-    pbar.close()

From f735e5949798f05ba3095018d600b7ba4aba9fa8 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 20:19:41 +0530
Subject: [PATCH 15/67] check for file before trimming

---
 aligner.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/aligner.py b/aligner.py
index 019ecfe..29f4928 100644
--- a/aligner.py
+++ b/aligner.py
@@ -48,9 +48,11 @@ def trim(base_filename,audio_file,start,end,offset,out_directory):
                                    "{:07d}".format(int((offset+end)*100))))
 
     duration = end-start
-    subprocess.call(["sox","{}".format(audio_file),"-r","16k",
-                "{}".format(segment),"trim","{}".format(start),
-                "{}".format(duration),"remix","-"])
+
+    if not os.path.isfile(segment):
+        subprocess.call(["sox","{}".format(audio_file),"-r","16k",
+                    "{}".format(segment),"trim","{}".format(start),
+                    "{}".format(duration),"remix","-"])
 
     return segment
 

From 59eb6f41c0a843efd1a5933ebbb138eee27ad227 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 23:35:08 +0530
Subject: [PATCH 16/67] convert to wav once instead of multiple times

---
 aligner.py           | 10 +++++++++-
 rename-alignments.py |  8 +++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/aligner.py b/aligner.py
index 29f4928..b83ff91 100644
--- a/aligner.py
+++ b/aligner.py
@@ -102,6 +102,12 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
             logger.warning("Could not download file {} from S3.".format(file_id))
             return
 
+    wav = os.path.join("/tmp", "{}.wav".format(file_id))
+    if not os.path.isfile(wav):
+        subprocess.call(["sox","{}".format(mp3),"-r","16k",
+                    "{}".format(wav),
+                    "remix","-"])
+
     # transcript
     txt_file = os.path.join(records_dir, "{}.txt".format(file_id))
     logger.info("Reading transcript {}...".format(file_id))
@@ -144,7 +150,7 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
             logger.info("Skipping paragraph {} (too few words)...".format(i))
             continue
 
-        temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
+        temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
 
         # unique name of json object to read/write
         paragraph_hash = hashlib.sha1("{}{}{}{}".format(
@@ -280,6 +286,8 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
         # delete the clip of this speaker
         os.remove(temp_wav)
 
+    os.remove(wav)
+
     # per-file logging
     total_dur = get_duration(mp3)
     logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}, ratio {:.2f}."\
diff --git a/rename-alignments.py b/rename-alignments.py
index 49ed9e2..6ce5b0e 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -90,6 +90,12 @@ def get_duration(audio_file):
             print("Could not download file {} from S3.".format(file_id))
             sys.exit()
 
+    wav = os.path.join("/tmp", "{}.wav".format(file_id))
+    if not os.path.isfile(wav):
+        subprocess.call(["sox","{}".format(mp3),"-r","16k",
+                    "{}".format(wav),
+                    "remix","-"])
+
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph
     paragraphs = []
@@ -120,7 +126,7 @@ def get_duration(audio_file):
 
         if not os.path.isfile(json_file):
 
-            temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"/tmp")
+            temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
 
             if not os.path.isfile(temp_wav):
                 continue

From 92681bb704798973ec68f7ba634dd544562e5391 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 23:43:36 +0530
Subject: [PATCH 17/67] remove sox warnings

---
 rename-alignments.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 6ce5b0e..63800b1 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -92,9 +92,10 @@ def get_duration(audio_file):
 
     wav = os.path.join("/tmp", "{}.wav".format(file_id))
     if not os.path.isfile(wav):
+        FNULL = open(os.devnull, 'w')
         subprocess.call(["sox","{}".format(mp3),"-r","16k",
                     "{}".format(wav),
-                    "remix","-"])
+                    "remix","-"], stdout=FULL, stderr=FNULL)
 
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph

From 629dfeaddee2a4b9a78e6e1d3604d0a5ac610987 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 27 May 2017 23:45:04 +0530
Subject: [PATCH 18/67] ignore sox warnings

---
 aligner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/aligner.py b/aligner.py
index b83ff91..a6a4f49 100644
--- a/aligner.py
+++ b/aligner.py
@@ -104,9 +104,10 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
     wav = os.path.join("/tmp", "{}.wav".format(file_id))
     if not os.path.isfile(wav):
+        FNULL = open(os.devnull, 'w')
         subprocess.call(["sox","{}".format(mp3),"-r","16k",
                     "{}".format(wav),
-                    "remix","-"])
+                    "remix","-"], stdout=FNULL, stderr=FNULL)
 
     # transcript
     txt_file = os.path.join(records_dir, "{}.txt".format(file_id))

From ca19120a625887bc03df27de0a2b4d69b1ce130e Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 00:01:50 +0530
Subject: [PATCH 19/67] typo

---
 rename-alignments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 63800b1..8d07333 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -95,7 +95,7 @@ def get_duration(audio_file):
         FNULL = open(os.devnull, 'w')
         subprocess.call(["sox","{}".format(mp3),"-r","16k",
                     "{}".format(wav),
-                    "remix","-"], stdout=FULL, stderr=FNULL)
+                    "remix","-"], stdout=FNULL, stderr=FNULL)
 
     # split transcript by speaker, and get timestamps (as seconds)
     # of the boundaries of each paragraph

From 8a42a54cc406365ba6faafd2006c30e0d6569288 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 10:27:45 +0530
Subject: [PATCH 20/67] added merge ted, sort and split train/val

---
 .gitignore          |   2 +
 pytorch_manifest.py | 115 +++++++++++++++++++++++++++++++++++++-------
 2 files changed, 99 insertions(+), 18 deletions(-)

diff --git a/.gitignore b/.gitignore
index 89357a6..a24b5af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,3 +90,5 @@ ENV/
 
 # OSX
 .DS_Store
+
+*.swp
diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 2fc7546..fcd30e6 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -1,40 +1,119 @@
 import os
 import argparse
 import scipy.io.wavfile as wav
-
+import re
+import shutil
+from tqdm import tqdm
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--files_dir",default="/home/aaron/data/deepspeech_data",type=str)
-parser.add_argument("--out_file",default="./train_manifest.csv",type=str)
+parser.add_argument("--dst_dir",default=".",type=str)
+parser.add_argument("--merge_ted",default=True,type=bool)
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.files_dir)
 wav_dir = os.path.join(parent_dir,"wav")
 txt_dir = os.path.join(parent_dir,"stm")
 
+dst_dir = os.path.abspath(args.dst_dir)
+
 keep_files = []
 
-# get filenames from wav directory
-for i,filename in enumerate(os.listdir(wav_dir)):
-    if i % 10000 == 0:
-        print("Processing file {}".format(i))
+files = os.listdir(wav_dir)
+
+def get_duration(wav_file):
+    # duration (number of frames divided by framerate) greater than one second
+    samp_rate,data = wav.read(wav_file)
+    duration = len(data)/float(samp_rate)
+    return data, duration
+
+def sort_func(element):
+    return element[0]
 
+# get filenames wav directory
+for i in tqdm(range(len(files)), ncols=100, desc='Copying files'):
+    filename = files[i]
     fid = os.path.splitext(filename)[0]
 
     wav_file = os.path.join(wav_dir,"{}.wav".format(fid))
-    samp_rate,data = wav.read(wav_file)
+    try:
+        data, duration = get_duration(wav_file)
+    except:
+        print("skipping %s wav file read failed" % (fid))
+        continue
 
-    # duration (number of frames divided by framerate) greater than one second
-    if len(data)/float(samp_rate) >= 1:
+    if duration <= 2. or duration >= 20.:
+      continue
+
+    txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
+    with open(txt_file) as raw_text:
+        transcript = raw_text.read().strip()
+
+    if len(transcript) == 0:
+        continue 
+
+    transcript = re.sub('\s+', ' ', transcript)
+
+    # at least two words in transcript
+    num_words = len(transcript.split())
+    if num_words <= 1:
+        print("skipping %s as num word is %d" % (fid, num_words))
+        continue 
+
+    oov = re.search("[^a-zA-Z ']", transcript)
+    if oov is not None:
+        print("skipping %s due to oov, %s" % (fid, transcript))
+        continue 
+
+    dst_wav = os.path.join(dst_dir, "wav/{}.wav".format(fid))
+    if not os.path.isfile(dst_wav):
+        with open(dst_wav, 'w') as f:
+            f.write(data)
+    
+    dst_txt = os.path.join(dst_dir, "stm/{}.txt".format(fid))
+    if not os.path.isfile(dst_txt):
+        with open(dst_txt, 'w') as f:
+            f.write(transcript + "\n")
+
+    keep_files.append((duration, "{},{}".format(dst_wav,dst_txt)))
+
+train_len = int(len(keep_files) * 0.90)
+
+train_set = keep_files[:train_len]
+val_set = keep_files[train_len:]
+
+if args.merge_ted is True:
+    ted_train = []
+    for line in open("ted_train_manifest.csv","r"):
+        ted_train.append(line)
+
+    for i in tqdm(range(len(ted_train)), ncols=100, desc='Merging TED train'):
+        line = ted_train[i]
+        _, duration = get_duration(line.split(',')[0])
+        train_set.append((duration, line))
+
+    ted_val = []
+    for line in open("ted_test_manifest.csv","r"):
+        ted_val.append(line)
+
+    for i in tqdm(range(len(ted_val)), ncols=100, desc='Merging TED val'):
+        line = ted_val[i]
+        _, duration = get_duration(line.split(',')[0])
+        val_set.append((duration, line))
+
+train_set.sort(key=sort_func)
+val_set.sort(key=sort_func)
 
-        txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
-        with open(txt_file) as raw_text:
-            transcript = raw_text.read().strip()
+total_train = 0
+with open('train.csv', 'w') as f:
+    for line in train_set:
+        f.write((line[1].strip() + "\n").encode('utf-8'))
+        total_train += line[0]
 
-            # at least two words in transcript
-            if len(transcript.split()) > 1 :
-                keep_files.append("{},{}".format(wav_file,txt_file))
+total_val = 0
+with open('val.csv', 'w') as f:
+    for line in val_set:
+        f.write((line[1].strip() + "\n").encode('utf-8'))
+        total_val += line[0]
 
-# write out all acceptable files to the same file
-with open(args.out_file,"w") as out:
-    out.write("\n".join(keep_files))
+print("Train {:.2f} hours, Val {:.2f} hours".format(total_train/3600, total_val/3600))

From d901ad2c00e2328d686a47de994360b25e4df956 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 12:23:40 +0530
Subject: [PATCH 21/67] fixed paths

---
 pytorch_manifest.py | 51 +++++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index fcd30e6..00dc169 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -4,28 +4,38 @@
 import re
 import shutil
 from tqdm import tqdm
+import numpy as np
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--files_dir",default="/home/aaron/data/deepspeech_data",type=str)
-parser.add_argument("--dst_dir",default=".",type=str)
+parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
+parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
+parser.add_argument("--min_seconds",default="2",type=float, help="Cutoff for minimum duration")
+parser.add_argument("--max_seconds",default="20",type=float, help="Cutoff for maximum duration")
 parser.add_argument("--merge_ted",default=True,type=bool)
 args = parser.parse_args()
 
-parent_dir = os.path.abspath(args.files_dir)
+parent_dir = os.path.abspath(args.data_dir)
 wav_dir = os.path.join(parent_dir,"wav")
 txt_dir = os.path.join(parent_dir,"stm")
 
 dst_dir = os.path.abspath(args.dst_dir)
+dst_wav = os.path.join(dst_dir, "wav")
+if not os.path.exists(dst_wav):
+    os.makedirs(dst_wav)
+
+dst_txt = os.path.join(dst_dir, "stm")
+if not os.path.exists(dst_txt):
+    os.makedirs(dst_txt)
 
 keep_files = []
 
 files = os.listdir(wav_dir)
 
-def get_duration(wav_file):
+def read_wav(wav_file):
     # duration (number of frames divided by framerate) greater than one second
     samp_rate,data = wav.read(wav_file)
     duration = len(data)/float(samp_rate)
-    return data, duration
+    return data, samp_rate, duration
 
 def sort_func(element):
     return element[0]
@@ -37,12 +47,15 @@ def sort_func(element):
 
     wav_file = os.path.join(wav_dir,"{}.wav".format(fid))
     try:
-        data, duration = get_duration(wav_file)
+        data, samp_rate, duration = read_wav(wav_file)
     except:
         print("skipping %s wav file read failed" % (fid))
         continue
 
-    if duration <= 2. or duration >= 20.:
+    if samp_rate != 16000:
+        continue
+
+    if duration <= args.min_seconds or duration >= args.max_seconds:
       continue
 
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
@@ -65,17 +78,16 @@ def sort_func(element):
         print("skipping %s due to oov, %s" % (fid, transcript))
         continue 
 
-    dst_wav = os.path.join(dst_dir, "wav/{}.wav".format(fid))
-    if not os.path.isfile(dst_wav):
-        with open(dst_wav, 'w') as f:
-            f.write(data)
+    dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
+    if not os.path.isfile(dst_wav_file):
+        wav.write(dst_wav_file, samp_rate, data)
     
-    dst_txt = os.path.join(dst_dir, "stm/{}.txt".format(fid))
-    if not os.path.isfile(dst_txt):
-        with open(dst_txt, 'w') as f:
+    dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
+    if not os.path.isfile(dst_txt_file):
+        with open(dst_txt_file, 'w') as f:
             f.write(transcript + "\n")
 
-    keep_files.append((duration, "{},{}".format(dst_wav,dst_txt)))
+    keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
 train_len = int(len(keep_files) * 0.90)
 
@@ -89,7 +101,7 @@ def sort_func(element):
 
     for i in tqdm(range(len(ted_train)), ncols=100, desc='Merging TED train'):
         line = ted_train[i]
-        _, duration = get_duration(line.split(',')[0])
+        _, _, duration = read_wav(line.split(',')[0])
         train_set.append((duration, line))
 
     ted_val = []
@@ -98,7 +110,7 @@ def sort_func(element):
 
     for i in tqdm(range(len(ted_val)), ncols=100, desc='Merging TED val'):
         line = ted_val[i]
-        _, duration = get_duration(line.split(',')[0])
+        _, _, duration = read_wav(line.split(',')[0])
         val_set.append((duration, line))
 
 train_set.sort(key=sort_func)
@@ -116,4 +128,7 @@ def sort_func(element):
         f.write((line[1].strip() + "\n").encode('utf-8'))
         total_val += line[0]
 
-print("Train {:.2f} hours, Val {:.2f} hours".format(total_train/3600, total_val/3600))
+durations = [t[0] for t in train_set]
+np.save("./durations_binary.npy",np.asarray(durations))
+
+print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, ratio {:.2f}".format((total_train + total_val)/3600, total_train/3600, total_val/3600, total_val/total_train))

From b1dd0609f604319c935a0db8a52894647a09b7d9 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 22:12:02 +0530
Subject: [PATCH 22/67] using soundfile for durations

---
 .gitignore          |  1 +
 pytorch_manifest.py | 98 +++++++++++++++++++++++++--------------------
 2 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/.gitignore b/.gitignore
index a24b5af..bb7f401 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,3 +92,4 @@ ENV/
 .DS_Store
 
 *.swp
+inspect.sh
diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 00dc169..a665126 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -5,13 +5,21 @@
 import shutil
 from tqdm import tqdm
 import numpy as np
+import sys
+import subprocess
+import soundfile as sf
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
 parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
 parser.add_argument("--min_seconds",default="2",type=float, help="Cutoff for minimum duration")
 parser.add_argument("--max_seconds",default="20",type=float, help="Cutoff for maximum duration")
-parser.add_argument("--merge_ted",default=True,type=bool)
+parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
+parser.add_argument("--dry_run", help="Don't write csv's or copy files", action='store_true', default=False)
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.data_dir)
@@ -31,11 +39,9 @@
 
 files = os.listdir(wav_dir)
 
-def read_wav(wav_file):
-    # duration (number of frames divided by framerate) greater than one second
-    samp_rate,data = wav.read(wav_file)
-    duration = len(data)/float(samp_rate)
-    return data, samp_rate, duration
+def get_duration(wav_file):
+    f = sf.SoundFile(wav_file)
+    return float(len(f)/f.samplerate)
 
 def sort_func(element):
     return element[0]
@@ -46,46 +52,44 @@ def sort_func(element):
     fid = os.path.splitext(filename)[0]
 
     wav_file = os.path.join(wav_dir,"{}.wav".format(fid))
-    try:
-        data, samp_rate, duration = read_wav(wav_file)
-    except:
-        print("skipping %s wav file read failed" % (fid))
-        continue
+    dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
 
-    if samp_rate != 16000:
-        continue
+    if not os.path.isfile(dst_wav_file):
+        duration = get_duration(wav_file)
 
-    if duration <= args.min_seconds or duration >= args.max_seconds:
-      continue
+        if duration < args.min_seconds or duration > args.max_seconds:
+            continue
+
+        if args.dry_run is False:
+            shutil.copy2(wav_file, dst_wav_file)
+    else:
+        duration = get_duration(dst_wav_file)
 
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
-    with open(txt_file) as raw_text:
-        transcript = raw_text.read().strip()
+    dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
+    if not os.path.isfile(dst_txt_file):
+        with open(txt_file) as raw_text:
+            transcript = raw_text.read().strip()
 
-    if len(transcript) == 0:
-        continue 
+        if len(transcript) == 0:
+            continue 
 
-    transcript = re.sub('\s+', ' ', transcript)
+        transcript = re.sub('\s+', ' ', transcript)
 
-    # at least two words in transcript
-    num_words = len(transcript.split())
-    if num_words <= 1:
-        print("skipping %s as num word is %d" % (fid, num_words))
-        continue 
+        # at least two words in transcript
+        num_words = len(transcript.split())
+        if num_words <= 1:
+            print("skipping %s as num word is %d" % (fid, num_words))
+            continue 
 
-    oov = re.search("[^a-zA-Z ']", transcript)
-    if oov is not None:
-        print("skipping %s due to oov, %s" % (fid, transcript))
-        continue 
+        oov = re.search("[^a-zA-Z ']", transcript)
+        if oov is not None:
+            print("skipping %s due to oov, %s" % (fid, transcript))
+            continue 
 
-    dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
-    if not os.path.isfile(dst_wav_file):
-        wav.write(dst_wav_file, samp_rate, data)
-    
-    dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
-    if not os.path.isfile(dst_txt_file):
-        with open(dst_txt_file, 'w') as f:
-            f.write(transcript + "\n")
+        if args.dry_run is False:
+            with open(dst_txt_file, 'w') as f:
+                f.write(transcript + "\n")
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
@@ -94,14 +98,14 @@ def sort_func(element):
 train_set = keep_files[:train_len]
 val_set = keep_files[train_len:]
 
-if args.merge_ted is True:
+if args.no_ted is False:
     ted_train = []
     for line in open("ted_train_manifest.csv","r"):
         ted_train.append(line)
 
     for i in tqdm(range(len(ted_train)), ncols=100, desc='Merging TED train'):
         line = ted_train[i]
-        _, _, duration = read_wav(line.split(',')[0])
+        duration = get_duration(line.split(',')[0])
         train_set.append((duration, line))
 
     ted_val = []
@@ -110,7 +114,7 @@ def sort_func(element):
 
     for i in tqdm(range(len(ted_val)), ncols=100, desc='Merging TED val'):
         line = ted_val[i]
-        _, _, duration = read_wav(line.split(',')[0])
+        duration = get_duration(line.split(',')[0])
         val_set.append((duration, line))
 
 train_set.sort(key=sort_func)
@@ -119,16 +123,24 @@ def sort_func(element):
 total_train = 0
 with open('train.csv', 'w') as f:
     for line in train_set:
-        f.write((line[1].strip() + "\n").encode('utf-8'))
+        if args.dry_run is False:
+            f.write((line[1].strip() + "\n").encode('utf-8'))
         total_train += line[0]
 
 total_val = 0
 with open('val.csv', 'w') as f:
     for line in val_set:
-        f.write((line[1].strip() + "\n").encode('utf-8'))
+        if args.dry_run is False:
+            f.write((line[1].strip() + "\n").encode('utf-8'))
         total_val += line[0]
 
 durations = [t[0] for t in train_set]
-np.save("./durations_binary.npy",np.asarray(durations))
+h, b = np.histogram(durations, bins=np.arange(args.min_seconds, args.max_seconds + 1))
+plt.bar(np.arange(args.min_seconds + 1, args.max_seconds + 1), h, align='center')
+plt.xlabel('Seconds')
+plt.ylabel('# of files')
+plt.grid(color='gray', linestyle='dashed')
+plt.title('Durations distribution')
+plt.savefig('durations.png')
 
 print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, ratio {:.2f}".format((total_train + total_val)/3600, total_train/3600, total_val/3600, total_val/total_train))

From 4f09ec1d6de5ac3b35b89dc94ec7e106715b4c26 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 22:29:11 +0530
Subject: [PATCH 23/67] always check duration

---
 pytorch_manifest.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index a665126..51a588e 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -56,15 +56,15 @@ def sort_func(element):
 
     if not os.path.isfile(dst_wav_file):
         duration = get_duration(wav_file)
-
-        if duration < args.min_seconds or duration > args.max_seconds:
-            continue
-
-        if args.dry_run is False:
-            shutil.copy2(wav_file, dst_wav_file)
     else:
         duration = get_duration(dst_wav_file)
 
+    if duration < args.min_seconds or duration > args.max_seconds:
+        continue
+
+    if args.dry_run is False:
+        shutil.copy2(wav_file, dst_wav_file)
+
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
     dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
     if not os.path.isfile(dst_txt_file):

From 34dee33bcd057f3e8da1378ed3a7e74fb440f87f Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 23:14:37 +0530
Subject: [PATCH 24/67] fixed xticks

---
 pytorch_manifest.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 51a588e..3dfb3c3 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -140,6 +140,7 @@ def sort_func(element):
 plt.xlabel('Seconds')
 plt.ylabel('# of files')
 plt.grid(color='gray', linestyle='dashed')
+plt.xticks(np.arange(args.max_seconds + 1))
 plt.title('Durations distribution')
 plt.savefig('durations.png')
 

From 94bd1c602c4b70feb8a665448cfeee2b49f06cdf Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 23:14:54 +0530
Subject: [PATCH 25/67] save wav files to mp3 dir as well

---
 aligner.py           | 20 ++++++++++----------
 rename-alignments.py | 23 +++++++++++++----------
 2 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/aligner.py b/aligner.py
index a6a4f49..e61abf7 100644
--- a/aligner.py
+++ b/aligner.py
@@ -92,18 +92,18 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
     # grab audio file from s3
     mp3 = os.path.join(mp3_dir, "{}.mp3".format(file_id))
+    wav = os.path.join(mp3_dir, "{}.wav".format(file_id))
 
-    if not os.path.isfile(mp3):
-        bucket = boto3.resource("s3").Bucket("cgws")
-        logger.info("Downloading file {} from S3...".format(file_id))
-        try:
-            bucket.download_file("{}.mp3".format(file_id),mp3)
-        except:
-            logger.warning("Could not download file {} from S3.".format(file_id))
-            return
-
-    wav = os.path.join("/tmp", "{}.wav".format(file_id))
     if not os.path.isfile(wav):
+        if not os.path.isfile(mp3):
+            bucket = boto3.resource("s3").Bucket("cgws")
+            logger.info("Downloading file {} from S3...".format(file_id))
+            try:
+                bucket.download_file("{}.mp3".format(file_id),mp3)
+            except:
+                logger.warning("Could not download file {} from S3.".format(file_id))
+                return
+
         FNULL = open(os.devnull, 'w')
         subprocess.call(["sox","{}".format(mp3),"-r","16k",
                     "{}".format(wav),
diff --git a/rename-alignments.py b/rename-alignments.py
index 8d07333..16d3a95 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -65,16 +65,20 @@ def get_duration(audio_file):
     # output
     wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
     json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
+    mp3_dir = "/home/aaron/data/mp3s/"
     txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
-    mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
 
     '''
     wav_out_dir = "/home/rajiv/host/align/"
     json_out_dir = "/home/rajiv/host/align/"
+    mp3_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
     mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
     '''
 
+    mp3 = "{}/{}.mp3".format(mp3_dir,file_id)
+    wav = "{}/{}.wav".format(mp3_dir,file_id)
+
     try:
         with open(txt_file,"r") as tr:
             transcript = tr.read()
@@ -82,16 +86,15 @@ def get_duration(audio_file):
         print("File {} does not exist.".format(txt_file))
         sys.exit()
 
-    if not os.path.isfile(mp3):
-        bucket = boto3.resource("s3").Bucket("cgws")
-        try:
-            bucket.download_file("{}.mp3".format(file_id),mp3)
-        except:
-            print("Could not download file {} from S3.".format(file_id))
-            sys.exit()
-
-    wav = os.path.join("/tmp", "{}.wav".format(file_id))
     if not os.path.isfile(wav):
+        if not os.path.isfile(mp3):
+            bucket = boto3.resource("s3").Bucket("cgws")
+            try:
+                bucket.download_file("{}.mp3".format(file_id),mp3)
+            except:
+                print("Could not download file {} from S3.".format(file_id))
+                sys.exit()
+
         FNULL = open(os.devnull, 'w')
         subprocess.call(["sox","{}".format(mp3),"-r","16k",
                     "{}".format(wav),

From 2e3f42059fd5dc59de94557d581d3f8179de25d4 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 23:15:55 +0530
Subject: [PATCH 26/67] keep wav file for future runs

---
 aligner.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/aligner.py b/aligner.py
index e61abf7..9e33032 100644
--- a/aligner.py
+++ b/aligner.py
@@ -287,8 +287,6 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
         # delete the clip of this speaker
         os.remove(temp_wav)
 
-    os.remove(wav)
-
     # per-file logging
     total_dur = get_duration(mp3)
     logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}, ratio {:.2f}."\

From d0e971b421c1afc18c6957a986dbaf168d0788f6 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 28 May 2017 23:51:43 +0530
Subject: [PATCH 27/67] added split ratio

---
 pytorch_manifest.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 3dfb3c3..48474f6 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -20,6 +20,7 @@
 parser.add_argument("--max_seconds",default="20",type=float, help="Cutoff for maximum duration")
 parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
 parser.add_argument("--dry_run", help="Don't write csv's or copy files", action='store_true', default=False)
+parser.add_argument("--split_ratio", help="Percent of files to keep in val set", type=float, default=0.1)
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.data_dir)
@@ -47,7 +48,7 @@ def sort_func(element):
     return element[0]
 
 # get filenames wav directory
-for i in tqdm(range(len(files)), ncols=100, desc='Copying files'):
+for i in tqdm(range(len(files)), ncols=100, desc='Checking files'):
     filename = files[i]
     fid = os.path.splitext(filename)[0]
 
@@ -93,10 +94,10 @@ def sort_func(element):
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
-train_len = int(len(keep_files) * 0.90)
+val_len = int(len(keep_files) * args.split_ratio)
 
-train_set = keep_files[:train_len]
-val_set = keep_files[train_len:]
+train_set = keep_files[:-val_len]
+val_set = keep_files[-val_len:]
 
 if args.no_ted is False:
     ted_train = []

From 87405df5545f453dd61ce654d7363a6fbd11b87a Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Mon, 29 May 2017 11:41:06 +0530
Subject: [PATCH 28/67] copy files during dry runs

---
 pytorch_manifest.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 48474f6..97e5c08 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -19,7 +19,7 @@
 parser.add_argument("--min_seconds",default="2",type=float, help="Cutoff for minimum duration")
 parser.add_argument("--max_seconds",default="20",type=float, help="Cutoff for maximum duration")
 parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
-parser.add_argument("--dry_run", help="Don't write csv's or copy files", action='store_true', default=False)
+parser.add_argument("--dry_run", help="Don't write manifest csv's", action='store_true', default=False)
 parser.add_argument("--split_ratio", help="Percent of files to keep in val set", type=float, default=0.1)
 args = parser.parse_args()
 
@@ -48,7 +48,7 @@ def sort_func(element):
     return element[0]
 
 # get filenames wav directory
-for i in tqdm(range(len(files)), ncols=100, desc='Checking files'):
+for i in tqdm(range(len(files)), ncols=100, desc='Copying files'):
     filename = files[i]
     fid = os.path.splitext(filename)[0]
 
@@ -57,15 +57,13 @@ def sort_func(element):
 
     if not os.path.isfile(dst_wav_file):
         duration = get_duration(wav_file)
+        shutil.copy2(wav_file, dst_wav_file)
     else:
         duration = get_duration(dst_wav_file)
 
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
-    if args.dry_run is False:
-        shutil.copy2(wav_file, dst_wav_file)
-
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
     dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
     if not os.path.isfile(dst_txt_file):
@@ -88,9 +86,8 @@ def sort_func(element):
             print("skipping %s due to oov, %s" % (fid, transcript))
             continue 
 
-        if args.dry_run is False:
-            with open(dst_txt_file, 'w') as f:
-                f.write(transcript + "\n")
+        with open(dst_txt_file, 'w') as f:
+            f.write(transcript + "\n")
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 

From c70a38df2e80fe4134da1f8102800c04f6af73bf Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Mon, 29 May 2017 11:42:46 +0530
Subject: [PATCH 29/67] after testing

---
 rename-alignments.py | 59 ++++++++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 16d3a95..fc48049 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -9,6 +9,7 @@
 from shutil import copyfile
 import argparse
 from tqdm import tqdm
+import traceback
 
 import boto3
 import gentle
@@ -73,7 +74,6 @@ def get_duration(audio_file):
     json_out_dir = "/home/rajiv/host/align/"
     mp3_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
-    mp3 = "/home/rajiv/host/align/{}.mp3".format(file_id)
     '''
 
     mp3 = "{}/{}.mp3".format(mp3_dir,file_id)
@@ -127,32 +127,33 @@ def get_duration(audio_file):
                             file_id,paragraph,
                             paragraph_start,paragraph_end)).hexdigest()
         json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
-
-        if not os.path.isfile(json_file):
-
-            temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
-
-            if not os.path.isfile(temp_wav):
-                continue
-
-            try:
-                with gentle.resampled(temp_wav) as wav_file:
-                    resources = gentle.Resources()
-                    cleaned = clean(paragraph)
-                    aligner = gentle.ForcedAligner(resources,cleaned,
-                                               nthreads=multiprocessing.cpu_count(),
-                                               disfluency=False,conservative=False,
-                                               disfluencies=set(["uh","um"]))
-                    result = aligner.transcribe(wav_file)
-
-                aligned_words = result.to_json()
-                with open(json_file,"w") as f:
-                    f.write(aligned_words)
-
-            except:
-                print(sys.exc_info())
-                os.remove(temp_wav)
-                continue
-
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
-        copyfile(json_file, new_json_file)
+        if not os.path.isfile(new_json_file):
+            if not os.path.isfile(json_file):
+
+                temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
+
+                if not os.path.isfile(temp_wav):
+                    continue
+
+                try:
+                    with gentle.resampled(temp_wav) as wav_file:
+                        resources = gentle.Resources()
+                        cleaned = clean(paragraph)
+                        aligner = gentle.ForcedAligner(resources,cleaned,
+                                                   nthreads=multiprocessing.cpu_count(),
+                                                   disfluency=False,conservative=False,
+                                                   disfluencies=set(["uh","um"]))
+                        result = aligner.transcribe(wav_file)
+
+                    aligned_words = result.to_json()
+                    with open(json_file,"w") as f:
+                        f.write(aligned_words)
+
+                except:
+                    exc_type, exc_value, exc_traceback = sys.exc_info()
+                    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
+                    print ''.join('!! ' + line for line in lines)
+                    continue
+
+            copyfile(json_file, new_json_file)

From 0f794d97eaeb535e9246cf8f4d502261d4b5ae94 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Mon, 29 May 2017 12:22:07 +0530
Subject: [PATCH 30/67] dotted grid

---
 pytorch_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 97e5c08..a329cb5 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -137,7 +137,7 @@ def sort_func(element):
 plt.bar(np.arange(args.min_seconds + 1, args.max_seconds + 1), h, align='center')
 plt.xlabel('Seconds')
 plt.ylabel('# of files')
-plt.grid(color='gray', linestyle='dashed')
+plt.grid(color='gray', linestyle='dotted')
 plt.xticks(np.arange(args.max_seconds + 1))
 plt.title('Durations distribution')
 plt.savefig('durations.png')

From 0e9e29feb0f29f4d30f2081d7ed6524b172dc4a6 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 02:18:51 +0530
Subject: [PATCH 31/67] changed defaults, added train subset

---
 pytorch_manifest.py | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index a329cb5..81eae52 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -16,11 +16,11 @@
 parser = argparse.ArgumentParser()
 parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
 parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
-parser.add_argument("--min_seconds",default="2",type=float, help="Cutoff for minimum duration")
-parser.add_argument("--max_seconds",default="20",type=float, help="Cutoff for maximum duration")
+parser.add_argument("--min_seconds",default=0.25,type=float, help="Cutoff for minimum duration")
+parser.add_argument("--max_seconds",default=20.0,type=float, help="Cutoff for maximum duration")
 parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
 parser.add_argument("--dry_run", help="Don't write manifest csv's", action='store_true', default=False)
-parser.add_argument("--split_ratio", help="Percent of files to keep in val set", type=float, default=0.1)
+parser.add_argument("--split_ratio", help="Percent of files to keep in val set", type=float, default=0.01)
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.data_dir)
@@ -42,7 +42,10 @@
 
 def get_duration(wav_file):
     f = sf.SoundFile(wav_file)
-    return float(len(f)/f.samplerate)
+    if f.samplerate != 16000:
+        return 0
+    else:
+        return float(len(f)/f.samplerate)
 
 def sort_func(element):
     return element[0]
@@ -87,7 +90,7 @@ def sort_func(element):
             continue 
 
         with open(dst_txt_file, 'w') as f:
-            f.write(transcript + "\n")
+            f.write(transcript.upper() + "\n")
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
@@ -96,7 +99,7 @@ def sort_func(element):
 train_set = keep_files[:-val_len]
 val_set = keep_files[-val_len:]
 
-if args.no_ted is False:
+if not args.no_ted and not args.dry_run:
     ted_train = []
     for line in open("ted_train_manifest.csv","r"):
         ted_train.append(line)
@@ -118,19 +121,25 @@ def sort_func(element):
 train_set.sort(key=sort_func)
 val_set.sort(key=sort_func)
 
-total_train = 0
-with open('train.csv', 'w') as f:
-    for line in train_set:
-        if args.dry_run is False:
+total_train = sum([line[0] for line in train_set])
+if not args.dry_run:
+    with open('train.csv', 'w') as f:
+        for line in train_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
-        total_train += line[0]
 
-total_val = 0
-with open('val.csv', 'w') as f:
-    for line in val_set:
-        if args.dry_run is False:
+total_val = sum([line[0] for line in val_set])
+if not args.dry_run:
+    with open('val.csv', 'w') as f:
+        for line in val_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
-        total_val += line[0]
+
+if not args.dry_run:
+    # train_set has already been written so modifying in place is ok
+    np.random.shuffle(train_set)
+    train_subset = train_set[:len(val_set)]
+    with open('train_subset.csv', 'w') as f:
+        for line in train_subset:
+            f.write((line[1].strip() + "\n").encode("utf-8"))
 
 durations = [t[0] for t in train_set]
 h, b = np.histogram(durations, bins=np.arange(args.min_seconds, args.max_seconds + 1))

From 437e322714cf79c431c45aac5c7e15a7cb268319 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 02:39:10 +0530
Subject: [PATCH 32/67] added abort, remove gentle resampling

---
 rename-alignments.py | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index fc48049..035402d 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -14,8 +14,10 @@
 import boto3
 import gentle
 
-parser = argparse.ArgumentParser(description='Generate paragraph alignments from Scribie transcripts')
+parser = argparse.ArgumentParser(description='Generate paragraph level alignments from Scribie data')
 parser.add_argument('file_id', type=str, help='file id to process')
+parser.add_argument('--file_index', type=str, help='file index to print', default="1")
+parser.add_argument('--abort', help='Abort if alignemnt already exists', action='store_true', default=False)
 args = parser.parse_args()
 
 def clean(text):
@@ -64,6 +66,7 @@ def get_duration(audio_file):
     file_id = args.file_id
 
     # output
+    '''
     wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
     json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
     mp3_dir = "/home/aaron/data/mp3s/"
@@ -74,7 +77,6 @@ def get_duration(audio_file):
     json_out_dir = "/home/rajiv/host/align/"
     mp3_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
-    '''
 
     mp3 = "{}/{}.mp3".format(mp3_dir,file_id)
     wav = "{}/{}.wav".format(mp3_dir,file_id)
@@ -115,11 +117,11 @@ def get_duration(audio_file):
     file_end = get_duration(mp3)
     times.append(file_end)
 
-    for i in tqdm(range(len(paragraphs)), desc=file_id, ncols=100):
+    for i in tqdm(range(len(paragraphs)), desc="({}) {}".format(args.file_index, file_id), ncols=100):
         paragraph = paragraphs[i]
         paragraph_start, paragraph_end = times[i], times[i+1]
 
-        if paragraph_end - paragraph_start <= 0:
+        if paragraph_end - paragraph_start <= 0.2:
             continue
 
         # unique name of json object to read/write
@@ -128,7 +130,11 @@ def get_duration(audio_file):
                             paragraph_start,paragraph_end)).hexdigest()
         json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
-        if not os.path.isfile(new_json_file):
+        if os.path.isfile(new_json_file):
+            if args.abort:
+                print("aborting")
+                break
+        else:
             if not os.path.isfile(json_file):
 
                 temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
@@ -137,14 +143,13 @@ def get_duration(audio_file):
                     continue
 
                 try:
-                    with gentle.resampled(temp_wav) as wav_file:
-                        resources = gentle.Resources()
-                        cleaned = clean(paragraph)
-                        aligner = gentle.ForcedAligner(resources,cleaned,
-                                                   nthreads=multiprocessing.cpu_count(),
-                                                   disfluency=False,conservative=False,
-                                                   disfluencies=set(["uh","um"]))
-                        result = aligner.transcribe(wav_file)
+                    resources = gentle.Resources()
+                    cleaned = clean(paragraph)
+                    aligner = gentle.ForcedAligner(resources,cleaned,
+                                               nthreads=multiprocessing.cpu_count(),
+                                               disfluency=False,conservative=False,
+                                               disfluencies=set(["uh","um"]))
+                    result = aligner.transcribe(temp_wav)
 
                     aligned_words = result.to_json()
                     with open(json_file,"w") as f:
@@ -153,7 +158,7 @@ def get_duration(audio_file):
                 except:
                     exc_type, exc_value, exc_traceback = sys.exc_info()
                     lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
-                    print ''.join('!! ' + line for line in lines)
+                    print ''.join(line for line in lines)
                     continue
 
             copyfile(json_file, new_json_file)

From d40e0a939a99a9d065fbd3ed4187511ad16c597e Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 02:40:16 +0530
Subject: [PATCH 33/67] changed paths

---
 rename-alignments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 035402d..05f72e0 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -66,7 +66,6 @@ def get_duration(audio_file):
     file_id = args.file_id
 
     # output
-    '''
     wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
     json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
     mp3_dir = "/home/aaron/data/mp3s/"
@@ -77,6 +76,7 @@ def get_duration(audio_file):
     json_out_dir = "/home/rajiv/host/align/"
     mp3_dir = "/home/rajiv/host/align/"
     txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
+    '''
 
     mp3 = "{}/{}.mp3".format(mp3_dir,file_id)
     wav = "{}/{}.wav".format(mp3_dir,file_id)

From 5b68e04f963c02b0118c9af769505942c8bf0681 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 09:56:01 +0530
Subject: [PATCH 34/67] adding back gentle resampling

---
 rename-alignments.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 05f72e0..140f45e 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -143,17 +143,18 @@ def get_duration(audio_file):
                     continue
 
                 try:
-                    resources = gentle.Resources()
-                    cleaned = clean(paragraph)
-                    aligner = gentle.ForcedAligner(resources,cleaned,
-                                               nthreads=multiprocessing.cpu_count(),
-                                               disfluency=False,conservative=False,
-                                               disfluencies=set(["uh","um"]))
-                    result = aligner.transcribe(temp_wav)
-
-                    aligned_words = result.to_json()
-                    with open(json_file,"w") as f:
-                        f.write(aligned_words)
+                    with gentle.resampled(temp_wav) as wav_file:
+                        resources = gentle.Resources()
+                        cleaned = clean(paragraph)
+                        aligner = gentle.ForcedAligner(resources,cleaned,
+                                                   nthreads=multiprocessing.cpu_count(),
+                                                   disfluency=False,conservative=False,
+                                                   disfluencies=set(["uh","um"]))
+                        result = aligner.transcribe(wav_file)
+
+                        aligned_words = result.to_json()
+                        with open(json_file,"w") as f:
+                            f.write(aligned_words)
 
                 except:
                     exc_type, exc_value, exc_traceback = sys.exc_info()

From 1baf19ee7546ad37a13e697fdd51b58440e82ec1 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 13:07:14 +0530
Subject: [PATCH 35/67] fixed durations distributon

---
 pytorch_manifest.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 81eae52..f681ea1 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -16,7 +16,7 @@
 parser = argparse.ArgumentParser()
 parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
 parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
-parser.add_argument("--min_seconds",default=0.25,type=float, help="Cutoff for minimum duration")
+parser.add_argument("--min_seconds",default=1.0,type=float, help="Cutoff for minimum duration")
 parser.add_argument("--max_seconds",default=20.0,type=float, help="Cutoff for maximum duration")
 parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
 parser.add_argument("--dry_run", help="Don't write manifest csv's", action='store_true', default=False)
@@ -43,6 +43,7 @@
 def get_duration(wav_file):
     f = sf.SoundFile(wav_file)
     if f.samplerate != 16000:
+        print("sample rate is {}".format(f.samplerate))
         return 0
     else:
         return float(len(f)/f.samplerate)
@@ -141,14 +142,16 @@ def sort_func(element):
         for line in train_subset:
             f.write((line[1].strip() + "\n").encode("utf-8"))
 
+total = total_train + total_val
+
 durations = [t[0] for t in train_set]
-h, b = np.histogram(durations, bins=np.arange(args.min_seconds, args.max_seconds + 1))
-plt.bar(np.arange(args.min_seconds + 1, args.max_seconds + 1), h, align='center')
+bins = np.arange(int(args.min_seconds), int(args.max_seconds) + 1)
+plt.hist(durations, bins=bins, rwidth=0.8)
 plt.xlabel('Seconds')
 plt.ylabel('# of files')
 plt.grid(color='gray', linestyle='dotted')
-plt.xticks(np.arange(args.max_seconds + 1))
-plt.title('Durations distribution')
+plt.xticks(bins)
+plt.title("Durations distribution @ {} hours".format(int(total/3600)))
 plt.savefig('durations.png')
 
-print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, ratio {:.2f}".format((total_train + total_val)/3600, total_train/3600, total_val/3600, total_val/total_train))
+print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, ratio {:.2f}".format(total/3600, total_train/3600, total_val/3600, total_val/total_train))

From 7c29be39ca336d55115fdd9db15776c93bf95397 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 16:05:42 +0530
Subject: [PATCH 36/67] added threads multiplier

---
 rename-alignments.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 140f45e..1d43d42 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -18,6 +18,7 @@
 parser.add_argument('file_id', type=str, help='file id to process')
 parser.add_argument('--file_index', type=str, help='file index to print', default="1")
 parser.add_argument('--abort', help='Abort if alignemnt already exists', action='store_true', default=False)
+parser.add_argument('--threads_multiplier', help='Multiplier for threads', type=int, default=1)
 args = parser.parse_args()
 
 def clean(text):
@@ -132,7 +133,7 @@ def get_duration(audio_file):
         new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
         if os.path.isfile(new_json_file):
             if args.abort:
-                print("aborting")
+                print(" aborting")
                 break
         else:
             if not os.path.isfile(json_file):
@@ -147,7 +148,7 @@ def get_duration(audio_file):
                         resources = gentle.Resources()
                         cleaned = clean(paragraph)
                         aligner = gentle.ForcedAligner(resources,cleaned,
-                                                   nthreads=multiprocessing.cpu_count(),
+                                                   nthreads=multiprocessing.cpu_count()*args.threads_multiplier,
                                                    disfluency=False,conservative=False,
                                                    disfluencies=set(["uh","um"]))
                         result = aligner.transcribe(wav_file)

From 3957e2f4beb356b9df4b0763e75aa6058a755aea Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 19:50:37 +0530
Subject: [PATCH 37/67] configurable paths

---
 rename-alignments.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 1d43d42..4a1e530 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -19,6 +19,8 @@
 parser.add_argument('--file_index', type=str, help='file index to print', default="1")
 parser.add_argument('--abort', help='Abort if alignemnt already exists', action='store_true', default=False)
 parser.add_argument('--threads_multiplier', help='Multiplier for threads', type=int, default=1)
+parser.add_argument('--use_align_dir', help='Read/write files from an align  directory', action='store_true', default=False)
+parser.add_argument('--align_dir', help='Path to the align directory', type=str, default='~/align')
 args = parser.parse_args()
 
 def clean(text):
@@ -67,17 +69,14 @@ def get_duration(audio_file):
     file_id = args.file_id
 
     # output
-    wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
-    json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
-    mp3_dir = "/home/aaron/data/mp3s/"
-    txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
-
-    '''
-    wav_out_dir = "/home/rajiv/host/align/"
-    json_out_dir = "/home/rajiv/host/align/"
-    mp3_dir = "/home/rajiv/host/align/"
-    txt_file = "/home/rajiv/host/align/{}.txt".format(file_id)
-    '''
+    if args.use_align_dir:
+        align_dir = wav_out_dir = json_out_dir = mp3_dir = os.path.expanduser(args.align_dir)
+        txt_file = "{}/{}.txt".format(align_dir, file_id)
+    else:
+        wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
+        json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
+        mp3_dir = "/home/aaron/data/mp3s/"
+        txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
 
     mp3 = "{}/{}.mp3".format(mp3_dir,file_id)
     wav = "{}/{}.wav".format(mp3_dir,file_id)
@@ -163,4 +162,9 @@ def get_duration(audio_file):
                     print ''.join(line for line in lines)
                     continue
 
+            os.remove(temp_wav)
             copyfile(json_file, new_json_file)
+
+    if args.use_align_dir:
+        os.remove(wav)
+        os.remove(mp3)

From 539228cd21583096264160ce041b8affc3b6d23c Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 19:50:47 +0530
Subject: [PATCH 38/67] align scripts

---
 align-master.sh | 26 ++++++++++++++++++++
 align-worker.sh | 65 +++++++++++++++++++++++++++++++++++++++++++++++++
 worker-sync.sh  | 22 +++++++++++++++++
 3 files changed, 113 insertions(+)
 create mode 100755 align-master.sh
 create mode 100755 align-worker.sh
 create mode 100755 worker-sync.sh

diff --git a/align-master.sh b/align-master.sh
new file mode 100755
index 0000000..ebb6320
--- /dev/null
+++ b/align-master.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#set -x
+
+if [ $# -lt 2 ]
+then
+    echo "usage: ./align-master.sh filelist index"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+index=$2
+
+for fid in `tail -n +$index $filelist`
+do
+	touch ~/data/deepspeech_data/alignments/${fid}.json
+	index=`grep -n $fid $filelist | cut -d ':' -f1`
+	python rename-alignments.py $fid --file_index $index
+	python asr_data_gen.py --file $fid 2>>alignment.log
+done
diff --git a/align-worker.sh b/align-worker.sh
new file mode 100755
index 0000000..9ca0904
--- /dev/null
+++ b/align-worker.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+#set -x
+
+if [ $# -lt 2 ]
+then
+    echo "usage: ./align-worker.sh filelist index"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+if [ ! -d ~/align ]
+then
+    mkdir ~/align
+fi
+
+index=$2
+host=`hostname`
+
+threads_multiplier=4
+if [ $host == 'eesen-worker' ]
+then
+    threads_multiplier=2
+fi
+
+for fid in `tail -n +$index $filelist`
+do
+    host=`hostname`
+    if [ $host == 'eesen-worker' ]
+    then
+        stat ~/align/${fid}.json 1>/dev/null 2>&1
+    else
+        ssh eesen-worker "stat ~/align/${fid}.json" 1>/dev/null 2>&1
+    fi
+
+    if [ $? -ne 0 ]
+    then
+        index=`grep -n $fid $filelist | cut -d ':' -f1` 
+
+        if [ $host == 'eesen-worker' ]
+        then
+            touch ~/align/$1.json
+        else 
+            while true
+            do
+                ssh eesen-worker "touch ~/align/$1.json" 1>/dev/null 2>&1
+                if [ $? -ne 0 ]
+                then
+                    sleep 1
+                else
+                    break
+                fi
+            done
+        fi
+        
+        scp scribie:~/scribie/records/${fid}.txt ~/align 1>/dev/null 2>&1
+        python rename-alignments.py $fid --file_index $index --abort --threads_multiplier $threads_multiplier --use_align_dir 
+    fi
+done
diff --git a/worker-sync.sh b/worker-sync.sh
new file mode 100755
index 0000000..f4bc240
--- /dev/null
+++ b/worker-sync.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+inotifywait -mqr -e close_write "/home/rajiv/align" | while read path action file
+do
+	echo "$file" | grep "_" 1>/dev/null 2>&1
+	if [ $? -eq 0 ]
+	then
+		while true
+		do
+			scp -p /home/rajiv/align/$file eesen-worker:~/align/
+			if [ $? -eq 0 ]
+			then
+				break
+			else
+				sleep 1
+			fi
+		done
+
+        fid=`echo $file | cut -d '_' -f1`
+        timeout 90 rsync -T /tmp -avztuq --timeout=60 -e ssh eesen-worker:~/align/$fid*.json ~/align/
+	fi
+done

From 8635bcdc45e81ada6e505607d0925d2bd58c52be Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 3 Jun 2017 21:05:41 +0530
Subject: [PATCH 39/67] fixed touch

---
 align-worker.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/align-worker.sh b/align-worker.sh
index 9ca0904..9ee491d 100755
--- a/align-worker.sh
+++ b/align-worker.sh
@@ -45,11 +45,11 @@ do
 
         if [ $host == 'eesen-worker' ]
         then
-            touch ~/align/$1.json
+            touch ~/align/${fid}.json
         else 
             while true
             do
-                ssh eesen-worker "touch ~/align/$1.json" 1>/dev/null 2>&1
+                ssh eesen-worker "touch ~/align/${fid}.json" 1>/dev/null 2>&1
                 if [ $? -ne 0 ]
                 then
                     sleep 1

From 84bb53ce05b8ad7d47eae2902841ee176cf83b6d Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Tue, 6 Jun 2017 20:27:50 +0530
Subject: [PATCH 40/67] using find instead of listdir

---
 pytorch_manifest.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index f681ea1..ec3c34b 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -36,10 +36,6 @@
 if not os.path.exists(dst_txt):
     os.makedirs(dst_txt)
 
-keep_files = []
-
-files = os.listdir(wav_dir)
-
 def get_duration(wav_file):
     f = sf.SoundFile(wav_file)
     if f.samplerate != 16000:
@@ -51,8 +47,23 @@ def get_duration(wav_file):
 def sort_func(element):
     return element[0]
 
+keep_files = []
+files= []
+
+FNULL = open(os.devnull, 'w')
+out, err = subprocess.Popen("find " + wav_dir + " -type f | wc -l", stdout=subprocess.PIPE, shell=True).communicate()
+num_files = int(out)
+
+find = subprocess.Popen(["find", wav_dir, "-type", "f"], stdout=subprocess.PIPE, stderr=FNULL)
+for i in tqdm(range(num_files), ncols=100, desc='Finding files'):
+    line = find.stdout.readline()
+    if len(line.strip()) > 0:
+        files.append(os.path.basename(line))
+    else:
+        break
+
 # get filenames wav directory
-for i in tqdm(range(len(files)), ncols=100, desc='Copying files'):
+for i in tqdm(range(num_files), ncols=100, desc='Copying files'):
     filename = files[i]
     fid = os.path.splitext(filename)[0]
 

From 7d72c943cc9d8b790a151f68e9198b39a94851fe Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Tue, 6 Jun 2017 20:28:40 +0530
Subject: [PATCH 41/67] file exists check

---
 rename-alignments.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rename-alignments.py b/rename-alignments.py
index 4a1e530..db5911f 100644
--- a/rename-alignments.py
+++ b/rename-alignments.py
@@ -162,9 +162,11 @@ def get_duration(audio_file):
                     print ''.join(line for line in lines)
                     continue
 
-            os.remove(temp_wav)
             copyfile(json_file, new_json_file)
 
+            if os.path.isfile(temp_wav):
+                os.remove(temp_wav)
+
     if args.use_align_dir:
         os.remove(wav)
         os.remove(mp3)

From 52175f39653d0129a6a3296844ef22a8feb151f2 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 7 Jun 2017 10:58:54 +0530
Subject: [PATCH 42/67] ignore bad wav files

---
 pytorch_manifest.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index ec3c34b..b0c4d5b 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -37,12 +37,15 @@
     os.makedirs(dst_txt)
 
 def get_duration(wav_file):
-    f = sf.SoundFile(wav_file)
-    if f.samplerate != 16000:
-        print("sample rate is {}".format(f.samplerate))
+    try:
+        f = sf.SoundFile(wav_file)
+        if f.samplerate != 16000:
+            print("sample rate is {}".format(f.samplerate))
+            return 0
+        else:
+            return float(len(f)/f.samplerate)
+    except:
         return 0
-    else:
-        return float(len(f)/f.samplerate)
 
 def sort_func(element):
     return element[0]

From 90c85013607f59c410605b5a3995fdbad03ac373 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 18 Jun 2017 13:51:15 +0530
Subject: [PATCH 43/67] fixed duration calc

---
 pytorch_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index b0c4d5b..11a9ec7 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -43,7 +43,7 @@ def get_duration(wav_file):
             print("sample rate is {}".format(f.samplerate))
             return 0
         else:
-            return float(len(f)/f.samplerate)
+            return len(f)/float(f.samplerate)
     except:
         return 0
 

From adbf0a4f6359ee7642e7b06c09d032a98f55b0b6 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Tue, 20 Jun 2017 12:23:03 +0530
Subject: [PATCH 44/67] added test manifest generation

---
 pytorch_manifest.py | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 11a9ec7..0c78571 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -16,7 +16,7 @@
 parser = argparse.ArgumentParser()
 parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
 parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
-parser.add_argument("--min_seconds",default=1.0,type=float, help="Cutoff for minimum duration")
+parser.add_argument("--min_seconds",default=2.0,type=float, help="Cutoff for minimum duration")
 parser.add_argument("--max_seconds",default=20.0,type=float, help="Cutoff for maximum duration")
 parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
 parser.add_argument("--dry_run", help="Don't write manifest csv's", action='store_true', default=False)
@@ -75,10 +75,15 @@ def sort_func(element):
 
     if not os.path.isfile(dst_wav_file):
         duration = get_duration(wav_file)
-        shutil.copy2(wav_file, dst_wav_file)
+
+        if duration >= 2.0:
+            shutil.copy2(wav_file, dst_wav_file)
     else:
         duration = get_duration(dst_wav_file)
 
+        if duration < 2.0:
+            os.remove(dst_wav_file)
+
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
@@ -111,8 +116,9 @@ def sort_func(element):
 
 val_len = int(len(keep_files) * args.split_ratio)
 
-train_set = keep_files[:-val_len]
-val_set = keep_files[-val_len:]
+train_set = keep_files[:-val_len*2]
+val_set = keep_files[len(train_set):-val_len]
+test_set = keep_files[-val_len:]
 
 if not args.no_ted and not args.dry_run:
     ted_train = []
@@ -135,6 +141,7 @@ def sort_func(element):
 
 train_set.sort(key=sort_func)
 val_set.sort(key=sort_func)
+test_set.sort(key=sort_func)
 
 total_train = sum([line[0] for line in train_set])
 if not args.dry_run:
@@ -148,15 +155,21 @@ def sort_func(element):
         for line in val_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
 
+total_test = sum([line[0] for line in test_set])
+if not args.dry_run:
+    with open('test.csv', 'w') as f:
+        for line in test_set:
+            f.write((line[1].strip() + "\n").encode('utf-8'))
+
 if not args.dry_run:
     # train_set has already been written so modifying in place is ok
     np.random.shuffle(train_set)
-    train_subset = train_set[:len(val_set)]
+    train_subset = train_set[:len(val_set)/2]
     with open('train_subset.csv', 'w') as f:
         for line in train_subset:
             f.write((line[1].strip() + "\n").encode("utf-8"))
 
-total = total_train + total_val
+total = total_train + total_val + total_test
 
 durations = [t[0] for t in train_set]
 bins = np.arange(int(args.min_seconds), int(args.max_seconds) + 1)
@@ -168,4 +181,4 @@ def sort_func(element):
 plt.title("Durations distribution @ {} hours".format(int(total/3600)))
 plt.savefig('durations.png')
 
-print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, ratio {:.2f}".format(total/3600, total_train/3600, total_val/3600, total_val/total_train))
+print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, test {:.2f}, ratio {:.5f}/{:.5f}/{:.5f}".format(total/3600, total_train/3600, total_val/3600, total_test/3600, total_train/total, total_val/total, total_test/total))

From 9f62a14669939f6063f9a90352ff223cacc3b65f Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 9 Jul 2017 17:36:58 +0530
Subject: [PATCH 45/67] backward forward algo for segmenting

---
 check-files.sh     |  12 ++++++
 difficult_words.py | 102 +++++++++++++++++++++++++++++++--------------
 2 files changed, 83 insertions(+), 31 deletions(-)
 create mode 100755 check-files.sh

diff --git a/check-files.sh b/check-files.sh
new file mode 100755
index 0000000..9b462ba
--- /dev/null
+++ b/check-files.sh
@@ -0,0 +1,12 @@
+#!/bin/zsh
+for i in `ls *.txt | sort`
+do
+    echo $i
+    echo -en "\033[32m"
+    cat $i
+    echo -en "\033[0m"
+    echo "playing audio..."
+    play -q ${i:0:-3}wav 1>/dev/null 2>&1
+    echo "press enter to play next file"
+    read
+done
diff --git a/difficult_words.py b/difficult_words.py
index 621d5b1..54ff10b 100644
--- a/difficult_words.py
+++ b/difficult_words.py
@@ -1,13 +1,14 @@
+import sys
 import argparse
 import json
 import os
 import subprocess
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--file_id")
+parser.add_argument('file_id', type=str, help='file id to process')
+parser.add_argument('--window-len', type=str, dest='window_len', help='number of words to look around the mismatch', default=5)
 args = parser.parse_args()
 
-
 ctm_file = "".join((args.file_id,"_align.json"))
 audio_file = ".".join((args.file_id,"mp3"))
 
@@ -22,43 +23,82 @@ def trim(base_filename,audio_file,start,end,offset):
                                    "{:07d}".format(int((offset+start)*100)),
                                    "{:07d}".format(int((offset+end)*100))))
     duration = end-start
+    FNULL = open(os.devnull, 'w')
     subprocess.call(["sox","{}".format(audio_file),"-r","16k",
                 "{}".format(segment),"trim","{}".format(start),
-                "{}".format(duration),"remix","-"])
+                "{}".format(duration),"remix","-"], stdout=FNULL, stderr=FNULL)
 
     return
 
+def save_txt(base_filename,txt,start,end,offset):
+    txt_file = os.path.join(".","{}_{}_{}.txt".format(
+                                    base_filename,
+                                   "{:07d}".format(int((offset+start)*100)),
+                                   "{:07d}".format(int((offset+end)*100))))
+    with open(txt_file, 'w') as f:
+        f.write(txt + "\n")
 
 with open(ctm_file) as f:
-    ctm = json.loads(f.read())
+    ctms = json.loads(f.read())
     # could eventually put it in a single loop, like
     #for word in json.loads(f.read()):
 
-last_end = 0
-capturing = False
-for word in ctm:
-    if not capturing:
-        # start capturing
-        if word['case'] == 'mismatch':
-            clip_start = last_end #word['start']-(word['start']-last_end)/2.
-            last_end = word['end']
-            capturing = True
-
-        # just keep track of the end of this word
-        elif word['case'] == 'success': # else:
-            last_end = word['end']
-    else:
-        # capture this word too
-        if word['case'] == 'mismatch':
-            last_end = word['end']
-
-        # stop capturing and write segment if it's long enough
-        elif word['case'] == 'success': # else:
-            clip_end = word['start'] #word['start']-(word['start']-last_end)/2.
-            words = [word['word'] for word in ctm \
-                if word['start'] >= clip_start and word['end'] <= clip_end]
-            if len(words) > 2:
-                print(words)
-                trim(args.file_id,audio_file,clip_start,clip_end,0)
-            capturing = False
+last_capture_end = 0
+for index, ctm in enumerate(ctms):
+    if ctm['case'] == 'mismatch':
+        start_index = None
+        end_index = None
+        captures = []
+
+        # we don't want overlaps between the segments
+        if ctm['start'] < last_capture_end:
+            continue
+
+        if index - args.window_len < 0:
+            # most probably this is mismatch is at the start
+            continue
+
+        if index + args.window_len > len(ctms):
+            # most probably this is mismatch is towards the end
+            continue
+
+        # find the previous word which is a success with a decent gap
+        for i in range(index - args.window_len, 0, -1):
+            p = i - 1
+            if p < 0:
+                break;
+
+            gap = ctms[i]['start'] - ctms[p]['end']
+            if ctms[i]['case'] == 'success' and ctms[p]['case'] == 'success' and gap > 0.25 and ctms[p]['end'] > last_capture_end:
+                start_index = i
+                break;
+
+        if not start_index:
+            continue;
+
+        # find the next word which is a success with a decent gap
+        for i in range(index + args.window_len, len(ctms)):
+            n = i + 1
+            if n >= len(ctms):
+                break;
+
+            gap = ctms[n]['start'] - ctms[i]['end']
+            if ctms[i]['case'] == 'success' and ctms[n]['case'] == 'success' and gap > 0.25:
+                end_index = i
+                break;
+
+        if not end_index:
+            continue;
+
+        captures = ctms[start_index:end_index+1]
+
+        words = ' '.join([c['word'] for c in captures])
+        print("mistmatch captured at index {}, start_index {}, end_index {}, {}".format(index, start_index, end_index, words))
+
+        clip_start = captures[0]['start']
+        clip_end = captures[-1]['end']
+
+        trim(args.file_id,audio_file, clip_start, clip_end, 0)
+        save_txt(args.file_id, words, clip_start, clip_end, 0)
 
+        last_capture_end = clip_end

From ba37305a82d2b59c261cf3cfe472468820664a80 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 9 Jul 2017 18:12:40 +0530
Subject: [PATCH 46/67] discard segments with long gaps

---
 difficult_words.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/difficult_words.py b/difficult_words.py
index 54ff10b..bd59996 100644
--- a/difficult_words.py
+++ b/difficult_words.py
@@ -91,6 +91,20 @@ def save_txt(base_filename,txt,start,end,offset):
             continue;
 
         captures = ctms[start_index:end_index+1]
+        
+        # if the gap between the words is too long in the captured segment then we should skip it
+        # since it might be fillers or background conversation
+        for i, c in enumerate(captures):
+            if i+1 >= len(captures):
+                break;
+
+            if captures[i+1]['start'] - c['end'] > 0.75:
+                captures = []
+                break;
+
+        # discard too short segments
+        if len(captures) < 5:
+            continue;
 
         words = ' '.join([c['word'] for c in captures])
         print("mistmatch captured at index {}, start_index {}, end_index {}, {}".format(index, start_index, end_index, words))

From e441dd1941e0ad02fcf021ee2a149496f6e60d8f Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 9 Jul 2017 18:14:01 +0530
Subject: [PATCH 47/67] increased log gap to 2

---
 difficult_words.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/difficult_words.py b/difficult_words.py
index bd59996..6fb2ec1 100644
--- a/difficult_words.py
+++ b/difficult_words.py
@@ -98,7 +98,7 @@ def save_txt(base_filename,txt,start,end,offset):
             if i+1 >= len(captures):
                 break;
 
-            if captures[i+1]['start'] - c['end'] > 0.75:
+            if captures[i+1]['start'] - c['end'] > 2:
                 captures = []
                 break;
 

From 80694305f9c03f6ae67809d295cd0080ac047201 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 9 Jul 2017 21:31:55 +0530
Subject: [PATCH 48/67] printing ratio

---
 difficult_words.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/difficult_words.py b/difficult_words.py
index 6fb2ec1..8fa9383 100644
--- a/difficult_words.py
+++ b/difficult_words.py
@@ -43,6 +43,7 @@ def save_txt(base_filename,txt,start,end,offset):
     # could eventually put it in a single loop, like
     #for word in json.loads(f.read()):
 
+total_seconds = 0
 last_capture_end = 0
 for index, ctm in enumerate(ctms):
     if ctm['case'] == 'mismatch':
@@ -112,7 +113,11 @@ def save_txt(base_filename,txt,start,end,offset):
         clip_start = captures[0]['start']
         clip_end = captures[-1]['end']
 
+        total_seconds += clip_end - clip_start
+
         trim(args.file_id,audio_file, clip_start, clip_end, 0)
         save_txt(args.file_id, words, clip_start, clip_end, 0)
 
         last_capture_end = clip_end
+
+print("\nratio {:.2f}".format(total_seconds/(ctms[-1]['end'] - ctms[0]['start'])))

From 5986f4de20334467b997c39d3340c6477aef5e79 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Tue, 11 Jul 2017 21:50:20 +0530
Subject: [PATCH 49/67] added gaps algo file

---
 gaps.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 gaps.py

diff --git a/gaps.py b/gaps.py
new file mode 100644
index 0000000..e8fb618
--- /dev/null
+++ b/gaps.py
@@ -0,0 +1,80 @@
+import argparse
+import json
+import os
+import subprocess
+import re
+from tqdm import tqdm
+import logging
+
+parser = argparse.ArgumentParser()
+parser.add_argument('file_id', type=str, help='file id to process')
+parser.add_argument("--min-gap", default=0.25, type=float, dest="min_gap", help="minimum gap between words to use for splitting")
+parser.add_argument('--file-index', type=str, default="1", dest="file_index", help='file index to print')
+parser.add_argument('--audio-dir', type=str, dest="audio_dir", default='.', help='Path to the directory containing audio files')
+parser.add_argument('--align-dir', type=str, dest="align_dir", default=".", help='Path to the directory containing alignments')
+parser.add_argument('--dataset-dir', type=str, dest="dataset_dir", default='.', help='Path to the dataset directory')
+args = parser.parse_args()
+
+logging.basicConfig(filename='gaps.log', level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+logger = logging.getLogger("info_logger")
+
+# temporarily running them from "others/"
+ctm_file = os.path.join(args.align_dir, args.file_id + "_align.json")
+audio_file = os.path.join(args.audio_dir, args.file_id + ".mp3")
+
+# leaving offset here in case the algo changes to require it
+def sox_trim(start, end):
+    """Write out a segment of an audio file to wav, based on start, end,
+    """
+    FNULL = open("/dev/null")
+    wav_file = os.path.join(args.dataset_dir, "wav/{}_{:07d}_{:07d}.wav".format(args.file_id, int(start*100), int(end*100))) 
+    subprocess.call(["sox", audio_file, "-r", "16k", wav_file, "trim", str(start), str(end - start), "remix", "-"], stdout=FNULL, stderr=FNULL)
+
+with open(ctm_file) as f:
+    ctms = json.loads(f.read())
+
+    null_word = {'start': 0, 'end':0}
+    gaps = [second['start']-first['end'] for first, second in zip([null_word]+ctms, ctms)]
+
+    # we split from one good gap to the next
+    # a good gap is when the silence between the words is long
+    # and the word *itself* is long and is not a mismatch
+    good_gaps = [(i, gap) for i, gap in enumerate(gaps) if gap > args.min_gap and ctms[i]['duration'] > args.min_gap and ctms[i]['case'] != 'mismatch']
+
+    total_written = 0
+    for i in tqdm(range(len(good_gaps)), desc="({}) {}".format(args.file_index, args.file_id), ncols=100):
+        ctm_index = good_gaps[i][0]
+        gap = good_gaps[i][1]
+
+        # to prevent out of bounds
+        if i+1 >= len(good_gaps):
+            continue
+
+        # we start splitting from this ctm_index to the next good gap
+        start_index = ctm_index
+        end_index = good_gaps[i+1][0]
+
+        # this is our clip
+        clip = ctms[start_index:end_index]
+
+        n_words = len(clip)
+        n_mismatches = sum([word['case'] == 'mismatch' for word in clip])
+        words = " ".join([word["word"] for word in clip])
+
+        if n_words >= 5 and n_mismatches >= 1:
+            start_sec = clip[0]['start']
+            end_sec = clip[-1]['end']
+
+            sox_trim(start_sec, end_sec)
+
+            txt_file = os.path.join(args.dataset_dir, "txt/{}_{:07d}_{:07d}.txt".format(args.file_id, int(start_sec*100), int(end_sec*100))) 
+
+            with open(txt_file, "w") as f:
+                f.write(words + "\n")
+
+            duration = end_sec - start_sec
+            logger.info("{}: gap {}s, start {}s, end {}s, duration {}s".format(ctm_index, gap, start_sec, end_sec, duration))
+
+            total_written += duration
+
+logger.info("Wrote {} seconds out of {} ({:.2f}%) for {}.".format(total_written,ctms[-1]['end'],(total_written/ctms[-1]['end'])*100, args.file_id))

From f89de6fe3cc67f73ac1f2dd4baabcde323bb90dc Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Tue, 11 Jul 2017 23:59:07 +0530
Subject: [PATCH 50/67] added ctm align generation file

---
 gen-ctm-align.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 gen-ctm-align.py

diff --git a/gen-ctm-align.py b/gen-ctm-align.py
new file mode 100644
index 0000000..c31319b
--- /dev/null
+++ b/gen-ctm-align.py
@@ -0,0 +1,15 @@
+import argparse
+import os
+import subprocess
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument('manifest', type=str, help='list of file ids')
+args = parser.parse_args()
+
+files = open(args.manifest).read().strip().split('\n')
+
+for i in tqdm(range(len(files)), ncols=100):
+    if os.path.exists(files[i] + ".ctm"):
+        subprocess.call("node ctm-align.js " + files[i], shell=True)
+

From e3380176452df21870423440c9065d1a7b1ce280 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 12 Jul 2017 02:28:12 +0530
Subject: [PATCH 51/67] download and convert audio file

---
 gaps.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/gaps.py b/gaps.py
index e8fb618..f928ab3 100644
--- a/gaps.py
+++ b/gaps.py
@@ -5,6 +5,8 @@
 import re
 from tqdm import tqdm
 import logging
+import boto3
+import sys
 
 parser = argparse.ArgumentParser()
 parser.add_argument('file_id', type=str, help='file id to process')
@@ -20,15 +22,27 @@
 
 # temporarily running them from "others/"
 ctm_file = os.path.join(args.align_dir, args.file_id + "_align.json")
-audio_file = os.path.join(args.audio_dir, args.file_id + ".mp3")
+mp3 = os.path.join(args.audio_dir, args.file_id + ".mp3")
+wav = os.path.join(args.audio_dir, args.file_id + ".wav")
+FNULL = open("/dev/null")
+
+if not os.path.isfile(wav):
+    if not os.path.isfile(mp3):
+        bucket = boto3.resource("s3").Bucket("cgws")
+        try:
+            bucket.download_file("{}.mp3".format(args.file_id), mp3)
+        except:
+            print("Could not download file {} from S3.".format(args.file_id))
+            sys.exit()
+
+    subprocess.call(["sox","{}".format(mp3),"-r","16k", "{}".format(wav), "remix","-"], stdout=FNULL, stderr=FNULL)
 
 # leaving offset here in case the algo changes to require it
 def sox_trim(start, end):
     """Write out a segment of an audio file to wav, based on start, end,
     """
-    FNULL = open("/dev/null")
-    wav_file = os.path.join(args.dataset_dir, "wav/{}_{:07d}_{:07d}.wav".format(args.file_id, int(start*100), int(end*100))) 
-    subprocess.call(["sox", audio_file, "-r", "16k", wav_file, "trim", str(start), str(end - start), "remix", "-"], stdout=FNULL, stderr=FNULL)
+    clip_file = os.path.join(args.dataset_dir, "wav/{}_{:07d}_{:07d}.wav".format(args.file_id, int(start*100), int(end*100))) 
+    subprocess.call(["sox", wav, clip_file, "trim", str(start), str(end - start)], stdout=FNULL, stderr=FNULL)
 
 with open(ctm_file) as f:
     ctms = json.loads(f.read())

From a94bcdef419e9e95224db4f975de72a69a2955ea Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 12 Jul 2017 09:44:53 +0530
Subject: [PATCH 52/67] unicode encoding

---
 gaps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gaps.py b/gaps.py
index f928ab3..1f9783c 100644
--- a/gaps.py
+++ b/gaps.py
@@ -73,7 +73,7 @@ def sox_trim(start, end):
 
         n_words = len(clip)
         n_mismatches = sum([word['case'] == 'mismatch' for word in clip])
-        words = " ".join([word["word"] for word in clip])
+        words = " ".join([word["word"] for word in clip]).encode('utf-8').strip()
 
         if n_words >= 5 and n_mismatches >= 1:
             start_sec = clip[0]['start']

From 0c2c91c84467477b1c0ff64d1f79df599ef3b418 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 12 Jul 2017 16:43:03 +0530
Subject: [PATCH 53/67] added max hours

---
 pytorch_manifest.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 0c78571..46fa386 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -14,25 +14,27 @@
 import matplotlib.pyplot as plt
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--data_dir",default="/home/aaron/data/deepspeech_data",type=str,help='Directory to read files from')
-parser.add_argument("--dst_dir",default=".",type=str, help="Directory to store dataset to")
-parser.add_argument("--min_seconds",default=2.0,type=float, help="Cutoff for minimum duration")
-parser.add_argument("--max_seconds",default=20.0,type=float, help="Cutoff for maximum duration")
-parser.add_argument("--no_ted", help="Merge with TED dataset", action='store_true', default=False)
-parser.add_argument("--dry_run", help="Don't write manifest csv's", action='store_true', default=False)
-parser.add_argument("--split_ratio", help="Percent of files to keep in val set", type=float, default=0.01)
+parser.add_argument("--data-dir", dest="data_dir", type=str, default="/home/aaron/data/phoenix-data", help='Directory to read files from')
+parser.add_argument("--dst-dir", dest="dst_dir", default=".", type=str, help="Directory to store dataset to")
+parser.add_argument("--min-seconds", dest="min_seconds", default=2.0, type=float, help="Cutoff for minimum duration")
+parser.add_argument("--max-seconds", dest="max_seconds", default=20.0, type=float, help="Cutoff for maximum duration")
+parser.add_argument("--max-hours", dest="max_hours", default=0, type=int, help="Size of the dataset in hours")
+parser.add_argument("--merge-ted", dest="merge_ted", help="Merge with TED dataset", action='store_true', default=False)
+parser.add_argument("--dry-run", dest="dry_run", help="Don't write manifest csv's", action='store_true', default=False)
+parser.add_argument("--split-ratio", dest="split_ratio", type=float, default=0.01, help="Percent of files to keep in val & test set")
+parser.add_argument("--txt-dir", dest="txt_dir", type=str, default="txt", help="Directory name for txt files")
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.data_dir)
 wav_dir = os.path.join(parent_dir,"wav")
-txt_dir = os.path.join(parent_dir,"stm")
+txt_dir = os.path.join(parent_dir, args.txt_dir)
 
 dst_dir = os.path.abspath(args.dst_dir)
 dst_wav = os.path.join(dst_dir, "wav")
 if not os.path.exists(dst_wav):
     os.makedirs(dst_wav)
 
-dst_txt = os.path.join(dst_dir, "stm")
+dst_txt = os.path.join(dst_dir, args.txt_dir)
 if not os.path.exists(dst_txt):
     os.makedirs(dst_txt)
 
@@ -65,6 +67,8 @@ def sort_func(element):
     else:
         break
 
+total_seconds = 0
+
 # get filenames wav directory
 for i in tqdm(range(num_files), ncols=100, desc='Copying files'):
     filename = files[i]
@@ -87,6 +91,10 @@ def sort_func(element):
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
+    total_seconds += duration
+    if args.max_hours != 0 and args.max_hours < total_seconds/3600:
+        break
+
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
     dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
     if not os.path.isfile(dst_txt_file):
@@ -101,12 +109,10 @@ def sort_func(element):
         # at least two words in transcript
         num_words = len(transcript.split())
         if num_words <= 1:
-            print("skipping %s as num word is %d" % (fid, num_words))
             continue 
 
         oov = re.search("[^a-zA-Z ']", transcript)
         if oov is not None:
-            print("skipping %s due to oov, %s" % (fid, transcript))
             continue 
 
         with open(dst_txt_file, 'w') as f:
@@ -120,7 +126,7 @@ def sort_func(element):
 val_set = keep_files[len(train_set):-val_len]
 test_set = keep_files[-val_len:]
 
-if not args.no_ted and not args.dry_run:
+if args.merge_ted and not args.dry_run:
     ted_train = []
     for line in open("ted_train_manifest.csv","r"):
         ted_train.append(line)

From 007c4894ec1c99fef9a1d5534106d52bba61e395 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 12 Jul 2017 16:50:12 +0530
Subject: [PATCH 54/67] fixed max hours check

---
 pytorch_manifest.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 46fa386..b280e88 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -91,10 +91,6 @@ def sort_func(element):
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
-    total_seconds += duration
-    if args.max_hours != 0 and args.max_hours < total_seconds/3600:
-        break
-
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
     dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
     if not os.path.isfile(dst_txt_file):
@@ -120,6 +116,11 @@ def sort_func(element):
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
+    total_seconds += duration
+    if args.max_hours != 0 and args.max_hours < total_seconds/3600:
+        print("\n")
+        break
+
 val_len = int(len(keep_files) * args.split_ratio)
 
 train_set = keep_files[:-val_len*2]

From eaf7e8da3e24d507c86c583dcb21da12b29336a2 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 12 Jul 2017 17:54:33 +0530
Subject: [PATCH 55/67] added prefix and manifests dir

---
 pytorch_manifest.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index b280e88..c944621 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -23,6 +23,8 @@
 parser.add_argument("--dry-run", dest="dry_run", help="Don't write manifest csv's", action='store_true', default=False)
 parser.add_argument("--split-ratio", dest="split_ratio", type=float, default=0.01, help="Percent of files to keep in val & test set")
 parser.add_argument("--txt-dir", dest="txt_dir", type=str, default="txt", help="Directory name for txt files")
+parser.add_argument("--prefix", dest="prefix", type=str, default="unnamed", help="Prefix for manifest files")
+parser.add_argument("--manifest-dir", dest="manifest_dir", type=str, default=".", help="Directory for manifest files")
 args = parser.parse_args()
 
 parent_dir = os.path.abspath(args.data_dir)
@@ -67,7 +69,7 @@ def sort_func(element):
     else:
         break
 
-total_seconds = 0
+total_hours = 0
 
 # get filenames wav directory
 for i in tqdm(range(num_files), ncols=100, desc='Copying files'):
@@ -116,8 +118,8 @@ def sort_func(element):
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 
-    total_seconds += duration
-    if args.max_hours != 0 and args.max_hours < total_seconds/3600:
+    total_hours += duration/3600
+    if args.max_hours != 0 and args.max_hours <= total_hours:
         print("\n")
         break
 
@@ -152,19 +154,19 @@ def sort_func(element):
 
 total_train = sum([line[0] for line in train_set])
 if not args.dry_run:
-    with open('train.csv', 'w') as f:
+    with open(os.path.join(args.manifest_dir, "{}-train-{}.csv".format(args.prefix, int(total_hours))), 'w') as f:
         for line in train_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
 
 total_val = sum([line[0] for line in val_set])
 if not args.dry_run:
-    with open('val.csv', 'w') as f:
+    with open(os.path.join(args.manifest_dir, "{}-val-{}.csv".format(args.prefix, int(total_hours))), 'w') as f:
         for line in val_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
 
 total_test = sum([line[0] for line in test_set])
 if not args.dry_run:
-    with open('test.csv', 'w') as f:
+    with open(os.path.join(args.manifest_dir, "{}-test-{}.csv".format(args.prefix, int(total_hours))), 'w') as f:
         for line in test_set:
             f.write((line[1].strip() + "\n").encode('utf-8'))
 
@@ -172,7 +174,7 @@ def sort_func(element):
     # train_set has already been written so modifying in place is ok
     np.random.shuffle(train_set)
     train_subset = train_set[:len(val_set)/2]
-    with open('train_subset.csv', 'w') as f:
+    with open(os.path.join(args.manifest_dir, "{}-train-subset-{}.csv".format(args.prefix, int(total_hours))), 'w') as f:
         for line in train_subset:
             f.write((line[1].strip() + "\n").encode("utf-8"))
 
@@ -185,7 +187,7 @@ def sort_func(element):
 plt.ylabel('# of files')
 plt.grid(color='gray', linestyle='dotted')
 plt.xticks(bins)
-plt.title("Durations distribution @ {} hours".format(int(total/3600)))
+plt.title("Durations distribution for {} {} hours".format(args.prefix, int(total_hours)))
 plt.savefig('durations.png')
 
 print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, test {:.2f}, ratio {:.5f}/{:.5f}/{:.5f}".format(total/3600, total_train/3600, total_val/3600, total_test/3600, total_train/total, total_val/total, total_test/total))

From 6af4bbdbdecf72532d2fbdedf23a90da469effc5 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sat, 15 Jul 2017 23:56:40 +0530
Subject: [PATCH 56/67] file exists check

---
 gaps.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/gaps.py b/gaps.py
index 1f9783c..8a4e507 100644
--- a/gaps.py
+++ b/gaps.py
@@ -26,6 +26,9 @@
 wav = os.path.join(args.audio_dir, args.file_id + ".wav")
 FNULL = open("/dev/null")
 
+if not os.path.isfile(ctm_file):
+    sys.exit()
+
 if not os.path.isfile(wav):
     if not os.path.isfile(mp3):
         bucket = boto3.resource("s3").Bucket("cgws")

From e5475a5eceaf5f1631d61d3f1d68254377bdb4b3 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 16 Jul 2017 12:19:59 +0530
Subject: [PATCH 57/67] skip copy files in dry run

---
 pytorch_manifest.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index c944621..305e37e 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -14,8 +14,8 @@
 import matplotlib.pyplot as plt
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--data-dir", dest="data_dir", type=str, default="/home/aaron/data/phoenix-data", help='Directory to read files from')
-parser.add_argument("--dst-dir", dest="dst_dir", default=".", type=str, help="Directory to store dataset to")
+parser.add_argument("--data-dir", dest="data_dir", type=str, default="/home/aaron/data/phoenix-files/gaps", help='Directory to read files from')
+parser.add_argument("--dst-dir", dest="dst_dir", default="/home/aaron/data/phoenix-files/pytorch", type=str, help="Directory to store dataset to")
 parser.add_argument("--min-seconds", dest="min_seconds", default=2.0, type=float, help="Cutoff for minimum duration")
 parser.add_argument("--max-seconds", dest="max_seconds", default=20.0, type=float, help="Cutoff for maximum duration")
 parser.add_argument("--max-hours", dest="max_hours", default=0, type=int, help="Size of the dataset in hours")
@@ -82,14 +82,11 @@ def sort_func(element):
     if not os.path.isfile(dst_wav_file):
         duration = get_duration(wav_file)
 
-        if duration >= 2.0:
+        if duration >= args.min_seconds and not args.dry_run::
             shutil.copy2(wav_file, dst_wav_file)
     else:
         duration = get_duration(dst_wav_file)
 
-        if duration < 2.0:
-            os.remove(dst_wav_file)
-
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
@@ -113,8 +110,9 @@ def sort_func(element):
         if oov is not None:
             continue 
 
-        with open(dst_txt_file, 'w') as f:
-            f.write(transcript.upper() + "\n")
+        if not args.dry_run:
+            with open(dst_txt_file, 'w') as f:
+                f.write(transcript.upper() + "\n")
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 

From 83a2cd3f8986fb4d37f72db71ba4c520d76ba69d Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 16 Jul 2017 13:28:42 +0530
Subject: [PATCH 58/67] syntax

---
 pytorch_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 305e37e..8321a62 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -82,7 +82,7 @@ def sort_func(element):
     if not os.path.isfile(dst_wav_file):
         duration = get_duration(wav_file)
 
-        if duration >= args.min_seconds and not args.dry_run::
+        if duration >= args.min_seconds and not args.dry_run:
             shutil.copy2(wav_file, dst_wav_file)
     else:
         duration = get_duration(dst_wav_file)

From 617bb56739a1f1c47839d938948b30c31af6451d Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 16 Jul 2017 21:49:35 +0530
Subject: [PATCH 59/67] only copy selected files

---
 pytorch_manifest.py | 47 ++++++++++++++++++---------------------------
 1 file changed, 19 insertions(+), 28 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 8321a62..4619b89 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -72,47 +72,38 @@ def sort_func(element):
 total_hours = 0
 
 # get filenames wav directory
-for i in tqdm(range(num_files), ncols=100, desc='Copying files'):
+for i in tqdm(range(num_files), ncols=100, desc='Checking files'):
     filename = files[i]
     fid = os.path.splitext(filename)[0]
 
     wav_file = os.path.join(wav_dir,"{}.wav".format(fid))
-    dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
-
-    if not os.path.isfile(dst_wav_file):
-        duration = get_duration(wav_file)
-
-        if duration >= args.min_seconds and not args.dry_run:
-            shutil.copy2(wav_file, dst_wav_file)
-    else:
-        duration = get_duration(dst_wav_file)
-
+    duration = get_duration(dst_wav_file)
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
-    dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
-    if not os.path.isfile(dst_txt_file):
-        with open(txt_file) as raw_text:
-            transcript = raw_text.read().strip()
+    with open(txt_file) as raw_text:
+        transcript = raw_text.read().strip()
 
-        if len(transcript) == 0:
-            continue 
+    transcript = re.sub('\s+', ' ', transcript)
 
-        transcript = re.sub('\s+', ' ', transcript)
+    # at least two words in transcript
+    num_words = len(transcript.split())
+    if num_words <= 1:
+        continue 
 
-        # at least two words in transcript
-        num_words = len(transcript.split())
-        if num_words <= 1:
-            continue 
+    oov = re.search("[^a-zA-Z ']", transcript)
+    if oov is not None:
+        continue 
 
-        oov = re.search("[^a-zA-Z ']", transcript)
-        if oov is not None:
-            continue 
+    if not args.dry_run:
+        dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
+        with open(dst_txt_file, 'w') as f:
+            f.write(transcript.upper() + "\n")
 
-        if not args.dry_run:
-            with open(dst_txt_file, 'w') as f:
-                f.write(transcript.upper() + "\n")
+        dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
+        if not os.path.isfile(dst_wav_file):
+            shutil.copy2(wav_file, dst_wav_file)
 
     keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file)))
 

From e49b7eaf98910bf247f01e2a5532ac9810904aab Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 16 Jul 2017 21:51:41 +0530
Subject: [PATCH 60/67] fixed reference error

---
 pytorch_manifest.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 4619b89..5e377ea 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -96,12 +96,13 @@ def sort_func(element):
     if oov is not None:
         continue 
 
+    dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
+    dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
+
     if not args.dry_run:
-        dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid))
         with open(dst_txt_file, 'w') as f:
             f.write(transcript.upper() + "\n")
 
-        dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
         if not os.path.isfile(dst_wav_file):
             shutil.copy2(wav_file, dst_wav_file)
 

From b13f93aa1bdaa5c92bf979c8bb0b6934b96222f3 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Sun, 16 Jul 2017 21:53:42 +0530
Subject: [PATCH 61/67] fixed reference error

---
 pytorch_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 5e377ea..ff001c5 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -77,7 +77,7 @@ def sort_func(element):
     fid = os.path.splitext(filename)[0]
 
     wav_file = os.path.join(wav_dir,"{}.wav".format(fid))
-    duration = get_duration(dst_wav_file)
+    duration = get_duration(wav_file)
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 

From bc0f6f07c4765bfd8e2a7dd24a0074ebf1573e68 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Mon, 17 Jul 2017 07:39:38 +0530
Subject: [PATCH 62/67] calculate duration from filename

---
 pytorch_manifest.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index ff001c5..74e7918 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -41,6 +41,11 @@
     os.makedirs(dst_txt)
 
 def get_duration(wav_file):
+    m = re.search("[a-z0-9]+_([0-9]+)_([0-9]+).wav", wav_file)
+    if m:
+        duration = (int(m.group(2)) - int(m.group(1)))/100.
+        return duration
+
     try:
         f = sf.SoundFile(wav_file)
         if f.samplerate != 16000:
@@ -85,13 +90,6 @@ def sort_func(element):
     with open(txt_file) as raw_text:
         transcript = raw_text.read().strip()
 
-    transcript = re.sub('\s+', ' ', transcript)
-
-    # at least two words in transcript
-    num_words = len(transcript.split())
-    if num_words <= 1:
-        continue 
-
     oov = re.search("[^a-zA-Z ']", transcript)
     if oov is not None:
         continue 
@@ -100,8 +98,9 @@ def sort_func(element):
     dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
 
     if not args.dry_run:
+        transcript = re.sub('\s+', ' ', transcript).upper() + "\n"
         with open(dst_txt_file, 'w') as f:
-            f.write(transcript.upper() + "\n")
+            f.write(transcript)
 
         if not os.path.isfile(dst_wav_file):
             shutil.copy2(wav_file, dst_wav_file)

From 3967feb96529cd54ea89d152b14c999341fcd911 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Mon, 17 Jul 2017 09:59:40 +0530
Subject: [PATCH 63/67] filesize check

---
 pytorch_manifest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 74e7918..cdaf2a6 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -63,10 +63,10 @@ def sort_func(element):
 files= []
 
 FNULL = open(os.devnull, 'w')
-out, err = subprocess.Popen("find " + wav_dir + " -type f | wc -l", stdout=subprocess.PIPE, shell=True).communicate()
+out, err = subprocess.Popen("find " + wav_dir + " -type f -size +100c | wc -l", stdout=subprocess.PIPE, shell=True).communicate()
 num_files = int(out)
 
-find = subprocess.Popen(["find", wav_dir, "-type", "f"], stdout=subprocess.PIPE, stderr=FNULL)
+find = subprocess.Popen(["find", wav_dir, "-size" + "+100c", "-type", "f"], stdout=subprocess.PIPE, stderr=FNULL)
 for i in tqdm(range(num_files), ncols=100, desc='Finding files'):
     line = find.stdout.readline()
     if len(line.strip()) > 0:

From 24152be06b839c2a9f83feca4f24973bf86f05ad Mon Sep 17 00:00:00 2001
From: aaron <aaron@scribie.com>
Date: Wed, 2 Aug 2017 05:20:09 -0700
Subject: [PATCH 64/67] added align-ctm file

---
 align-ctm.sh | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100755 align-ctm.sh

diff --git a/align-ctm.sh b/align-ctm.sh
new file mode 100755
index 0000000..0c9d962
--- /dev/null
+++ b/align-ctm.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#set -x
+#set -euxo pipefail
+
+if [ $# -lt 2 ]
+then
+    echo "usage: ./align-ctm.sh filelist index"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+index=$2
+
+for fid in `tail -n +$index $filelist`
+do
+	num_files=`find /home/aaron/data/phoenix-files/gaps/txt/${fid}_*.txt 2>/dev/null | wc -l`
+	if [ $num_files -eq 0 ]
+	then
+		index=`grep -n $fid $filelist | cut -d ':' -f1`
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data/phoenix-files/gaps/
+	fi
+done

From fb61e6ca7518c44860c7ba89e42b34475415b2e3 Mon Sep 17 00:00:00 2001
From: Rajiv Poddar <rajiv@scribie.com>
Date: Wed, 2 Aug 2017 17:52:32 +0530
Subject: [PATCH 65/67] speaker seg test

---
 align-ctm.sh |  4 ++--
 gaps.py      | 23 ++++++++++++++++++-----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/align-ctm.sh b/align-ctm.sh
index 0c9d962..eeec2e4 100755
--- a/align-ctm.sh
+++ b/align-ctm.sh
@@ -20,10 +20,10 @@ index=$2
 
 for fid in `tail -n +$index $filelist`
 do
-	num_files=`find /home/aaron/data/phoenix-files/gaps/txt/${fid}_*.txt 2>/dev/null | wc -l`
+	num_files=`find /home/aaron/data/phoenix-files/speaker-seg-test/txt/${fid}_*.txt 2>/dev/null | wc -l`
 	if [ $num_files -eq 0 ]
 	then
 		index=`grep -n $fid $filelist | cut -d ':' -f1`
-		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data/phoenix-files/gaps/
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data/phoenix-files/speaker-seg-test/ --speaker-turns
 	fi
 done
diff --git a/gaps.py b/gaps.py
index 8a4e507..862225a 100644
--- a/gaps.py
+++ b/gaps.py
@@ -1,3 +1,6 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
 import argparse
 import json
 import os
@@ -15,12 +18,12 @@
 parser.add_argument('--audio-dir', type=str, dest="audio_dir", default='.', help='Path to the directory containing audio files')
 parser.add_argument('--align-dir', type=str, dest="align_dir", default=".", help='Path to the directory containing alignments')
 parser.add_argument('--dataset-dir', type=str, dest="dataset_dir", default='.', help='Path to the dataset directory')
+parser.add_argument('--speaker-turns', action="store_true", dest="speaker_turns", default=False, help='Add speaker turn markers')
 args = parser.parse_args()
 
 logging.basicConfig(filename='gaps.log', level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
 logger = logging.getLogger("info_logger")
 
-# temporarily running them from "others/"
 ctm_file = os.path.join(args.align_dir, args.file_id + "_align.json")
 mp3 = os.path.join(args.audio_dir, args.file_id + ".mp3")
 wav = os.path.join(args.audio_dir, args.file_id + ".wav")
@@ -75,10 +78,20 @@ def sox_trim(start, end):
         clip = ctms[start_index:end_index]
 
         n_words = len(clip)
-        n_mismatches = sum([word['case'] == 'mismatch' for word in clip])
-        words = " ".join([word["word"] for word in clip]).encode('utf-8').strip()
-
-        if n_words >= 5 and n_mismatches >= 1:
+        count = 0
+        if args.speaker_turns:
+            words = " ".join([word["orig"] for word in clip]).encode('utf-8').strip()
+            words = re.sub("\-", " ", words)
+            words = re.sub(r"[^a-zA-Z0-9¶\' ]", "", words, re.UNICODE)
+            words = re.sub(r"¶", " ¶", words)
+            words = re.sub("\s{2,}", " ", words)
+            words = words.lower()
+            count = words.count('¶')
+        else:
+            words = " ".join([word["word"] for word in clip]).encode('utf-8').strip()
+            count = sum([word['case'] == 'mismatch' for word in clip])
+
+        if n_words >= 5 and count >= 1:
             start_sec = clip[0]['start']
             end_sec = clip[-1]['end']
 

From 43f057042d3d4d2670942ce206a47f91eae568f4 Mon Sep 17 00:00:00 2001
From: aaron <aaron@scribie.com>
Date: Sat, 12 Aug 2017 07:26:57 -0700
Subject: [PATCH 66/67] added lm scripts

---
 align-ctm.sh        |  4 ++--
 gaps.py             | 12 +++++++++---
 generator.py        |  3 ++-
 pytorch_manifest.py |  7 +++++--
 sentence_cleaner.py | 38 ++++++++++++++++++++++++++++++++++++++
 watch-align.sh      | 10 ++++++++++
 6 files changed, 66 insertions(+), 8 deletions(-)
 create mode 100644 sentence_cleaner.py
 create mode 100755 watch-align.sh

diff --git a/align-ctm.sh b/align-ctm.sh
index eeec2e4..08a03dc 100755
--- a/align-ctm.sh
+++ b/align-ctm.sh
@@ -20,10 +20,10 @@ index=$2
 
 for fid in `tail -n +$index $filelist`
 do
-	num_files=`find /home/aaron/data/phoenix-files/speaker-seg-test/txt/${fid}_*.txt 2>/dev/null | wc -l`
+	num_files=`find /home/aaron/data2/phoenix-files/speaker-seg-test/txt/${fid}_*.txt 2>/dev/null | wc -l`
 	if [ $num_files -eq 0 ]
 	then
 		index=`grep -n $fid $filelist | cut -d ':' -f1`
-		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data/phoenix-files/speaker-seg-test/ --speaker-turns
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data2/phoenix-files/speaker-seg-test/ --speaker-turns
 	fi
 done
diff --git a/gaps.py b/gaps.py
index 862225a..e4489a0 100644
--- a/gaps.py
+++ b/gaps.py
@@ -48,7 +48,10 @@ def sox_trim(start, end):
     """Write out a segment of an audio file to wav, based on start, end,
     """
     clip_file = os.path.join(args.dataset_dir, "wav/{}_{:07d}_{:07d}.wav".format(args.file_id, int(start*100), int(end*100))) 
-    subprocess.call(["sox", wav, clip_file, "trim", str(start), str(end - start)], stdout=FNULL, stderr=FNULL)
+    ret = subprocess.call(["sox", wav, clip_file, "trim", str(start), str(end - start)], stdout=FNULL, stderr=FNULL)
+    if ret != 0:
+        logger.error("sox failed: {}, error: {}".format(clip_file, ret))
+        sys.exit()
 
 with open(ctm_file) as f:
     ctms = json.loads(f.read())
@@ -84,14 +87,17 @@ def sox_trim(start, end):
             words = re.sub("\-", " ", words)
             words = re.sub(r"[^a-zA-Z0-9¶\' ]", "", words, re.UNICODE)
             words = re.sub(r"¶", " ¶", words)
+            words = re.sub(r"^¶", "", words)
+            words = re.sub(r"¶$", "", words)
             words = re.sub("\s{2,}", " ", words)
             words = words.lower()
-            count = words.count('¶')
+            # count = words.count('¶')
         else:
             words = " ".join([word["word"] for word in clip]).encode('utf-8').strip()
             count = sum([word['case'] == 'mismatch' for word in clip])
 
-        if n_words >= 5 and count >= 1:
+        # if n_words >= 5 and count >= 1:
+        if n_words >= 5:
             start_sec = clip[0]['start']
             end_sec = clip[-1]['end']
 
diff --git a/generator.py b/generator.py
index db91709..0f0c703 100644
--- a/generator.py
+++ b/generator.py
@@ -62,7 +62,7 @@ def data_generator(file_path,shuffle,vocabulary_size,train_size,test_size):
                     if "_" in sentence:
                         continue
 
-                    sentence = re_decimal.sub("\\1 point \\2", sentence) # decimals
+                    #sentence = re_decimal.sub("\\1 point \\2", sentence) # decimals
                     sentence = re_url.sub(" dot \\1", sentence) # URLs
                     sentence = re_negative.sub(" minus \\1", sentence) # negative numbers
                     sentence = re_pattern.sub(" ", sentence).lower() # space in case it was connected to a word
@@ -84,6 +84,7 @@ def data_generator(file_path,shuffle,vocabulary_size,train_size,test_size):
                     doc_sents.append(sentence)
 
         sentences.extend([sent.split() for sent in doc_sents])
+        print("{} sentences after {}.".format(len(sentences),txt_file))
 
     # word frequencies
     word_freq = nltk.FreqDist(itertools.chain(*sentences))
diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index cdaf2a6..2cf3b58 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -1,3 +1,5 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
 import os
 import argparse
 import scipy.io.wavfile as wav
@@ -66,7 +68,7 @@ def sort_func(element):
 out, err = subprocess.Popen("find " + wav_dir + " -type f -size +100c | wc -l", stdout=subprocess.PIPE, shell=True).communicate()
 num_files = int(out)
 
-find = subprocess.Popen(["find", wav_dir, "-size" + "+100c", "-type", "f"], stdout=subprocess.PIPE, stderr=FNULL)
+find = subprocess.Popen("find " + wav_dir + " -type f -size +100c", stdout=subprocess.PIPE, stderr=FNULL, shell=True)
 for i in tqdm(range(num_files), ncols=100, desc='Finding files'):
     line = find.stdout.readline()
     if len(line.strip()) > 0:
@@ -86,11 +88,12 @@ def sort_func(element):
     if duration < args.min_seconds or duration > args.max_seconds:
         continue
 
+
     txt_file = os.path.join(txt_dir,"{}.txt".format(fid))
     with open(txt_file) as raw_text:
         transcript = raw_text.read().strip()
 
-    oov = re.search("[^a-zA-Z ']", transcript)
+    oov = re.search("[^a-zA-Z '¶]", transcript)
     if oov is not None:
         continue 
 
diff --git a/sentence_cleaner.py b/sentence_cleaner.py
new file mode 100644
index 0000000..f57df8c
--- /dev/null
+++ b/sentence_cleaner.py
@@ -0,0 +1,38 @@
+import re
+import nltk
+
+
+def clean(text,word_to_id):
+
+    unknown_token="UNK"
+    sentence_start_token="START"
+    sentence_end_token="END"
+
+    # metas and punctuation (keeping hyphens)
+    timestamps = "\d+:\d+:\d+(\.\d+)*"
+    speakers = "S\d*:"
+    metas = "\[.{5,24}\]"
+    punctuation = "[\_\"\.\!\?\:]+"
+    new_line = "\n"
+    pattern = "|".join([timestamps,speakers,metas,punctuation,new_line])
+
+    # numbers not part of a word like CO2, mp3, etc.
+    numbers = "(?<!\w)\d+"
+
+    line = text.decode("utf-8")
+
+    # remove timestamps, speakers, metas, and punctuation. then lower and strip, then sub numbers
+    cleaned = re.sub(pattern,"",line).lower().strip()
+    sentence = re.sub(numbers,"N",cleaned)
+
+    # was != "", but there were a number of sentences of just "s"
+    sentence = "{} {} {}".format(sentence_start_token,sentence,sentence_end_token)
+
+    # word tokenize
+    word_tokenized = nltk.word_tokenize(sentence)
+
+    out_sentence = [word_to_id[word] if word in word_to_id else unknown_token for word in word_tokenized]
+
+    return out_sentence
+
+
diff --git a/watch-align.sh b/watch-align.sh
new file mode 100755
index 0000000..c321474
--- /dev/null
+++ b/watch-align.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+inotifywait -mqr -e move "/home/aaron/data/ctm-alignments" | while read path action file
+do
+        if [ ${file:(-5)} == '.json' ]
+        then
+		fid=${file:0:-11}
+		index=`grep -n $fid ~/phoenix/file-ids-2017.csv | cut -d ':' -f1`
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data/phoenix-files/gaps/
+        fi
+done

From 08c814e4f1b770921dbd5639f7f976493cfc61bc Mon Sep 17 00:00:00 2001
From: aaron <aaron@scribie.com>
Date: Thu, 24 Aug 2017 05:04:15 -0700
Subject: [PATCH 67/67] punctuations + speaker turns

---
 align-ctm.sh        |  9 +++++----
 gaps.py             | 15 +++++++++++----
 pytorch_manifest.py |  2 ++
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/align-ctm.sh b/align-ctm.sh
index 08a03dc..87f641b 100755
--- a/align-ctm.sh
+++ b/align-ctm.sh
@@ -3,9 +3,9 @@
 #set -x
 #set -euxo pipefail
 
-if [ $# -lt 2 ]
+if [ $# -lt 3 ]
 then
-    echo "usage: ./align-ctm.sh filelist index"
+    echo "usage: ./align-ctm.sh filelist index dataset_dir"
     exit 1
 fi
 
@@ -17,13 +17,14 @@ then
 fi
 
 index=$2
+dataset_dir=$3
 
 for fid in `tail -n +$index $filelist`
 do
-	num_files=`find /home/aaron/data2/phoenix-files/speaker-seg-test/txt/${fid}_*.txt 2>/dev/null | wc -l`
+	num_files=`find ${dataset_dir}/txt/${fid}_*.txt 2>/dev/null | wc -l`
 	if [ $num_files -eq 0 ]
 	then
 		index=`grep -n $fid $filelist | cut -d ':' -f1`
-		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir /home/aaron/data2/phoenix-files/speaker-seg-test/ --speaker-turns
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir ${dataset_dir} --speaker-turns
 	fi
 done
diff --git a/gaps.py b/gaps.py
index e4489a0..a748b4e 100644
--- a/gaps.py
+++ b/gaps.py
@@ -21,7 +21,7 @@
 parser.add_argument('--speaker-turns', action="store_true", dest="speaker_turns", default=False, help='Add speaker turn markers')
 args = parser.parse_args()
 
-logging.basicConfig(filename='gaps.log', level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+logging.basicConfig(filename='punctuations.log', level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
 logger = logging.getLogger("info_logger")
 
 ctm_file = os.path.join(args.align_dir, args.file_id + "_align.json")
@@ -29,6 +29,12 @@
 wav = os.path.join(args.audio_dir, args.file_id + ".wav")
 FNULL = open("/dev/null")
 
+if not os.path.isdir(os.path.join(args.dataset_dir, "txt")):
+    os.makedirs(os.path.join(args.dataset_dir, "txt"))
+
+if not os.path.isdir(os.path.join(args.dataset_dir, "wav")):
+    os.makedirs(os.path.join(args.dataset_dir, "wav"))
+
 if not os.path.isfile(ctm_file):
     sys.exit()
 
@@ -58,7 +64,6 @@ def sox_trim(start, end):
 
     null_word = {'start': 0, 'end':0}
     gaps = [second['start']-first['end'] for first, second in zip([null_word]+ctms, ctms)]
-
     # we split from one good gap to the next
     # a good gap is when the silence between the words is long
     # and the word *itself* is long and is not a mismatch
@@ -85,9 +90,11 @@ def sox_trim(start, end):
         if args.speaker_turns:
             words = " ".join([word["orig"] for word in clip]).encode('utf-8').strip()
             words = re.sub("\-", " ", words)
-            words = re.sub(r"[^a-zA-Z0-9¶\' ]", "", words, re.UNICODE)
+            # trying with some punctuation retained
+            words = re.sub(r"[\?!]",".",words)
+            words = re.sub(r"[^a-zA-Z0-9¶\.\,\' ]", "", words, re.UNICODE)
+            words = re.sub(r"^¶|¶$", "", words)
             words = re.sub(r"¶", " ¶", words)
-            words = re.sub(r"^¶", "", words)
             words = re.sub(r"¶$", "", words)
             words = re.sub("\s{2,}", " ", words)
             words = words.lower()
diff --git a/pytorch_manifest.py b/pytorch_manifest.py
index 2cf3b58..e57359c 100644
--- a/pytorch_manifest.py
+++ b/pytorch_manifest.py
@@ -101,6 +101,8 @@ def sort_func(element):
     dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid))
 
     if not args.dry_run:
+        # replacing pilcrows with pipes
+        transcript = re.sub('¶','|',transcript)
         transcript = re.sub('\s+', ' ', transcript).upper() + "\n"
         with open(dst_txt_file, 'w') as f:
             f.write(transcript)