Higher ratio of generation

.gitignore

@@ -90,3 +90,6 @@ ENV/
 
 # OSX
 .DS_Store
+
+*.swp
+inspect.sh

align-ctm.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#set -x
+#set -euxo pipefail
+
+if [ $# -lt 3 ]
+then
+    echo "usage: ./align-ctm.sh filelist index dataset_dir"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+index=$2
+dataset_dir=$3
+
+for fid in `tail -n +$index $filelist`
+do
+	num_files=`find ${dataset_dir}/txt/${fid}_*.txt 2>/dev/null | wc -l`
+	if [ $num_files -eq 0 ]
+	then
+		index=`grep -n $fid $filelist | cut -d ':' -f1`
+		python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir ${dataset_dir} --speaker-turns
+	fi
+done

align-master.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#set -x
+
+if [ $# -lt 2 ]
+then
+    echo "usage: ./align-master.sh filelist index"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+index=$2
+
+for fid in `tail -n +$index $filelist`
+do
+	touch ~/data/deepspeech_data/alignments/${fid}.json
+	index=`grep -n $fid $filelist | cut -d ':' -f1`
+	python rename-alignments.py $fid --file_index $index
+	python asr_data_gen.py --file $fid 2>>alignment.log
+done

align-worker.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+#set -x
+
+if [ $# -lt 2 ]
+then
+    echo "usage: ./align-worker.sh filelist index"
+    exit 1
+fi
+
+filelist=$1
+if [ ! -f $filelist ]
+then
+    echo "$filelist not found"
+    exit 1
+fi
+
+if [ ! -d ~/align ]
+then
+    mkdir ~/align
+fi
+
+index=$2
+host=`hostname`
+
+threads_multiplier=4
+if [ $host == 'eesen-worker' ]
+then
+    threads_multiplier=2
+fi
+
+for fid in `tail -n +$index $filelist`
+do
+    host=`hostname`
+    if [ $host == 'eesen-worker' ]
+    then
+        stat ~/align/${fid}.json 1>/dev/null 2>&1
+    else
+        ssh eesen-worker "stat ~/align/${fid}.json" 1>/dev/null 2>&1
+    fi
+
+    if [ $? -ne 0 ]
+    then
+        index=`grep -n $fid $filelist | cut -d ':' -f1` 
+
+        if [ $host == 'eesen-worker' ]
+        then
+            touch ~/align/${fid}.json
+        else 
+            while true
+            do
+                ssh eesen-worker "touch ~/align/${fid}.json" 1>/dev/null 2>&1
+                if [ $? -ne 0 ]
+                then
+                    sleep 1
+                else
+                    break
+                fi
+            done
+        fi
+
+        scp scribie:~/scribie/records/${fid}.txt ~/align 1>/dev/null 2>&1
+        python rename-alignments.py $fid --file_index $index --abort --threads_multiplier $threads_multiplier --use_align_dir 
+    fi
+done

aligner.py

@@ -7,13 +7,22 @@
 import hashlib
 import random
 import logging
+import argparse
 
 import boto3
 import gentle
 
 logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
 logger = logging.getLogger("info_logger")
 
+data_dir = '/home/aaron/data'
+records_dir = os.path.join(data_dir, 'records')
+mp3_dir = os.path.join(data_dir, 'mp3s')
+text_out_dir = os.path.join(data_dir, 'deepspeech_data/stm')
+wav_out_dir = os.path.join(data_dir, 'deepspeech_data/wav')
+json_out_dir = os.path.join(data_dir, 'deepspeech_data/alignments')
+use_filename_json = False
+
 def clean(text):
     """Clean transcript of timestamps and speaker trackings, metas,
     punctuation, and extra whitespace.
@@ -39,9 +48,11 @@ def trim(base_filename,audio_file,start,end,offset,out_directory):
                                    "{:07d}".format(int((offset+end)*100))))
 
     duration = end-start
-    subprocess.call(["sox","{}".format(audio_file),"-r","16k",
-                "{}".format(segment),"trim","{}".format(start),
-                "{}".format(duration),"remix","-"])
+
+    if not os.path.isfile(segment):
+        subprocess.call(["sox","{}".format(audio_file),"-r","16k",
+                    "{}".format(segment),"trim","{}".format(start),
+                    "{}".format(duration),"remix","-"])
 
     return segment
 
@@ -82,25 +93,26 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
     logger.info("Processing file id {}...".format(file_id))
 
     # grab audio file from s3
-    mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id)
-
-    if not os.path.isfile(mp3):
-        bucket = boto3.resource("s3").Bucket("cgws")
-        logger.info("Downloading file {} from S3...".format(file_id))
-        try:
-            bucket.download_file("{}.mp3".format(file_id),mp3)
-        except:
-            logger.warning("Could not download file {} from S3.".format(file_id))
-            return
+    mp3 = os.path.join(mp3_dir, "{}.mp3".format(file_id))
+    wav = os.path.join(mp3_dir, "{}.wav".format(file_id))
 
+    if not os.path.isfile(wav):
+        if not os.path.isfile(mp3):
+            bucket = boto3.resource("s3").Bucket("cgws")
+            logger.info("Downloading file {} from S3...".format(file_id))
+            try:
+                bucket.download_file("{}.mp3".format(file_id),mp3)
+            except:
+                logger.warning("Could not download file {} from S3.".format(file_id))
+                return
 
-    # output
-    text_out_dir = "/home/aaron/data/deepspeech_data/stm"
-    wav_out_dir = "/home/aaron/data/deepspeech_data/wav"
-    json_out_dir = "/home/aaron/data/deepspeech_data/alignments"
+        FNULL = open(os.devnull, 'w')
+        subprocess.call(["sox","{}".format(mp3),"-r","16k",
+                    "{}".format(wav),
+                    "remix","-"], stdout=FNULL, stderr=FNULL)
 
     # transcript
-    txt_file = "/home/aaron/data/records/{}.txt".format(file_id)
+    txt_file = os.path.join(records_dir, "{}.txt".format(file_id))
     logger.info("Reading transcript {}...".format(file_id))
     try:
         with open(txt_file,"r") as tr:
@@ -141,13 +153,17 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
             logger.info("Skipping paragraph {} (too few words)...".format(i))
             continue
 
-        temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"./temp")
+        temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp")
 
         # unique name of json object to read/write
         paragraph_hash = hashlib.sha1("{}{}{}{}".format(
                             file_id,paragraph,
                             paragraph_start,paragraph_end)).hexdigest()
-        json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
+
+        if use_filename_json is True:
+            json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end))
+        else:
+            json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash))
 
         result = None
 
@@ -203,14 +219,13 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
         # first two seconds will be skipped even if it contains a capture
         for catch in aligned["words"]:
-
             # successful capture
-            if catch["case"] == "success" and catch["alignedWord"] != "<unk>":
+            if catch["case"] == "success" and catch["alignedWord"] != "<unk>" and catch['start'] > 5 and catch['end'] - catch['start'] > .07:
 
                 # new capture group
                 if not current:
                     # begin capturing if it has been two seconds since the last word
-                    if catch["start"]-end_time > 2:
+                    if catch["start"]-end_time > 1:
                         current = [catch["alignedWord"]]
                         start_time = catch["start"]
                         end_time = catch["end"]
@@ -219,8 +234,8 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
                 else:
                     # large gap between last capture and this one
                     # likely that something was missing in the transcript
-                    if catch["start"]-end_time > .5:
-                        save_capture(captures,start_time,end_time,current,min_dur)
+                    if catch["start"]-end_time > 1:
+                        save_capture(captures,start_time,end_time,current)
                         current = []
 
                     # adding this word would equal or exceed max_length
@@ -268,9 +283,24 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False):
 
     # per-file logging
     total_dur = get_duration(mp3)
-    logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}."\
-          .format(total_captures,file_id,captures_dur,total_dur))
+    logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}, ratio {:.2f}."\
+          .format(total_captures,file_id,captures_dur,total_dur,captures_dur/total_dur))
 
     return
 
-
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Generate deepspeech data from Scribie transcripts')
+    parser.add_argument('file_id', type=str, help='file id to process')
+    parser.add_argument('--data_dir', type=str, help='path to data dir', default='/home/rajiv/host/align')
+    parser.add_argument('--use_filename_json', type=bool, help='read alignment json from filename_start_end.json file', default=True)
+    args = parser.parse_args()
+
+    data_dir = args.data_dir
+    records_dir = args.data_dir
+    mp3_dir = args.data_dir
+    text_out_dir = args.data_dir
+    wav_out_dir = args.data_dir
+    json_out_dir = args.data_dir
+    use_filename_json = True
+
+    data_generator(args.file_id)

check-files.sh

@@ -0,0 +1,12 @@
+#!/bin/zsh
+for i in `ls *.txt | sort`
+do
+    echo $i
+    echo -en "\033[32m"
+    cat $i
+    echo -en "\033[0m"
+    echo "playing audio..."
+    play -q ${i:0:-3}wav 1>/dev/null 2>&1
+    echo "press enter to play next file"
+    read
+done