diff --git a/.gitignore b/.gitignore index 89357a6..bb7f401 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ ENV/ # OSX .DS_Store + +*.swp +inspect.sh diff --git a/align-ctm.sh b/align-ctm.sh new file mode 100755 index 0000000..87f641b --- /dev/null +++ b/align-ctm.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +#set -x +#set -euxo pipefail + +if [ $# -lt 3 ] +then + echo "usage: ./align-ctm.sh filelist index dataset_dir" + exit 1 +fi + +filelist=$1 +if [ ! -f $filelist ] +then + echo "$filelist not found" + exit 1 +fi + +index=$2 +dataset_dir=$3 + +for fid in `tail -n +$index $filelist` +do + num_files=`find ${dataset_dir}/txt/${fid}_*.txt 2>/dev/null | wc -l` + if [ $num_files -eq 0 ] + then + index=`grep -n $fid $filelist | cut -d ':' -f1` + python gaps.py $fid --file-index $index --audio-dir /home/aaron/data/mp3s/ --align-dir ~/data/ctm-alignments/ --dataset-dir ${dataset_dir} --speaker-turns + fi +done diff --git a/align-master.sh b/align-master.sh new file mode 100755 index 0000000..ebb6320 --- /dev/null +++ b/align-master.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#set -x + +if [ $# -lt 2 ] +then + echo "usage: ./align-master.sh filelist index" + exit 1 +fi + +filelist=$1 +if [ ! -f $filelist ] +then + echo "$filelist not found" + exit 1 +fi + +index=$2 + +for fid in `tail -n +$index $filelist` +do + touch ~/data/deepspeech_data/alignments/${fid}.json + index=`grep -n $fid $filelist | cut -d ':' -f1` + python rename-alignments.py $fid --file_index $index + python asr_data_gen.py --file $fid 2>>alignment.log +done diff --git a/align-worker.sh b/align-worker.sh new file mode 100755 index 0000000..9ee491d --- /dev/null +++ b/align-worker.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +#set -x + +if [ $# -lt 2 ] +then + echo "usage: ./align-worker.sh filelist index" + exit 1 +fi + +filelist=$1 +if [ ! -f $filelist ] +then + echo "$filelist not found" + exit 1 +fi + +if [ ! -d ~/align ] +then + mkdir ~/align +fi + +index=$2 +host=`hostname` + +threads_multiplier=4 +if [ $host == 'eesen-worker' ] +then + threads_multiplier=2 +fi + +for fid in `tail -n +$index $filelist` +do + host=`hostname` + if [ $host == 'eesen-worker' ] + then + stat ~/align/${fid}.json 1>/dev/null 2>&1 + else + ssh eesen-worker "stat ~/align/${fid}.json" 1>/dev/null 2>&1 + fi + + if [ $? -ne 0 ] + then + index=`grep -n $fid $filelist | cut -d ':' -f1` + + if [ $host == 'eesen-worker' ] + then + touch ~/align/${fid}.json + else + while true + do + ssh eesen-worker "touch ~/align/${fid}.json" 1>/dev/null 2>&1 + if [ $? -ne 0 ] + then + sleep 1 + else + break + fi + done + fi + + scp scribie:~/scribie/records/${fid}.txt ~/align 1>/dev/null 2>&1 + python rename-alignments.py $fid --file_index $index --abort --threads_multiplier $threads_multiplier --use_align_dir + fi +done diff --git a/aligner.py b/aligner.py index 43653e5..377b8d5 100644 --- a/aligner.py +++ b/aligner.py @@ -7,6 +7,7 @@ import hashlib import random import logging +import argparse import boto3 import gentle @@ -14,6 +15,14 @@ logging.basicConfig(level=logging.INFO,format="%(asctime)s - %(levelname)s - %(message)s",datefmt="%Y-%m-%d %H:%M:%S") logger = logging.getLogger("info_logger") +data_dir = '/home/aaron/data' +records_dir = os.path.join(data_dir, 'records') +mp3_dir = os.path.join(data_dir, 'mp3s') +text_out_dir = os.path.join(data_dir, 'deepspeech_data/stm') +wav_out_dir = os.path.join(data_dir, 'deepspeech_data/wav') +json_out_dir = os.path.join(data_dir, 'deepspeech_data/alignments') +use_filename_json = False + def clean(text): """Clean transcript of timestamps and speaker trackings, metas, punctuation, and extra whitespace. @@ -39,9 +48,11 @@ def trim(base_filename,audio_file,start,end,offset,out_directory): "{:07d}".format(int((offset+end)*100)))) duration = end-start - subprocess.call(["sox","{}".format(audio_file),"-r","16k", - "{}".format(segment),"trim","{}".format(start), - "{}".format(duration),"remix","-"]) + + if not os.path.isfile(segment): + subprocess.call(["sox","{}".format(audio_file),"-r","16k", + "{}".format(segment),"trim","{}".format(start), + "{}".format(duration),"remix","-"]) return segment @@ -82,25 +93,26 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False): logger.info("Processing file id {}...".format(file_id)) # grab audio file from s3 - mp3 = "/home/aaron/data/mp3s/{}.mp3".format(file_id) - - if not os.path.isfile(mp3): - bucket = boto3.resource("s3").Bucket("cgws") - logger.info("Downloading file {} from S3...".format(file_id)) - try: - bucket.download_file("{}.mp3".format(file_id),mp3) - except: - logger.warning("Could not download file {} from S3.".format(file_id)) - return + mp3 = os.path.join(mp3_dir, "{}.mp3".format(file_id)) + wav = os.path.join(mp3_dir, "{}.wav".format(file_id)) + if not os.path.isfile(wav): + if not os.path.isfile(mp3): + bucket = boto3.resource("s3").Bucket("cgws") + logger.info("Downloading file {} from S3...".format(file_id)) + try: + bucket.download_file("{}.mp3".format(file_id),mp3) + except: + logger.warning("Could not download file {} from S3.".format(file_id)) + return - # output - text_out_dir = "/home/aaron/data/deepspeech_data/stm" - wav_out_dir = "/home/aaron/data/deepspeech_data/wav" - json_out_dir = "/home/aaron/data/deepspeech_data/alignments" + FNULL = open(os.devnull, 'w') + subprocess.call(["sox","{}".format(mp3),"-r","16k", + "{}".format(wav), + "remix","-"], stdout=FNULL, stderr=FNULL) # transcript - txt_file = "/home/aaron/data/records/{}.txt".format(file_id) + txt_file = os.path.join(records_dir, "{}.txt".format(file_id)) logger.info("Reading transcript {}...".format(file_id)) try: with open(txt_file,"r") as tr: @@ -141,13 +153,17 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False): logger.info("Skipping paragraph {} (too few words)...".format(i)) continue - temp_wav = trim(file_id,mp3,paragraph_start,paragraph_end,0,"./temp") + temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp") # unique name of json object to read/write paragraph_hash = hashlib.sha1("{}{}{}{}".format( file_id,paragraph, paragraph_start,paragraph_end)).hexdigest() - json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash)) + + if use_filename_json is True: + json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end)) + else: + json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash)) result = None @@ -203,14 +219,13 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False): # first two seconds will be skipped even if it contains a capture for catch in aligned["words"]: - # successful capture - if catch["case"] == "success" and catch["alignedWord"] != "": + if catch["case"] == "success" and catch["alignedWord"] != "" and catch['start'] > 5 and catch['end'] - catch['start'] > .07: # new capture group if not current: # begin capturing if it has been two seconds since the last word - if catch["start"]-end_time > 2: + if catch["start"]-end_time > 1: current = [catch["alignedWord"]] start_time = catch["start"] end_time = catch["end"] @@ -219,8 +234,8 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False): else: # large gap between last capture and this one # likely that something was missing in the transcript - if catch["start"]-end_time > .5: - save_capture(captures,start_time,end_time,current,min_dur) + if catch["start"]-end_time > 1: + save_capture(captures,start_time,end_time,current) current = [] # adding this word would equal or exceed max_length @@ -268,9 +283,24 @@ def data_generator(file_id,min_dur=2,max_dur=(5,20),randomize=False): # per-file logging total_dur = get_duration(mp3) - logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}."\ - .format(total_captures,file_id,captures_dur,total_dur)) + logger.info("Wrote {} segments from {}, totalling {} seconds, out of a possible {}, ratio {:.2f}."\ + .format(total_captures,file_id,captures_dur,total_dur,captures_dur/total_dur)) return - +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate deepspeech data from Scribie transcripts') + parser.add_argument('file_id', type=str, help='file id to process') + parser.add_argument('--data_dir', type=str, help='path to data dir', default='/home/rajiv/host/align') + parser.add_argument('--use_filename_json', type=bool, help='read alignment json from filename_start_end.json file', default=True) + args = parser.parse_args() + + data_dir = args.data_dir + records_dir = args.data_dir + mp3_dir = args.data_dir + text_out_dir = args.data_dir + wav_out_dir = args.data_dir + json_out_dir = args.data_dir + use_filename_json = True + + data_generator(args.file_id) diff --git a/check-files.sh b/check-files.sh new file mode 100755 index 0000000..9b462ba --- /dev/null +++ b/check-files.sh @@ -0,0 +1,12 @@ +#!/bin/zsh +for i in `ls *.txt | sort` +do + echo $i + echo -en "\033[32m" + cat $i + echo -en "\033[0m" + echo "playing audio..." + play -q ${i:0:-3}wav 1>/dev/null 2>&1 + echo "press enter to play next file" + read +done diff --git a/difficult_words.py b/difficult_words.py index 621d5b1..8fa9383 100644 --- a/difficult_words.py +++ b/difficult_words.py @@ -1,13 +1,14 @@ +import sys import argparse import json import os import subprocess parser = argparse.ArgumentParser() -parser.add_argument("--file_id") +parser.add_argument('file_id', type=str, help='file id to process') +parser.add_argument('--window-len', type=str, dest='window_len', help='number of words to look around the mismatch', default=5) args = parser.parse_args() - ctm_file = "".join((args.file_id,"_align.json")) audio_file = ".".join((args.file_id,"mp3")) @@ -22,43 +23,101 @@ def trim(base_filename,audio_file,start,end,offset): "{:07d}".format(int((offset+start)*100)), "{:07d}".format(int((offset+end)*100)))) duration = end-start + FNULL = open(os.devnull, 'w') subprocess.call(["sox","{}".format(audio_file),"-r","16k", "{}".format(segment),"trim","{}".format(start), - "{}".format(duration),"remix","-"]) + "{}".format(duration),"remix","-"], stdout=FNULL, stderr=FNULL) return +def save_txt(base_filename,txt,start,end,offset): + txt_file = os.path.join(".","{}_{}_{}.txt".format( + base_filename, + "{:07d}".format(int((offset+start)*100)), + "{:07d}".format(int((offset+end)*100)))) + with open(txt_file, 'w') as f: + f.write(txt + "\n") with open(ctm_file) as f: - ctm = json.loads(f.read()) + ctms = json.loads(f.read()) # could eventually put it in a single loop, like #for word in json.loads(f.read()): -last_end = 0 -capturing = False -for word in ctm: - if not capturing: - # start capturing - if word['case'] == 'mismatch': - clip_start = last_end #word['start']-(word['start']-last_end)/2. - last_end = word['end'] - capturing = True - - # just keep track of the end of this word - elif word['case'] == 'success': # else: - last_end = word['end'] - else: - # capture this word too - if word['case'] == 'mismatch': - last_end = word['end'] - - # stop capturing and write segment if it's long enough - elif word['case'] == 'success': # else: - clip_end = word['start'] #word['start']-(word['start']-last_end)/2. - words = [word['word'] for word in ctm \ - if word['start'] >= clip_start and word['end'] <= clip_end] - if len(words) > 2: - print(words) - trim(args.file_id,audio_file,clip_start,clip_end,0) - capturing = False +total_seconds = 0 +last_capture_end = 0 +for index, ctm in enumerate(ctms): + if ctm['case'] == 'mismatch': + start_index = None + end_index = None + captures = [] + + # we don't want overlaps between the segments + if ctm['start'] < last_capture_end: + continue + + if index - args.window_len < 0: + # most probably this is mismatch is at the start + continue + + if index + args.window_len > len(ctms): + # most probably this is mismatch is towards the end + continue + + # find the previous word which is a success with a decent gap + for i in range(index - args.window_len, 0, -1): + p = i - 1 + if p < 0: + break; + + gap = ctms[i]['start'] - ctms[p]['end'] + if ctms[i]['case'] == 'success' and ctms[p]['case'] == 'success' and gap > 0.25 and ctms[p]['end'] > last_capture_end: + start_index = i + break; + + if not start_index: + continue; + + # find the next word which is a success with a decent gap + for i in range(index + args.window_len, len(ctms)): + n = i + 1 + if n >= len(ctms): + break; + + gap = ctms[n]['start'] - ctms[i]['end'] + if ctms[i]['case'] == 'success' and ctms[n]['case'] == 'success' and gap > 0.25: + end_index = i + break; + + if not end_index: + continue; + + captures = ctms[start_index:end_index+1] + + # if the gap between the words is too long in the captured segment then we should skip it + # since it might be fillers or background conversation + for i, c in enumerate(captures): + if i+1 >= len(captures): + break; + + if captures[i+1]['start'] - c['end'] > 2: + captures = [] + break; + + # discard too short segments + if len(captures) < 5: + continue; + + words = ' '.join([c['word'] for c in captures]) + print("mistmatch captured at index {}, start_index {}, end_index {}, {}".format(index, start_index, end_index, words)) + + clip_start = captures[0]['start'] + clip_end = captures[-1]['end'] + + total_seconds += clip_end - clip_start + + trim(args.file_id,audio_file, clip_start, clip_end, 0) + save_txt(args.file_id, words, clip_start, clip_end, 0) + + last_capture_end = clip_end +print("\nratio {:.2f}".format(total_seconds/(ctms[-1]['end'] - ctms[0]['start']))) diff --git a/gaps.py b/gaps.py new file mode 100644 index 0000000..a748b4e --- /dev/null +++ b/gaps.py @@ -0,0 +1,123 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import argparse +import json +import os +import subprocess +import re +from tqdm import tqdm +import logging +import boto3 +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('file_id', type=str, help='file id to process') +parser.add_argument("--min-gap", default=0.25, type=float, dest="min_gap", help="minimum gap between words to use for splitting") +parser.add_argument('--file-index', type=str, default="1", dest="file_index", help='file index to print') +parser.add_argument('--audio-dir', type=str, dest="audio_dir", default='.', help='Path to the directory containing audio files') +parser.add_argument('--align-dir', type=str, dest="align_dir", default=".", help='Path to the directory containing alignments') +parser.add_argument('--dataset-dir', type=str, dest="dataset_dir", default='.', help='Path to the dataset directory') +parser.add_argument('--speaker-turns', action="store_true", dest="speaker_turns", default=False, help='Add speaker turn markers') +args = parser.parse_args() + +logging.basicConfig(filename='punctuations.log', level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S") +logger = logging.getLogger("info_logger") + +ctm_file = os.path.join(args.align_dir, args.file_id + "_align.json") +mp3 = os.path.join(args.audio_dir, args.file_id + ".mp3") +wav = os.path.join(args.audio_dir, args.file_id + ".wav") +FNULL = open("/dev/null") + +if not os.path.isdir(os.path.join(args.dataset_dir, "txt")): + os.makedirs(os.path.join(args.dataset_dir, "txt")) + +if not os.path.isdir(os.path.join(args.dataset_dir, "wav")): + os.makedirs(os.path.join(args.dataset_dir, "wav")) + +if not os.path.isfile(ctm_file): + sys.exit() + +if not os.path.isfile(wav): + if not os.path.isfile(mp3): + bucket = boto3.resource("s3").Bucket("cgws") + try: + bucket.download_file("{}.mp3".format(args.file_id), mp3) + except: + print("Could not download file {} from S3.".format(args.file_id)) + sys.exit() + + subprocess.call(["sox","{}".format(mp3),"-r","16k", "{}".format(wav), "remix","-"], stdout=FNULL, stderr=FNULL) + +# leaving offset here in case the algo changes to require it +def sox_trim(start, end): + """Write out a segment of an audio file to wav, based on start, end, + """ + clip_file = os.path.join(args.dataset_dir, "wav/{}_{:07d}_{:07d}.wav".format(args.file_id, int(start*100), int(end*100))) + ret = subprocess.call(["sox", wav, clip_file, "trim", str(start), str(end - start)], stdout=FNULL, stderr=FNULL) + if ret != 0: + logger.error("sox failed: {}, error: {}".format(clip_file, ret)) + sys.exit() + +with open(ctm_file) as f: + ctms = json.loads(f.read()) + + null_word = {'start': 0, 'end':0} + gaps = [second['start']-first['end'] for first, second in zip([null_word]+ctms, ctms)] + # we split from one good gap to the next + # a good gap is when the silence between the words is long + # and the word *itself* is long and is not a mismatch + good_gaps = [(i, gap) for i, gap in enumerate(gaps) if gap > args.min_gap and ctms[i]['duration'] > args.min_gap and ctms[i]['case'] != 'mismatch'] + + total_written = 0 + for i in tqdm(range(len(good_gaps)), desc="({}) {}".format(args.file_index, args.file_id), ncols=100): + ctm_index = good_gaps[i][0] + gap = good_gaps[i][1] + + # to prevent out of bounds + if i+1 >= len(good_gaps): + continue + + # we start splitting from this ctm_index to the next good gap + start_index = ctm_index + end_index = good_gaps[i+1][0] + + # this is our clip + clip = ctms[start_index:end_index] + + n_words = len(clip) + count = 0 + if args.speaker_turns: + words = " ".join([word["orig"] for word in clip]).encode('utf-8').strip() + words = re.sub("\-", " ", words) + # trying with some punctuation retained + words = re.sub(r"[\?!]",".",words) + words = re.sub(r"[^a-zA-Z0-9¶\.\,\' ]", "", words, re.UNICODE) + words = re.sub(r"^¶|¶$", "", words) + words = re.sub(r"¶", " ¶", words) + words = re.sub(r"¶$", "", words) + words = re.sub("\s{2,}", " ", words) + words = words.lower() + # count = words.count('¶') + else: + words = " ".join([word["word"] for word in clip]).encode('utf-8').strip() + count = sum([word['case'] == 'mismatch' for word in clip]) + + # if n_words >= 5 and count >= 1: + if n_words >= 5: + start_sec = clip[0]['start'] + end_sec = clip[-1]['end'] + + sox_trim(start_sec, end_sec) + + txt_file = os.path.join(args.dataset_dir, "txt/{}_{:07d}_{:07d}.txt".format(args.file_id, int(start_sec*100), int(end_sec*100))) + + with open(txt_file, "w") as f: + f.write(words + "\n") + + duration = end_sec - start_sec + logger.info("{}: gap {}s, start {}s, end {}s, duration {}s".format(ctm_index, gap, start_sec, end_sec, duration)) + + total_written += duration + +logger.info("Wrote {} seconds out of {} ({:.2f}%) for {}.".format(total_written,ctms[-1]['end'],(total_written/ctms[-1]['end'])*100, args.file_id)) diff --git a/gen-ctm-align.py b/gen-ctm-align.py new file mode 100644 index 0000000..c31319b --- /dev/null +++ b/gen-ctm-align.py @@ -0,0 +1,15 @@ +import argparse +import os +import subprocess +from tqdm import tqdm + +parser = argparse.ArgumentParser() +parser.add_argument('manifest', type=str, help='list of file ids') +args = parser.parse_args() + +files = open(args.manifest).read().strip().split('\n') + +for i in tqdm(range(len(files)), ncols=100): + if os.path.exists(files[i] + ".ctm"): + subprocess.call("node ctm-align.js " + files[i], shell=True) + diff --git a/generator.py b/generator.py index db91709..0f0c703 100644 --- a/generator.py +++ b/generator.py @@ -62,7 +62,7 @@ def data_generator(file_path,shuffle,vocabulary_size,train_size,test_size): if "_" in sentence: continue - sentence = re_decimal.sub("\\1 point \\2", sentence) # decimals + #sentence = re_decimal.sub("\\1 point \\2", sentence) # decimals sentence = re_url.sub(" dot \\1", sentence) # URLs sentence = re_negative.sub(" minus \\1", sentence) # negative numbers sentence = re_pattern.sub(" ", sentence).lower() # space in case it was connected to a word @@ -84,6 +84,7 @@ def data_generator(file_path,shuffle,vocabulary_size,train_size,test_size): doc_sents.append(sentence) sentences.extend([sent.split() for sent in doc_sents]) + print("{} sentences after {}.".format(len(sentences),txt_file)) # word frequencies word_freq = nltk.FreqDist(itertools.chain(*sentences)) diff --git a/pytorch_manifest.py b/pytorch_manifest.py index 2fc7546..e57359c 100644 --- a/pytorch_manifest.py +++ b/pytorch_manifest.py @@ -1,40 +1,187 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- import os import argparse import scipy.io.wavfile as wav +import re +import shutil +from tqdm import tqdm +import numpy as np +import sys +import subprocess +import soundfile as sf +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt parser = argparse.ArgumentParser() -parser.add_argument("--files_dir",default="/home/aaron/data/deepspeech_data",type=str) -parser.add_argument("--out_file",default="./train_manifest.csv",type=str) +parser.add_argument("--data-dir", dest="data_dir", type=str, default="/home/aaron/data/phoenix-files/gaps", help='Directory to read files from') +parser.add_argument("--dst-dir", dest="dst_dir", default="/home/aaron/data/phoenix-files/pytorch", type=str, help="Directory to store dataset to") +parser.add_argument("--min-seconds", dest="min_seconds", default=2.0, type=float, help="Cutoff for minimum duration") +parser.add_argument("--max-seconds", dest="max_seconds", default=20.0, type=float, help="Cutoff for maximum duration") +parser.add_argument("--max-hours", dest="max_hours", default=0, type=int, help="Size of the dataset in hours") +parser.add_argument("--merge-ted", dest="merge_ted", help="Merge with TED dataset", action='store_true', default=False) +parser.add_argument("--dry-run", dest="dry_run", help="Don't write manifest csv's", action='store_true', default=False) +parser.add_argument("--split-ratio", dest="split_ratio", type=float, default=0.01, help="Percent of files to keep in val & test set") +parser.add_argument("--txt-dir", dest="txt_dir", type=str, default="txt", help="Directory name for txt files") +parser.add_argument("--prefix", dest="prefix", type=str, default="unnamed", help="Prefix for manifest files") +parser.add_argument("--manifest-dir", dest="manifest_dir", type=str, default=".", help="Directory for manifest files") args = parser.parse_args() -parent_dir = os.path.abspath(args.files_dir) +parent_dir = os.path.abspath(args.data_dir) wav_dir = os.path.join(parent_dir,"wav") -txt_dir = os.path.join(parent_dir,"stm") +txt_dir = os.path.join(parent_dir, args.txt_dir) + +dst_dir = os.path.abspath(args.dst_dir) +dst_wav = os.path.join(dst_dir, "wav") +if not os.path.exists(dst_wav): + os.makedirs(dst_wav) + +dst_txt = os.path.join(dst_dir, args.txt_dir) +if not os.path.exists(dst_txt): + os.makedirs(dst_txt) + +def get_duration(wav_file): + m = re.search("[a-z0-9]+_([0-9]+)_([0-9]+).wav", wav_file) + if m: + duration = (int(m.group(2)) - int(m.group(1)))/100. + return duration + + try: + f = sf.SoundFile(wav_file) + if f.samplerate != 16000: + print("sample rate is {}".format(f.samplerate)) + return 0 + else: + return len(f)/float(f.samplerate) + except: + return 0 + +def sort_func(element): + return element[0] keep_files = [] +files= [] + +FNULL = open(os.devnull, 'w') +out, err = subprocess.Popen("find " + wav_dir + " -type f -size +100c | wc -l", stdout=subprocess.PIPE, shell=True).communicate() +num_files = int(out) + +find = subprocess.Popen("find " + wav_dir + " -type f -size +100c", stdout=subprocess.PIPE, stderr=FNULL, shell=True) +for i in tqdm(range(num_files), ncols=100, desc='Finding files'): + line = find.stdout.readline() + if len(line.strip()) > 0: + files.append(os.path.basename(line)) + else: + break -# get filenames from wav directory -for i,filename in enumerate(os.listdir(wav_dir)): - if i % 10000 == 0: - print("Processing file {}".format(i)) +total_hours = 0 +# get filenames wav directory +for i in tqdm(range(num_files), ncols=100, desc='Checking files'): + filename = files[i] fid = os.path.splitext(filename)[0] wav_file = os.path.join(wav_dir,"{}.wav".format(fid)) - samp_rate,data = wav.read(wav_file) + duration = get_duration(wav_file) + if duration < args.min_seconds or duration > args.max_seconds: + continue + + + txt_file = os.path.join(txt_dir,"{}.txt".format(fid)) + with open(txt_file) as raw_text: + transcript = raw_text.read().strip() + + oov = re.search("[^a-zA-Z '¶]", transcript) + if oov is not None: + continue + + dst_txt_file = os.path.join(dst_txt, "{}.txt".format(fid)) + dst_wav_file = os.path.join(dst_wav, "{}.wav".format(fid)) + + if not args.dry_run: + # replacing pilcrows with pipes + transcript = re.sub('¶','|',transcript) + transcript = re.sub('\s+', ' ', transcript).upper() + "\n" + with open(dst_txt_file, 'w') as f: + f.write(transcript) + + if not os.path.isfile(dst_wav_file): + shutil.copy2(wav_file, dst_wav_file) + + keep_files.append((duration, "{},{}".format(dst_wav_file,dst_txt_file))) + + total_hours += duration/3600 + if args.max_hours != 0 and args.max_hours <= total_hours: + print("\n") + break + +val_len = int(len(keep_files) * args.split_ratio) + +train_set = keep_files[:-val_len*2] +val_set = keep_files[len(train_set):-val_len] +test_set = keep_files[-val_len:] + +if args.merge_ted and not args.dry_run: + ted_train = [] + for line in open("ted_train_manifest.csv","r"): + ted_train.append(line) + + for i in tqdm(range(len(ted_train)), ncols=100, desc='Merging TED train'): + line = ted_train[i] + duration = get_duration(line.split(',')[0]) + train_set.append((duration, line)) + + ted_val = [] + for line in open("ted_test_manifest.csv","r"): + ted_val.append(line) + + for i in tqdm(range(len(ted_val)), ncols=100, desc='Merging TED val'): + line = ted_val[i] + duration = get_duration(line.split(',')[0]) + val_set.append((duration, line)) + +train_set.sort(key=sort_func) +val_set.sort(key=sort_func) +test_set.sort(key=sort_func) + +total_train = sum([line[0] for line in train_set]) +if not args.dry_run: + with open(os.path.join(args.manifest_dir, "{}-train-{}.csv".format(args.prefix, int(total_hours))), 'w') as f: + for line in train_set: + f.write((line[1].strip() + "\n").encode('utf-8')) + +total_val = sum([line[0] for line in val_set]) +if not args.dry_run: + with open(os.path.join(args.manifest_dir, "{}-val-{}.csv".format(args.prefix, int(total_hours))), 'w') as f: + for line in val_set: + f.write((line[1].strip() + "\n").encode('utf-8')) + +total_test = sum([line[0] for line in test_set]) +if not args.dry_run: + with open(os.path.join(args.manifest_dir, "{}-test-{}.csv".format(args.prefix, int(total_hours))), 'w') as f: + for line in test_set: + f.write((line[1].strip() + "\n").encode('utf-8')) - # duration (number of frames divided by framerate) greater than one second - if len(data)/float(samp_rate) >= 1: +if not args.dry_run: + # train_set has already been written so modifying in place is ok + np.random.shuffle(train_set) + train_subset = train_set[:len(val_set)/2] + with open(os.path.join(args.manifest_dir, "{}-train-subset-{}.csv".format(args.prefix, int(total_hours))), 'w') as f: + for line in train_subset: + f.write((line[1].strip() + "\n").encode("utf-8")) - txt_file = os.path.join(txt_dir,"{}.txt".format(fid)) - with open(txt_file) as raw_text: - transcript = raw_text.read().strip() +total = total_train + total_val + total_test - # at least two words in transcript - if len(transcript.split()) > 1 : - keep_files.append("{},{}".format(wav_file,txt_file)) +durations = [t[0] for t in train_set] +bins = np.arange(int(args.min_seconds), int(args.max_seconds) + 1) +plt.hist(durations, bins=bins, rwidth=0.8) +plt.xlabel('Seconds') +plt.ylabel('# of files') +plt.grid(color='gray', linestyle='dotted') +plt.xticks(bins) +plt.title("Durations distribution for {} {} hours".format(args.prefix, int(total_hours))) +plt.savefig('durations.png') -# write out all acceptable files to the same file -with open(args.out_file,"w") as out: - out.write("\n".join(keep_files)) +print("Total {:.2f} hours, train {:.2f} hours, val {:.2f} hours, test {:.2f}, ratio {:.5f}/{:.5f}/{:.5f}".format(total/3600, total_train/3600, total_val/3600, total_test/3600, total_train/total, total_val/total, total_test/total)) diff --git a/rename-alignments.py b/rename-alignments.py new file mode 100644 index 0000000..db5911f --- /dev/null +++ b/rename-alignments.py @@ -0,0 +1,172 @@ +import os +import sys +import re +import multiprocessing +import json +import subprocess +import hashlib +import random +from shutil import copyfile +import argparse +from tqdm import tqdm +import traceback + +import boto3 +import gentle + +parser = argparse.ArgumentParser(description='Generate paragraph level alignments from Scribie data') +parser.add_argument('file_id', type=str, help='file id to process') +parser.add_argument('--file_index', type=str, help='file index to print', default="1") +parser.add_argument('--abort', help='Abort if alignemnt already exists', action='store_true', default=False) +parser.add_argument('--threads_multiplier', help='Multiplier for threads', type=int, default=1) +parser.add_argument('--use_align_dir', help='Read/write files from an align directory', action='store_true', default=False) +parser.add_argument('--align_dir', help='Path to the align directory', type=str, default='~/align') +args = parser.parse_args() + +def clean(text): + """Clean transcript of timestamps and speaker trackings, metas, + punctuation, and extra whitespace. + """ + + text = re.sub("\d:\d+:\d+\.\d S(\d+|\?): ","",text) + text = re.sub("\[.+?\]","",text) + text = re.sub("\-"," ",text) + text = re.sub(r"[^a-zA-Z0-9\' ]","",text,re.UNICODE) + ### don't worry about converting to ascii for now + cleaned = re.sub("\s{2,}"," ",text) + + return cleaned + +def trim(base_filename,audio_file,start,end,offset,out_directory): + """Write out a segment of an audio file to wav, based on start, end, + and offset times in seconds. + """ + + segment = os.path.join(out_directory,"{}_{}_{}.wav".format( + base_filename, + "{:07d}".format(int((offset+start)*100)), + "{:07d}".format(int((offset+end)*100)))) + + duration = end-start + FNULL = open(os.devnull, 'w') + subprocess.call(["sox","{}".format(audio_file),"-r","16k", + "{}".format(segment),"trim","{}".format(start), + "{}".format(duration),"remix","-"], stdout=FNULL, stderr=FNULL) + + return segment + +def get_duration(audio_file): + """Determine the length of an audio file in seconds""" + + duration = float(subprocess.Popen( + ["soxi","-D","{}".format(audio_file)], + stdout=subprocess.PIPE).stdout.read().strip()) + + return duration + +if __name__ == '__main__': + + file_id = args.file_id + + # output + if args.use_align_dir: + align_dir = wav_out_dir = json_out_dir = mp3_dir = os.path.expanduser(args.align_dir) + txt_file = "{}/{}.txt".format(align_dir, file_id) + else: + wav_out_dir = "/home/aaron/data/deepspeech_data/wav" + json_out_dir = "/home/aaron/data/deepspeech_data/alignments" + mp3_dir = "/home/aaron/data/mp3s/" + txt_file = "/home/aaron/data/records/{}.txt".format(file_id) + + mp3 = "{}/{}.mp3".format(mp3_dir,file_id) + wav = "{}/{}.wav".format(mp3_dir,file_id) + + try: + with open(txt_file,"r") as tr: + transcript = tr.read() + except IOError: + print("File {} does not exist.".format(txt_file)) + sys.exit() + + if not os.path.isfile(wav): + if not os.path.isfile(mp3): + bucket = boto3.resource("s3").Bucket("cgws") + try: + bucket.download_file("{}.mp3".format(file_id),mp3) + except: + print("Could not download file {} from S3.".format(file_id)) + sys.exit() + + FNULL = open(os.devnull, 'w') + subprocess.call(["sox","{}".format(mp3),"-r","16k", + "{}".format(wav), + "remix","-"], stdout=FNULL, stderr=FNULL) + + # split transcript by speaker, and get timestamps (as seconds) + # of the boundaries of each paragraph + paragraphs = [] + times = [] + for paragraph in transcript.split("\n"): + catch = re.match("\d:\d+:\d+\.\d",paragraph) + if catch: + timestamp = catch.group() + h,m,s = timestamp.split(":") + time = int(h)*60*60 + int(m)*60 + float(s) + paragraphs.append(paragraph) + times.append(time) + file_end = get_duration(mp3) + times.append(file_end) + + for i in tqdm(range(len(paragraphs)), desc="({}) {}".format(args.file_index, file_id), ncols=100): + paragraph = paragraphs[i] + paragraph_start, paragraph_end = times[i], times[i+1] + + if paragraph_end - paragraph_start <= 0.2: + continue + + # unique name of json object to read/write + paragraph_hash = hashlib.sha1("{}{}{}{}".format( + file_id,paragraph, + paragraph_start,paragraph_end)).hexdigest() + json_file = os.path.join(json_out_dir,"{}.json".format(paragraph_hash)) + new_json_file = os.path.join(json_out_dir,"{}_{}_{}.json".format(file_id, paragraph_start, paragraph_end)) + if os.path.isfile(new_json_file): + if args.abort: + print(" aborting") + break + else: + if not os.path.isfile(json_file): + + temp_wav = trim(file_id,wav,paragraph_start,paragraph_end,0,"/tmp") + + if not os.path.isfile(temp_wav): + continue + + try: + with gentle.resampled(temp_wav) as wav_file: + resources = gentle.Resources() + cleaned = clean(paragraph) + aligner = gentle.ForcedAligner(resources,cleaned, + nthreads=multiprocessing.cpu_count()*args.threads_multiplier, + disfluency=False,conservative=False, + disfluencies=set(["uh","um"])) + result = aligner.transcribe(wav_file) + + aligned_words = result.to_json() + with open(json_file,"w") as f: + f.write(aligned_words) + + except: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + print ''.join(line for line in lines) + continue + + copyfile(json_file, new_json_file) + + if os.path.isfile(temp_wav): + os.remove(temp_wav) + + if args.use_align_dir: + os.remove(wav) + os.remove(mp3) diff --git a/sentence_cleaner.py b/sentence_cleaner.py new file mode 100644 index 0000000..f57df8c --- /dev/null +++ b/sentence_cleaner.py @@ -0,0 +1,38 @@ +import re +import nltk + + +def clean(text,word_to_id): + + unknown_token="UNK" + sentence_start_token="START" + sentence_end_token="END" + + # metas and punctuation (keeping hyphens) + timestamps = "\d+:\d+:\d+(\.\d+)*" + speakers = "S\d*:" + metas = "\[.{5,24}\]" + punctuation = "[\_\"\.\!\?\:]+" + new_line = "\n" + pattern = "|".join([timestamps,speakers,metas,punctuation,new_line]) + + # numbers not part of a word like CO2, mp3, etc. + numbers = "(?/dev/null 2>&1 + if [ $? -eq 0 ] + then + while true + do + scp -p /home/rajiv/align/$file eesen-worker:~/align/ + if [ $? -eq 0 ] + then + break + else + sleep 1 + fi + done + + fid=`echo $file | cut -d '_' -f1` + timeout 90 rsync -T /tmp -avztuq --timeout=60 -e ssh eesen-worker:~/align/$fid*.json ~/align/ + fi +done