From ab8176b21771530fa9e2d843313eb0fcfd391b6b Mon Sep 17 00:00:00 2001 From: James Betker Date: Thu, 28 Apr 2022 10:08:38 -0600 Subject: [PATCH] audio prep misc --- codes/scripts/audio/preparation/phase_1_split_files.py | 6 +++--- .../audio/preparation/phase_2_sample_and_filter.py | 10 +++++----- .../audio/preparation/phase_3_generate_similarities.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/codes/scripts/audio/preparation/phase_1_split_files.py b/codes/scripts/audio/preparation/phase_1_split_files.py index 653eade7..b403e71f 100644 --- a/codes/scripts/audio/preparation/phase_1_split_files.py +++ b/codes/scripts/audio/preparation/phase_1_split_files.py @@ -47,9 +47,9 @@ def process_file(file, base_path, output_path, progress_file): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('-path', type=str, help='Path to search for files', default='Y:\\sources\\big_podcast') - parser.add_argument('-progress_file', type=str, help='Place to store all files that have already been processed', default='Y:\\sources\\big_podcast\\already_processed.txt') - parser.add_argument('-output_path', type=str, help='Path for output files', default='Y:\\split\\big_podcast') + parser.add_argument('-path', type=str, help='Path to search for files', default='Y:\\clips\\red_rising') + parser.add_argument('-progress_file', type=str, help='Place to store all files that have already been processed', default='Y:\\clips\\red_rising\\already_processed.txt') + parser.add_argument('-output_path', type=str, help='Path for output files', default='Y:\\clips\\red_rising_split') parser.add_argument('-num_threads', type=int, help='Number of concurrent workers processing files.', default=4) args = parser.parse_args() diff --git a/codes/scripts/audio/preparation/phase_2_sample_and_filter.py b/codes/scripts/audio/preparation/phase_2_sample_and_filter.py index 350c6fe8..91fef310 100644 --- a/codes/scripts/audio/preparation/phase_2_sample_and_filter.py +++ b/codes/scripts/audio/preparation/phase_2_sample_and_filter.py @@ -72,7 +72,7 @@ def process_folder(folder, output_path, base_path, progress_file, max_files): stft = torch.stft(clip, n_fft=22000, hop_length=1024, return_complex=True) stft = stft[0, -2000:, :] return (stft.real ** 2 + stft.imag ** 2).sqrt() - no_hifreq_data = get_spec_mags(clips).mean(dim=1) < .15 + no_hifreq_data = get_spec_mags(clips).mean(dim=1) < .01 if torch.all(no_hifreq_data): continue @@ -104,11 +104,11 @@ def process_folder(folder, output_path, base_path, progress_file, max_files): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-path', type=str, help='Path to search for split files (should be the direct output of phase 1)', - default='Y:\\split\\youtube') - parser.add_argument('-progress_file', type=str, help='Place to store all folders that have already been processed', default='Y:\\filtered\\big_podcast\\already_processed.txt') - parser.add_argument('-output_path', type=str, help='Path where sampled&filtered files are sent', default='Y:\\filtered\\youtube') + default='Y:\\clips\\red_rising_split') + parser.add_argument('-progress_file', type=str, help='Place to store all folders that have already been processed', default='Y:\\clips\\red_rising_filtered\\already_processed.txt') + parser.add_argument('-output_path', type=str, help='Path where sampled&filtered files are sent', default='Y:\\clips\\red_rising_filtered') parser.add_argument('-num_threads', type=int, help='Number of concurrent workers processing files.', default=6) - parser.add_argument('-max_samples_per_folder', type=int, help='Maximum number of clips that can be extracted from each folder.', default=1000) + parser.add_argument('-max_samples_per_folder', type=int, help='Maximum number of clips that can be extracted from each folder.', default=999999) parser.add_argument('-classifier_model_opt', type=str, help='Train/test options file that configures the model used to classify the audio clips.', default='../options/test_noisy_audio_clips_classifier.yml') args = parser.parse_args() diff --git a/codes/scripts/audio/preparation/phase_3_generate_similarities.py b/codes/scripts/audio/preparation/phase_3_generate_similarities.py index 9468219a..ad564bda 100644 --- a/codes/scripts/audio/preparation/phase_3_generate_similarities.py +++ b/codes/scripts/audio/preparation/phase_3_generate_similarities.py @@ -92,7 +92,7 @@ def process_subdir(subdir, options, clip_sz): else: for i in range(1,n): # The first entry is always the file itself. top_ind = top3.indices[i] - simpaths.append(os.path.relpath(paths[top_ind], root)) + simpaths.append(str(os.path.relpath(paths[top_ind], root)).replace('\\', '/')) simmap[rel] = simpaths torch.save(simmap, output_file) @@ -107,7 +107,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-o', type=str, help='Path to the options YAML file used to train the CLIP model', default='../options/train_voice_voice_clip.yml') parser.add_argument('--num_workers', type=int, help='Number concurrent processes to use', default=4) - parser.add_argument('--root_path', type=str, help='Root path to search for audio directories from', default='Y:\\filtered\\youtube') + parser.add_argument('--root_path', type=str, help='Root path to search for audio directories from', default='Y:\\clips\\red_rising_filtered') parser.add_argument('--clip_size', type=int, help='Amount of audio samples to pull from each file', default=22050) args = parser.parse_args()