DL-Art-School/codes/data/audio/unsupervised_audio_dataset.py

import os
import pathlib
import random

import torch
import torch.utils.data
import torch.nn.functional as F
import torchaudio
from audio2numpy import open_audio
from tqdm import tqdm

from data.audio.wav_aug import WavAugmentor
from data.util import find_files_of_type, is_wav_file, is_audio_file
from models.tacotron2.taco_utils import load_wav_to_torch
from utils.util import opt_get


def load_audio(audiopath, sampling_rate):
    if audiopath[-4:] == '.wav':
        audio, lsr = load_wav_to_torch(audiopath)
    else:
        audio, lsr = open_audio(audiopath)
        audio = torch.FloatTensor(audio)

    # Remove any channel data.
    if len(audio.shape) > 1:
        if audio.shape[0] < 5:
            audio = audio[0]
        else:
            assert audio.shape[1] < 5
            audio = audio[:, 0]

    if lsr != sampling_rate:
        if lsr < sampling_rate:
            print(f'{audiopath} has a sample rate of {sampling_rate} which is lower than the requested sample rate of {sampling_rate}. This is not a good idea.')
        audio = torch.nn.functional.interpolate(audio.unsqueeze(0).unsqueeze(1), scale_factor=sampling_rate/lsr, mode='nearest', recompute_scale_factor=False).squeeze()

    # Check some assumptions about audio range. This should be automatically fixed in load_wav_to_torch, but might not be in some edge cases, where we should squawk.
    # '2' is arbitrarily chosen since it seems like audio will often "overdrive" the [-1,1] bounds.
    if torch.any(audio > 2) or not torch.any(audio < 0):
        print(f"Error with {audiopath}. Max={audio.max()} min={audio.min()}")
    audio.clip_(-1, 1)

    return audio.unsqueeze(0)


class UnsupervisedAudioDataset(torch.utils.data.Dataset):

    def __init__(self, opt):
        path = opt['path']
        cache_path = opt['cache_path']  # Will fail when multiple paths specified, must be specified in this case.
        if not isinstance(path, list):
            path = [path]
        if os.path.exists(cache_path):
            self.audiopaths = torch.load(cache_path)
        else:
            print("Building cache..")
            self.audiopaths = []
            for p in path:
                self.audiopaths.extend(find_files_of_type('img', p, qualifier=is_audio_file)[0])
            torch.save(self.audiopaths, cache_path)

        # Parse options
        self.sampling_rate = opt_get(opt, ['sampling_rate'], 22050)
        self.pad_to = opt_get(opt, ['pad_to_seconds'], None)
        if self.pad_to is not None:
            self.pad_to *= self.sampling_rate
        self.pad_to = opt_get(opt, ['pad_to_samples'], self.pad_to)

        self.extra_samples = opt_get(opt, ['extra_samples'], 0)
        self.extra_sample_len = opt_get(opt, ['extra_sample_length'], 2)
        self.extra_sample_len *= self.sampling_rate

    def get_audio_for_index(self, index):
        audiopath = self.audiopaths[index]
        audio = load_audio(audiopath, self.sampling_rate)
        return audio, audiopath

    def get_related_audio_for_index(self, index):
        if self.extra_samples <= 0:
            return None, 0
        audiopath = self.audiopaths[index]
        related_files = find_files_of_type('img', os.path.dirname(audiopath), qualifier=is_audio_file)[0]
        assert audiopath in related_files
        assert len(related_files) < 50000  # Sanity check to ensure we aren't loading "related files" that aren't actually related.
        if len(related_files) == 0:
            print(f"No related files for {audiopath}")
        related_files.remove(audiopath)
        related_clips = []
        random.shuffle(related_clips)
        i = 0
        for related_file in related_files:
            rel_clip = load_audio(related_file, self.sampling_rate)
            gap = rel_clip.shape[-1] - self.extra_sample_len
            if gap < 0:
                rel_clip = F.pad(rel_clip, pad=(0, abs(gap)))
            elif gap > 0:
                rand_start = random.randint(0, gap)
                rel_clip = rel_clip[:, rand_start:rand_start+self.extra_sample_len]
            related_clips.append(rel_clip)
            i += 1
            if i >= self.extra_samples:
                break
        actual_extra_samples = i
        while i < self.extra_samples:
            related_clips.append(torch.zeros(1, self.extra_sample_len))
            i += 1
        return torch.stack(related_clips, dim=0), actual_extra_samples

    def __getitem__(self, index):
        try:
            # Split audio_norm into two tensors of equal size.
            audio_norm, filename = self.get_audio_for_index(index)
            alt_files, actual_samples = self.get_related_audio_for_index(index)
        except:
            print(f"Error loading audio for file {filename} or {alt_files}")
            return self[index+1]

        # This is required when training to make sure all clips align.
        if self.pad_to is not None:
            if audio_norm.shape[-1] <= self.pad_to:
                audio_norm = torch.nn.functional.pad(audio_norm, (0, self.pad_to - audio_norm.shape[-1]))
            else:
                gap = audio_norm.shape[-1] - self.pad_to
                start = random.randint(0, gap-1)
                audio_norm = audio_norm[:, start:start+self.pad_to]

        output = {
            'clip': audio_norm,
            'path': filename,
        }
        if self.extra_samples > 0:
            output['alt_clips'] = alt_files
            output['num_alt_clips'] = actual_samples
        return output

    def __len__(self):
        return len(self.audiopaths)


if __name__ == '__main__':
    params = {
        'mode': 'unsupervised_audio',
        'path': ['Z:\\split\\cleaned\\books0', 'Z:\\split\\cleaned\\books2'],
        'cache_path': 'E:\\audio\\remote-cache.pth',
        'sampling_rate': 22050,
        'pad_to_seconds': 5,
        'phase': 'train',
        'n_workers': 4,
        'batch_size': 16,
        'extra_samples': 4,
    }
    from data import create_dataset, create_dataloader, util

    ds = create_dataset(params)
    dl = create_dataloader(ds, params)
    i = 0
    for b in tqdm(dl):
        for b_ in range(16):
            pass
            #torchaudio.save(f'{i}_clip1_{b_}.wav', b['clip1'][b_], ds.sampling_rate)
            #torchaudio.save(f'{i}_clip2_{b_}.wav', b['clip2'][b_], ds.sampling_rate)
            #i += 1