mrq/DL-Art-School
1
3
Fork 5
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dataset mod

Browse Source
This commit is contained in:
James Betker 2021-12-23 15:21:30 -07:00
parent 8b19c37409
commit 9677f7084c
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
codes/data/text/hf_datasets_wrapper.py
View File

@ -10,7 +10,7 @@ class HfDataset(Dataset):
self.hfd = [] self.hfd = []
for corpus in corpi: for corpus in corpi:
dataset_name, config = corpus dataset_name, config = corpus
if config == '': if config == '' or config == 'None':
config = None config = None
self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key]) self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key])
self.key_maps = key_maps self.key_maps = key_maps