dataset mod
This commit is contained in:
parent
8b19c37409
commit
9677f7084c
|
@ -10,7 +10,7 @@ class HfDataset(Dataset):
|
||||||
self.hfd = []
|
self.hfd = []
|
||||||
for corpus in corpi:
|
for corpus in corpi:
|
||||||
dataset_name, config = corpus
|
dataset_name, config = corpus
|
||||||
if config == '':
|
if config == '' or config == 'None':
|
||||||
config = None
|
config = None
|
||||||
self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key])
|
self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key])
|
||||||
self.key_maps = key_maps
|
self.key_maps = key_maps
|
||||||
|
|
Loading…
Reference in New Issue
Block a user