diff --git a/codes/data/text/hf_datasets_wrapper.py b/codes/data/text/hf_datasets_wrapper.py index 03e7c165..dfed9d7b 100644 --- a/codes/data/text/hf_datasets_wrapper.py +++ b/codes/data/text/hf_datasets_wrapper.py @@ -10,7 +10,7 @@ class HfDataset(Dataset): self.hfd = [] for corpus in corpi: dataset_name, config = corpus - if config == '': + if config == '' or config == 'None': config = None self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key]) self.key_maps = key_maps