From 9677f7084c4538225b59789e4c4a44758509427a Mon Sep 17 00:00:00 2001 From: James Betker Date: Thu, 23 Dec 2021 15:21:30 -0700 Subject: [PATCH] dataset mod --- codes/data/text/hf_datasets_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codes/data/text/hf_datasets_wrapper.py b/codes/data/text/hf_datasets_wrapper.py index 03e7c165..dfed9d7b 100644 --- a/codes/data/text/hf_datasets_wrapper.py +++ b/codes/data/text/hf_datasets_wrapper.py @@ -10,7 +10,7 @@ class HfDataset(Dataset): self.hfd = [] for corpus in corpi: dataset_name, config = corpus - if config == '': + if config == '' or config == 'None': config = None self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key]) self.key_maps = key_maps