dataset mod

2021-12-23 15:21:30 -07:00 · 2021-12-23 15:21:30 -07:00 · 9677f7084c
commit 9677f7084c
parent 8b19c37409
1 changed files with 1 additions and 1 deletions
--- a/codes/data/text/hf_datasets_wrapper.py
+++ b/codes/data/text/hf_datasets_wrapper.py
@ -10,7 +10,7 @@ class HfDataset(Dataset):
        self.hfd = []
        for corpus in corpi:
            dataset_name, config = corpus
-            if config == '':
+            if config == '' or config == 'None':
                config = None
            self.hfd.append(datasets.load_dataset(dataset_name, config, cache_dir=cache_path)[dataset_spec_key])
        self.key_maps = key_maps