From 9b3c3b1227958837a647edfda046f4acca28c781 Mon Sep 17 00:00:00 2001 From: James Betker Date: Sun, 7 Nov 2021 20:45:57 -0700 Subject: [PATCH] use sets instead of list ops --- codes/data/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codes/data/util.py b/codes/data/util.py index c948fa4b..5e73e160 100644 --- a/codes/data/util.py +++ b/codes/data/util.py @@ -591,7 +591,9 @@ def load_paths_from_cache(paths, cache_path, exclusion_list=[]): if exclusion_list is not None and len(exclusion_list) > 0: print(f"Removing exclusion lists..") before = len(output) - output = filter(lambda p: p not in exclusion_list, output) + master_set = set(output) + exclusion_set = set(exclusion_list) + output = list(master_set - exclusion_set) print(f"Excluded {before-len(output)} files.") print("Done.") torch.save(output, cache_path)