This commit is contained in:
James Betker 2022-03-05 23:12:34 -07:00
parent 64d764ccd7
commit dcf98df0c2

View File

@ -27,6 +27,7 @@ def load_tsv(filename):
if bad_lines > 10:
print(f'{filename} contains 10+ bad entries. Failing. Sample last entry: {line}')
raise ValueError
continue
filepaths_and_text.append([os.path.join(base, f'{components[1]}'), components[0]])
return filepaths_and_text
@ -52,6 +53,7 @@ def load_tsv_aligned_codes(filename):
if bad_lines > 10:
print(f'{filename} contains 10+ bad entries. Failing. Sample last entry: {line}')
raise ValueError
continue
filepaths_and_text.append([os.path.join(base, f'{components[1]}'), components[0], convert_string_list_to_tensor(components[2])])
return filepaths_and_text