Add recover_tensorboard_log

Generates a tb_logger from raw console output. Useful for colab sessions
that crash.
This commit is contained in:
James Betker 2020-06-27 08:26:57 -06:00
parent c8a670842e
commit db08dedfe2

View File

@ -0,0 +1,21 @@
from torch.utils.tensorboard import SummaryWriter
if __name__ == "__main__":
writer = SummaryWriter("../experiments/train_div2k_feat_rg2_more_stuff")
f = open("../experiments/train_div2k_feat_rg2_more_stuff/console_output")
console = f.readlines()
search_terms = [
("iter", ", iter: ", ", lr:"),
("l_g_total", " l_g_total: ", " switch_temperature:")
]
iter = 0
for line in console:
if " - INFO: [epoch:" not in line:
continue
for name, start, end in search_terms:
val = line[line.find(start)+len(start):line.find(end)].replace(",", "")
if name == "iter":
iter = int(val)
else:
writer.add_scalar(name, float(val), iter)
writer.close()