mrq/DL-Art-School
1
3
Fork 5
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix issue where eval was not being used by all ddp processes

Browse Source
This commit is contained in:
James Betker 2021-06-14 09:50:04 -06:00
parent 60079a1572
commit 54bff35171
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
codes/train.py
View File

@ -257,10 +257,11 @@ class Trainer:
self.tb_logger.add_scalar('val_psnr', avg_psnr, self.current_step)
self.tb_logger.add_scalar('val_fea', avg_fea_loss, self.current_step)
if len(self.evaluators) != 0 and self.current_step % opt['train']['val_freq'] == 0 and self.rank <= 0:
if len(self.evaluators) != 0 and self.current_step % opt['train']['val_freq'] == 0:
eval_dict = {}
for eval in self.evaluators:
eval_dict.update(eval.perform_eval())
if eval.uses_all_ddp() or self.rank <= 0:
eval_dict.update(eval.perform_eval())
if self.rank <= 0:
print("Evaluator results: ", eval_dict)
for ek, ev in eval_dict.items():

1
codes/trainer/eval/evaluator.py
View File

@ -11,6 +11,7 @@ class Evaluator:
self.model = model.module if hasattr(model, 'module') else model
self.opt = opt_eval
self.env = env
self.uses_all_ddp = opt_get(opt_eval, ['uses_all_ddp'], True)
def perform_eval(self):
return {}