eval fix for nar-len

This commit is contained in:
mrq 2024-11-06 23:14:16 -06:00
parent 105ed51159
commit d606a693ff

View File

@ -145,8 +145,11 @@ def run_eval(engines, eval_name, dl, args=None):
if engine.hyper_config.experimental.hf: if engine.hyper_config.experimental.hf:
resps_list = engine( **base_kwargs ) resps_list = engine( **base_kwargs )
elif "len" in engine.hyper_config.capabilities: elif "len" in engine.hyper_config.capabilities:
len_list = engine( **base_kwargs, max_steps=10 ) # don't need more than that kwargs = base_kwargs | cfg.evaluation.ar_kwargs
len_list = [ min( l, cfg.evaluation.steps ) for l in len_list ] max_steps = kwargs.pop("max_steps", 500)
kwargs["max_steps"] = 10
len_list = engine( **kwargs ) # don't need more than that
len_list = [ min( l, max_steps ) for l in len_list ]
kwargs = base_kwargs | cfg.evaluation.nar_kwargs kwargs = base_kwargs | cfg.evaluation.nar_kwargs
resps_list = engine( **kwargs, len_list=len_list ) resps_list = engine( **kwargs, len_list=len_list )