eval fix for nar-len
This commit is contained in:
parent
105ed51159
commit
d606a693ff
|
@ -145,8 +145,11 @@ def run_eval(engines, eval_name, dl, args=None):
|
||||||
if engine.hyper_config.experimental.hf:
|
if engine.hyper_config.experimental.hf:
|
||||||
resps_list = engine( **base_kwargs )
|
resps_list = engine( **base_kwargs )
|
||||||
elif "len" in engine.hyper_config.capabilities:
|
elif "len" in engine.hyper_config.capabilities:
|
||||||
len_list = engine( **base_kwargs, max_steps=10 ) # don't need more than that
|
kwargs = base_kwargs | cfg.evaluation.ar_kwargs
|
||||||
len_list = [ min( l, cfg.evaluation.steps ) for l in len_list ]
|
max_steps = kwargs.pop("max_steps", 500)
|
||||||
|
kwargs["max_steps"] = 10
|
||||||
|
len_list = engine( **kwargs ) # don't need more than that
|
||||||
|
len_list = [ min( l, max_steps ) for l in len_list ]
|
||||||
|
|
||||||
kwargs = base_kwargs | cfg.evaluation.nar_kwargs
|
kwargs = base_kwargs | cfg.evaluation.nar_kwargs
|
||||||
resps_list = engine( **kwargs, len_list=len_list )
|
resps_list = engine( **kwargs, len_list=len_list )
|
||||||
|
|
Loading…
Reference in New Issue
Block a user