fix comment
This commit is contained in:
parent
978ba2db57
commit
2a6828e6fb
|
@ -1708,8 +1708,7 @@ kOptimizerStatic8bit1StateBlockwise(T* p, T* __restrict__ const g, unsigned char
|
|||
s1_vals[j] = (s1_vals[j]*beta1) + g_val;
|
||||
break;
|
||||
case LION:
|
||||
// here, using gvals[j] to store the gradient smoothed by beta1
|
||||
// then update the momentum state1, to make sure the order is correct
|
||||
// here, using gvals[j] to store the gradient smoothed by beta1 for the following parameter update, before the momentum is updated by beta2
|
||||
g_vals[j] = lr*sgn(((float)s1_vals[j])*beta1 + ((1.0f-beta1)*g_val));
|
||||
s1_vals[j] = s1_vals[j]*beta2 + ((1.0f-beta2)*g_val);
|
||||
break;
|
||||
|
|
Loading…
Reference in New Issue
Block a user