mrq/bitsandbytes-rocm
1
0
Fork 1
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

beta2 is actually accessible in kOptimizerStatic8bit1StateBlockwise

Browse Source
This commit is contained in:
Phil Wang 2023-03-10 12:50:14 -08:00
parent 19b9ef34b9
commit abbe65adfc
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
csrc/kernels.cu
View File

@ -1712,8 +1712,7 @@ kOptimizerStatic8bit1StateBlockwise(T* p, T* __restrict__ const g, unsigned char
s1_vals[j] = (s1_vals[j]*beta1) + g_val;
break;
case LION:
// using eps as beta2
s1_vals[j] = s1_vals[j]*eps + ((1.0f-eps)*g_val);
s1_vals[j] = s1_vals[j]*beta2 + ((1.0f-beta2)*g_val);
break;
case RMSPROP:
s1_vals[j] = s1_vals[j]*beta1 + ((1.0f-beta1)*(g_val*g_val));