From 2a6828e6fbc4bd81d44ca67de45563b3b2876c14 Mon Sep 17 00:00:00 2001
From: Phil Wang <lucidrains@gmail.com>
Date: Wed, 22 Mar 2023 09:56:50 -0700
Subject: [PATCH] fix comment

---
 csrc/kernels.cu | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/csrc/kernels.cu b/csrc/kernels.cu
index 1778c98..e0df802 100644
--- a/csrc/kernels.cu
+++ b/csrc/kernels.cu
@@ -1708,8 +1708,7 @@ kOptimizerStatic8bit1StateBlockwise(T* p, T* __restrict__ const g, unsigned char
 											s1_vals[j] = (s1_vals[j]*beta1) + g_val;
 										break;
 									case LION:
-										// here, using gvals[j] to store the gradient smoothed by beta1
-										// then update the momentum state1, to make sure the order is correct
+										// here, using gvals[j] to store the gradient smoothed by beta1 for the following parameter update, before the momentum is updated by beta2
 										g_vals[j] = lr*sgn(((float)s1_vals[j])*beta1 + ((1.0f-beta1)*g_val));
 										s1_vals[j] = s1_vals[j]*beta2 + ((1.0f-beta2)*g_val);
 										break;