From c5582724d5213e558f2a8ceef25e09c115cb8ef8 Mon Sep 17 00:00:00 2001
From: Phil Wang <lucidrains@gmail.com>
Date: Thu, 9 Mar 2023 14:05:45 -0800
Subject: [PATCH] missed adagrad

---
 csrc/kernels.cu | 1 +
 1 file changed, 1 insertion(+)

diff --git a/csrc/kernels.cu b/csrc/kernels.cu
index 8fa5a34..0c5493b 100644
--- a/csrc/kernels.cu
+++ b/csrc/kernels.cu
@@ -1691,6 +1691,7 @@ kOptimizerStatic8bit1StateBlockwise(T* p, T* __restrict__ const g, unsigned char
 							if(weight_decay > 0.0f) {
 								switch(OPTIMIZER) {
                   case MOMENTUM:
+                  case ADAGRAD:
                   case RMSPROP:
                     g_val += ((float)p_vals[j])*weight_decay;
                     break;