From 97b2567ada4ec411d47db05775e670be9b40917f Mon Sep 17 00:00:00 2001 From: rapsealk Date: Tue, 25 Apr 2023 17:00:02 +0900 Subject: [PATCH 1/5] fix: Replace libcudart with pytorch api --- bitsandbytes/cuda_setup/main.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/bitsandbytes/cuda_setup/main.py b/bitsandbytes/cuda_setup/main.py index 3c4e7f3..f629b63 100644 --- a/bitsandbytes/cuda_setup/main.py +++ b/bitsandbytes/cuda_setup/main.py @@ -326,31 +326,10 @@ def get_cuda_lib_handle(): def get_compute_capabilities(cuda): - """ - 1. find libcuda.so library (GPU driver) (/usr/lib) - init_device -> init variables -> call function by reference - 2. call extern C function to determine CC - (https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html) - 3. Check for CUDA errors - https://stackoverflow.com/questions/14038589/what-is-the-canonical-way-to-check-for-errors-using-the-cuda-runtime-api - # bits taken from https://gist.github.com/f0k/63a664160d016a491b2cbea15913d549 - """ - - nGpus = ct.c_int() - cc_major = ct.c_int() - cc_minor = ct.c_int() - - device = ct.c_int() - - check_cuda_result(cuda, cuda.cuDeviceGetCount(ct.byref(nGpus))) ccs = [] - for i in range(nGpus.value): - check_cuda_result(cuda, cuda.cuDeviceGet(ct.byref(device), i)) - ref_major = ct.byref(cc_major) - ref_minor = ct.byref(cc_minor) - # 2. call extern C function to determine CC - check_cuda_result(cuda, cuda.cuDeviceComputeCapability(ref_major, ref_minor, device)) - ccs.append(f"{cc_major.value}.{cc_minor.value}") + # for i in range(torch.cuda.device_count()): + # device = torch.cuda.device(i) + ccs.append(torch.version.cuda) return ccs From eb54c55b61c16419dfdb37a3314aa519e460e6de Mon Sep 17 00:00:00 2001 From: rapsealk Date: Tue, 25 Apr 2023 17:08:22 +0900 Subject: [PATCH 2/5] fix: Get CUDA compiled version through pytorch --- bitsandbytes/cuda_setup/main.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/bitsandbytes/cuda_setup/main.py b/bitsandbytes/cuda_setup/main.py index f629b63..10e91a9 100644 --- a/bitsandbytes/cuda_setup/main.py +++ b/bitsandbytes/cuda_setup/main.py @@ -290,19 +290,7 @@ def check_cuda_result(cuda, result_val): def get_cuda_version(cuda, cudart_path): if cuda is None: return None - try: - cudart = ct.CDLL(cudart_path) - except OSError: - CUDASetup.get_instance().add_log_entry(f'ERROR: libcudart.so could not be read from path: {cudart_path}!') - return None - - version = ct.c_int() - try: - check_cuda_result(cuda, cudart.cudaRuntimeGetVersion(ct.byref(version))) - except AttributeError as e: - CUDASetup.get_instance().add_log_entry(f'ERROR: {str(e)}') - CUDASetup.get_instance().add_log_entry(f'CUDA SETUP: libcudart.so path is {cudart_path}') - CUDASetup.get_instance().add_log_entry(f'CUDA SETUP: Is seems that your cuda installation is not in your path. See https://github.com/TimDettmers/bitsandbytes/issues/85 for more information.') + version = torch._C._cuda_getCompiledVersion() version = int(version.value) major = version//1000 minor = (version-(major*1000))//10 From 9836b0b90fbf67a0bb479cbea288a0f9c77810ae Mon Sep 17 00:00:00 2001 From: rapsealk Date: Tue, 25 Apr 2023 17:12:27 +0900 Subject: [PATCH 3/5] fix: Use raw int --- bitsandbytes/cuda_setup/main.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/bitsandbytes/cuda_setup/main.py b/bitsandbytes/cuda_setup/main.py index 10e91a9..49edcbb 100644 --- a/bitsandbytes/cuda_setup/main.py +++ b/bitsandbytes/cuda_setup/main.py @@ -275,15 +275,15 @@ def determine_cuda_runtime_lib_path() -> Union[Path, None]: return next(iter(cuda_runtime_libs)) if cuda_runtime_libs else None -def check_cuda_result(cuda, result_val): - # 3. Check for CUDA errors - if result_val != 0: - error_str = ct.c_char_p() - cuda.cuGetErrorString(result_val, ct.byref(error_str)) - if error_str.value is not None: - CUDASetup.get_instance().add_log_entry(f"CUDA exception! Error code: {error_str.value.decode()}") - else: - CUDASetup.get_instance().add_log_entry(f"Unknown CUDA exception! Please check your CUDA install. It might also be that your GPU is too old.") +# def check_cuda_result(cuda, result_val): +# # 3. Check for CUDA errors +# if result_val != 0: +# error_str = ct.c_char_p() +# cuda.cuGetErrorString(result_val, ct.byref(error_str)) +# if error_str.value is not None: +# CUDASetup.get_instance().add_log_entry(f"CUDA exception! Error code: {error_str.value.decode()}") +# else: +# CUDASetup.get_instance().add_log_entry(f"Unknown CUDA exception! Please check your CUDA install. It might also be that your GPU is too old.") # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART____VERSION.html#group__CUDART____VERSION @@ -291,7 +291,6 @@ def get_cuda_version(cuda, cudart_path): if cuda is None: return None version = torch._C._cuda_getCompiledVersion() - version = int(version.value) major = version//1000 minor = (version-(major*1000))//10 @@ -308,7 +307,7 @@ def get_cuda_lib_handle(): except OSError: CUDASetup.get_instance().add_log_entry('CUDA SETUP: WARNING! libcuda.so not found! Do you have a CUDA driver installed? If you are on a cluster, make sure you are on a CUDA machine!') return None - check_cuda_result(cuda, cuda.cuInit(0)) + # check_cuda_result(cuda, cuda.cuInit(0)) return cuda From f5110265ff0074b813191352e388657642afe6a3 Mon Sep 17 00:00:00 2001 From: Jeongseok Kang Date: Wed, 26 Apr 2023 11:54:17 +0900 Subject: [PATCH 4/5] fix: Remove unused code --- bitsandbytes/cuda_setup/main.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/bitsandbytes/cuda_setup/main.py b/bitsandbytes/cuda_setup/main.py index 49edcbb..94f9b9c 100644 --- a/bitsandbytes/cuda_setup/main.py +++ b/bitsandbytes/cuda_setup/main.py @@ -275,17 +275,6 @@ def determine_cuda_runtime_lib_path() -> Union[Path, None]: return next(iter(cuda_runtime_libs)) if cuda_runtime_libs else None -# def check_cuda_result(cuda, result_val): -# # 3. Check for CUDA errors -# if result_val != 0: -# error_str = ct.c_char_p() -# cuda.cuGetErrorString(result_val, ct.byref(error_str)) -# if error_str.value is not None: -# CUDASetup.get_instance().add_log_entry(f"CUDA exception! Error code: {error_str.value.decode()}") -# else: -# CUDASetup.get_instance().add_log_entry(f"Unknown CUDA exception! Please check your CUDA install. It might also be that your GPU is too old.") - - # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART____VERSION.html#group__CUDART____VERSION def get_cuda_version(cuda, cudart_path): if cuda is None: return None @@ -307,15 +296,12 @@ def get_cuda_lib_handle(): except OSError: CUDASetup.get_instance().add_log_entry('CUDA SETUP: WARNING! libcuda.so not found! Do you have a CUDA driver installed? If you are on a cluster, make sure you are on a CUDA machine!') return None - # check_cuda_result(cuda, cuda.cuInit(0)) return cuda def get_compute_capabilities(cuda): ccs = [] - # for i in range(torch.cuda.device_count()): - # device = torch.cuda.device(i) ccs.append(torch.version.cuda) return ccs From 2b4cc256f67be3a4a7d49f597d27920c6093ad9f Mon Sep 17 00:00:00 2001 From: rapsealk Date: Fri, 28 Apr 2023 11:18:54 +0900 Subject: [PATCH 5/5] fix: Get device's compute capability --- bitsandbytes/cuda_setup/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bitsandbytes/cuda_setup/main.py b/bitsandbytes/cuda_setup/main.py index 94f9b9c..ac0fb02 100644 --- a/bitsandbytes/cuda_setup/main.py +++ b/bitsandbytes/cuda_setup/main.py @@ -279,9 +279,7 @@ def determine_cuda_runtime_lib_path() -> Union[Path, None]: def get_cuda_version(cuda, cudart_path): if cuda is None: return None - version = torch._C._cuda_getCompiledVersion() - major = version//1000 - minor = (version-(major*1000))//10 + major, minor = map(int, torch.version.cuda.split(".")) if major < 11: CUDASetup.get_instance().add_log_entry('CUDA SETUP: CUDA version lower than 11 are currently not supported for LLM.int8(). You will be only to use 8-bit optimizers and quantization routines!!') @@ -302,7 +300,9 @@ def get_cuda_lib_handle(): def get_compute_capabilities(cuda): ccs = [] - ccs.append(torch.version.cuda) + for i in range(torch.cuda.device_count()): + cc_major, cc_minor = torch.cuda.get_device_capability(torch.cuda.device(i)) + ccs.append(f"{cc_major}.{cc_minor}") return ccs