deepseed No such file or directory: '/usr/local/cuda/bin/nvcc' #32
Description
Traceback (most recent call last):
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/bin/accelerate", line 8, in
sys.exit(main())
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/accelerate/commands/accelerate_cli.py", line 48, in main
args.func(args)
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/accelerate/commands/launch.py", line 1177, in launch_command
deepspeed_launcher(args)
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/accelerate/commands/launch.py", line 826, in deepspeed_launcher
from deepspeed.launcher.runner import DEEPSPEED_ENVIRONMENT_NAME
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/deepspeed/init.py", line 25, in
from . import ops
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/deepspeed/ops/init.py", line 15, in
from ..git_version_info import compatible_ops as compatible_ops
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/deepspeed/git_version_info.py", line 29, in
op_compatible = builder.is_compatible()
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/deepspeed/ops/op_builder/fp_quantizer.py", line 35, in is_compatible
sys_cuda_major, _ = installed_cuda_version()
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 53, in installed_cuda_version
output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"], universal_newlines=True)
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/subprocess.py", line 421, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/subprocess.py", line 503, in run
with Popen(*popenargs, **kwargs) as process:
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/subprocess.py", line 971, in init
self._execute_child(args, executable, preexec_fn, close_fds,
File "/lab/tangb_lab/12232387/anaconda3/envs/faithdiff/lib/python3.10/subprocess.py", line 1863, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: '/usr/local/cuda/bin/nvcc'