CMakeLists.txt
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)set(CMAKE_EXPORT_COMPILE_COMMANDS ON)include(CheckLanguage)include(cmake/modules/set_ifndef.cmake)include(cmake/modules/find_library_create_target.cmake)project(tensorrt_llm LANGUAGES CXX)
macro(find_library_create_target target_name lib libtype hints) message( STATUS "========================= Importing and creating target ${target_name} ==========================" ) message(STATUS "Looking for library ${lib}") if(CMAKE_BUILD_TYPE STREQUAL "Debug") find_library( ${lib}_LIB_PATH ${lib}${TRT_DEBUG_POSTFIX} HINTS ${hints} NO_DEFAULT_PATH) endif() find_library(${lib}_LIB_PATH ${lib} HINTS ${hints} NO_DEFAULT_PATH) find_library(${lib}_LIB_PATH ${lib}) message(STATUS "Library that was found ${${lib}_LIB_PATH}") add_library(${target_name} ${libtype} IMPORTED) set_target_properties( ${target_name} PROPERTIES IMPORTED_LOCATION ${${lib}_LIB_PATH} IMPORTED_IMPLIB ${${lib}_LIB_PATH}) message( STATUS "==========================================================================================" )endmacro()
function(set_ifndef variable value) if(NOT DEFINED ${variable}) set(${variable} ${value} PARENT_SCOPE) endif()endfunction()
# Build optionsoption(BUILD_PYT "Build in PyTorch TorchScript class mode" ON)option(BUILD_TESTS "Build Google tests" ON)option(BUILD_BENCHMARKS "Build benchmarks" ON)option(NVTX_DISABLE "Disable all NVTX features" ON)if(NVTX_DISABLE) add_compile_definitions("NVTX_DISABLE") message(STATUS "NVTX is disabled")else() message(STATUS "NVTX is enabled")endif()
# 开源版本不提供 batch manager 的源码。if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tensorrt_llm/batch_manager/CMakeLists.txt") set(BUILD_BATCH_MANAGER_DEFAULT ON)else() set(BUILD_BATCH_MANAGER_DEFAULT OFF)endif()option(BUILD_BATCH_MANAGER "Build batch manager from source" ${BUILD_BATCH_MANAGER_DEFAULT})if(BUILD_BATCH_MANAGER) message(STATUS "Building batch manager")else() message(STATUS "Importing batch manager")endif()
# Determine CUDA version before enabling the language extensioncheck_language(CUDA)if(CMAKE_CUDA_COMPILER) message(STATUS "CUDA compiler: ${CMAKE_CUDA_COMPILER}") if(NOT WIN32) # Linux execute_process( COMMAND "bash" "-c" "${CMAKE_CUDA_COMPILER} --version | egrep -o 'V[0-9]+.[0-9]+.[0-9]+' | cut -c2-" RESULT_VARIABLE _BASH_SUCCESS OUTPUT_VARIABLE CMAKE_CUDA_COMPILER_VERSION # cuda compiler version OUTPUT_STRIP_TRAILING_WHITESPACE) if(NOT _BASH_SUCCESS EQUAL 0) message(FATAL_ERROR "Failed to determine CUDA version") endif() else() # Windows ... endif()else() message(FATAL_ERROR "No CUDA compiler found")endif()
# 最低 cuda 版本 11.2set(CUDA_REQUIRED_VERSION "11.2")if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS CUDA_REQUIRED_VERSION) message( FATAL_ERROR "CUDA version ${CMAKE_CUDA_COMPILER_VERSION} must be at least ${CUDA_REQUIRED_VERSION}" )endif()
# Initialize CMAKE_CUDA_ARCHITECTURES before enabling CUDAif(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL "11.8") # >= 11.8 set(CMAKE_CUDA_ARCHITECTURES 70-real 80-real 86-real 89-real 90-real) else() set(CMAKE_CUDA_ARCHITECTURES 70-real 80-real 86-real) endif()endif()message(STATUS "GPU architectures: ${CMAKE_CUDA_ARCHITECTURES}")
# 超找 CUDA 相关库enable_language(CUDA)find_package(CUDAToolkit REQUIRED)find_library( CUDNN_LIB cudnn HINTS ${CUDNN_ROOT_DIR} ${CUDAToolkit_LIBRARY_DIR} PATH_SUFFIXES lib64 lib lib/x64)find_library( CUBLAS_LIB cublas HINTS ${CUDAToolkit_LIBRARY_DIR} PATH_SUFFIXES lib64 lib lib/stubs)find_library( CUBLASLT_LIB cublasLt HINTS ${CUDAToolkit_LIBRARY_DIR} PATH_SUFFIXES lib64 lib lib/stubs)find_library( CUDA_DRV_LIB cuda HINTS ${CUDAToolkit_LIBRARY_DIR} PATH_SUFFIXES stubs lib lib64 lib/stubs lib64/stubs)set(CMAKE_CUDA_RUNTIME_LIBRARY Static)find_library(RT_LIB rt)set_ifndef(ENABLE_MULTI_DEVICE 1)if(ENABLE_MULTI_DEVICE EQUAL 1) # NCCL dependencies set_ifndef(NCCL_LIB_DIR /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/) set_ifndef(NCCL_INCLUDE_DIR /usr/include/) find_library(NCCL_LIB nccl HINTS ${NCCL_LIB_DIR})endif()
# 新增 inc 路径get_filename_component(TRT_LLM_ROOT_DIR ${CMAKE_SOURCE_DIR} PATH)set(3RDPARTY_DIR ${TRT_LLM_ROOT_DIR}/3rdparty)include_directories( ${CUDA_INCLUDE_DIRS} ${CUDNN_ROOT_DIR}/include ${NCCL_INCLUDE_DIR} ${3RDPARTY_DIR}/cutlass/include ${3RDPARTY_DIR}/NVTX/include ${3RDPARTY_DIR}/json/include)
# TRT dependenciesset_ifndef(TRT_LIB_DIR ${CMAKE_BINARY_DIR})set_ifndef(TRT_INCLUDE_DIR /usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu)set(TRT_LIB nvinfer)find_library_create_target(${TRT_LIB} nvinfer SHARED ${TRT_LIB_DIR})find_library_create_target(nvuffparser nvparsers SHARED ${TRT_LIB_DIR})
if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11") # >= 11 支持 BF16 add_definitions("-DENABLE_BF16") message( STATUS "CUDAToolkit_VERSION ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR} is greater or equal than 11.0, enable -DENABLE_BF16 flag" )endif()if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11.8") # >= 11.8 支持 FP8 add_definitions("-DENABLE_FP8") message( STATUS "CUDAToolkit_VERSION ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR} is greater or equal than 11.8, enable -DENABLE_FP8 flag" )endif()
# MPI MPI isn't used until tensorrt_llm/CMakeLists.txt is invoked. However, if# it's not called before "CMAKE_CXX_FLAGS" is set, it breaks on Windows for some# reason, so we just call it here as a workaround.find_package(MPI REQUIRED)add_definitions("-DOMPI_SKIP_MPICXX")
# C++17set(CMAKE_CXX_STANDARD 17)set(CMAKE_CXX_STANDARD_REQUIRED ON)set(CMAKE_CXX_EXTENSIONS OFF)set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILD_SYSTEM=cmake_oss -DENABLE_MULTI_DEVICE=${ENABLE_MULTI_DEVICE}")# Disable deprecated declarations warningsif(NOT WIN32) set(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations ${CMAKE_CXX_FLAGS}")else() ...endif()set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda")set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr")
set(COMMON_HEADER_DIRS ${PROJECT_SOURCE_DIR} ${CUDAToolkit_INCLUDE_DIR})message(STATUS "COMMON_HEADER_DIRS: ${COMMON_HEADER_DIRS}")
if(BUILD_PYT) # Build TORCH_CUDA_ARCH_LIST set(TORCH_CUDA_ARCH_LIST "") foreach(CUDA_ARCH IN LISTS CMAKE_CUDA_ARCHITECTURES) if(CUDA_ARCH MATCHES "^([0-9])([0-9])(-real)*$") set(TORCH_ARCH "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}") elseif(CUDA_ARCH STREQUAL "native") set(TORCH_ARCH "Auto") else() message(FATAL_ERROR "${CUDA_ARCH} is not supported") endif() if(NOT CUDA_ARCH MATCHES "-real$" AND NOT CUDA_ARCH STREQUAL "native") string(APPEND TORCH_ARCH "+PTX") endif() list(APPEND TORCH_CUDA_ARCH_LIST ${TORCH_ARCH}) endforeach() message(STATUS "TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}") # ignore values passed from the environment if(DEFINED ENV{TORCH_CUDA_ARCH_LIST}) message( WARNING "Ignoring environment variable TORCH_CUDA_ARCH_LIST=$ENV{TORCH_CUDA_ARCH_LIST}" ) endif() unset(ENV{TORCH_CUDA_ARCH_LIST}) find_package(Python3 COMPONENTS Interpreter Development REQUIRED) message(STATUS "Found Python executable at ${Python3_EXECUTABLE}") message(STATUS "Found Python libraries at ${Python3_LIBRARY_DIRS}") link_directories("${Python3_LIBRARY_DIRS}") list(APPEND COMMON_HEADER_DIRS ${Python3_INCLUDE_DIRS}) execute_process( COMMAND ${Python3_EXECUTABLE} "-c" "from __future__ import print_function; import torch; print(torch.__version__,end='');" RESULT_VARIABLE _PYTHON_SUCCESS OUTPUT_VARIABLE TORCH_VERSION) if(TORCH_VERSION VERSION_LESS "1.5.0") message(FATAL_ERROR "PyTorch >= 1.5.0 is needed for TorchScript mode.") endif() execute_process( COMMAND ${Python3_EXECUTABLE} "-c" "from __future__ import print_function; import os; import torch;print(os.path.dirname(torch.__file__),end='');" RESULT_VARIABLE _PYTHON_SUCCESS OUTPUT_VARIABLE TORCH_DIR) if(NOT _PYTHON_SUCCESS MATCHES 0) message(FATAL_ERROR "Torch config Error.") endif() list(APPEND CMAKE_PREFIX_PATH ${TORCH_DIR}) find_package(Torch REQUIRED) message(STATUS "TORCH_CXX_FLAGS: ${TORCH_CXX_FLAGS}") add_compile_options(${TORCH_CXX_FLAGS}) add_compile_definitions(TORCH_CUDA=1)endif()
file(STRINGS "${TRT_INCLUDE_DIR}/NvInferVersion.h" VERSION_STRINGS REGEX "#define NV_TENSORRT_.*")foreach(TYPE MAJOR MINOR PATCH BUILD) string(REGEX MATCH "NV_TENSORRT_${TYPE} [0-9]" TRT_TYPE_STRING ${VERSION_STRINGS}) string(REGEX MATCH "[0-9]" TRT_${TYPE} ${TRT_TYPE_STRING})endforeach(TYPE)foreach(TYPE MAJOR MINOR PATCH) string(REGEX MATCH "NV_TENSORRT_SONAME_${TYPE} [0-9]" TRT_TYPE_STRING ${VERSION_STRINGS}) string(REGEX MATCH "[0-9]" TRT_SO_${TYPE} ${TRT_TYPE_STRING})endforeach(TYPE)set(TRT_VERSION "${TRT_MAJOR}.${TRT_MINOR}.${TRT_PATCH}" CACHE STRING "TensorRT project version")set(TRT_SOVERSION "${TRT_SO_MAJOR}" CACHE STRING "TensorRT library so version")message( STATUS "Building for TensorRT version: ${TRT_VERSION}, library version: ${TRT_SOVERSION}")
list(APPEND COMMON_HEADER_DIRS)include_directories(${COMMON_HEADER_DIRS})include_directories(SYSTEM ${TORCH_INCLUDE_DIRS} ${TRT_INCLUDE_DIR})
add_subdirectory(tensorrt_llm)
tensorrt_llm
set(TARGET_NAME tensorrt_llm)set(SHARED_TARGET ${TARGET_NAME})set(SHARED_TARGET ${SHARED_TARGET} PARENT_SCOPE)set(STATIC_TARGET ${TARGET_NAME}_static)set(STATIC_TARGET ${STATIC_TARGET} PARENT_SCOPE)set(API_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/include)
find_package(MPI REQUIRED)message(STATUS "Using MPI_CXX_INCLUDE_DIRS: ${MPI_CXX_INCLUDE_DIRS}")message(STATUS "Using MPI_CXX_LIBRARIES: ${MPI_CXX_LIBRARIES}")include_directories(${CMAKE_CURRENT_SOURCE_DIR}/cutlass_extensions/include ${API_INCLUDE_DIR} ${MPI_INCLUDE_PATH})
add_subdirectory(common)add_subdirectory(kernels)add_subdirectory(layers)add_subdirectory(runtime)
set(BATCH_MANAGER_TARGET tensorrt_llm_batch_manager_static)set(BATCH_MANAGER_TARGET_ARCH "unknown")message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")if(NOT WIN32) # Linux if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") set(BATCH_MANAGER_TARGET_ARCH "x86_64-linux-gnu") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") set(BATCH_MANAGER_TARGET_ARCH "aarch64-linux-gnu") else() message( FATAL_ERROR "The system processor type is unsupported: ${CMAKE_SYSTEM_PROCESSOR}") endif()else() # Windows ...endif()
if(BUILD_BATCH_MANAGER) add_subdirectory(batch_manager)else() add_library(${BATCH_MANAGER_TARGET} STATIC IMPORTED) if(NOT WIN32) # Linux execute_process( COMMAND ${Python3_EXECUTABLE} "-c" "import torch; print(torch.compiled_with_cxx11_abi(),end='');" RESULT_VARIABLE _PYTHON_SUCCESS OUTPUT_VARIABLE USE_CXX11_ABI) message(STATUS "USE_CXX11_ABI: ${USE_CXX11_ABI}") if(USE_CXX11_ABI) set_property( TARGET ${BATCH_MANAGER_TARGET} PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/batch_manager/${BATCH_MANAGER_TARGET_ARCH}/libtensorrt_llm_batch_manager_static.a" ) else() set_property( TARGET ${BATCH_MANAGER_TARGET} PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/batch_manager/${BATCH_MANAGER_TARGET_ARCH}/libtensorrt_llm_batch_manager_static.pre_cxx11.a" ) endif() else() # Windows ... endif()endif()
set(TRTLLM_LINK_LIBS ${CUBLAS_LIB} ${CUBLASLT_LIB} ${CUDNN_LIB} ${CMAKE_DL_LIBS} ${MPI_CXX_LIBRARIES} ${NCCL_LIB} ${TRT_LIB} common_src kernels_src layers_src runtime_src ${BATCH_MANAGER_TARGET})
# ################################# SHARED LIBRARY# ##############################################################################set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)add_library(${SHARED_TARGET} SHARED)set_target_properties( ${SHARED_TARGET} PROPERTIES CXX_STANDARD "17" CXX_STANDARD_REQUIRED "YES" CXX_EXTENSIONS "NO")if(NOT MSVC) # Unix-like compilers set(ALLOW_UNDEFINED_FLAG "-Wl, --no-undefined")else() # MSVC set(UNDEFINED_FLAG "")endif()target_link_libraries(${SHARED_TARGET} PUBLIC ${TRTLLM_LINK_LIBS} ${UNDEFINED_FLAG})
# ################################# STATIC LIBRARY# ##############################################################################add_library(${STATIC_TARGET} STATIC)set_target_properties( ${STATIC_TARGET} PROPERTIES CXX_STANDARD "17" CXX_STANDARD_REQUIRED "YES" CXX_EXTENSIONS "NO" POSITION_INDEPENDENT_CODE ON)target_link_libraries(${STATIC_TARGET} PUBLIC ${TRTLLM_LINK_LIBS})# Cyclic dependency of batch manager on TRT-LLMtarget_link_libraries(${BATCH_MANAGER_TARGET} INTERFACE ${STATIC_TARGET})
if(BUILD_PYT) add_subdirectory(thop)endif()add_subdirectory(plugins)
参考文献
- • https://github.com/NVIDIA/TensorRT-LLM/blob/release/0.5.0/cpp/CMakeLists.txt
夜雨聆风