TensorRT-LLM 0.5.0 源码之十八

cpp_library

#include <dlfcn.h>

template <typename tSymbolSignature>
tSymbolSignature getTrtLLMFunction(std::string libFileSoName, std::string symbol)
{
#if !defined(_WIN32)
    std::cout << "Trying to load " << libFileSoName << " ..." << std::endl;

    // 1. Defining a handle to the library
    void* handle = dlopen(libFileSoName.c_str(), RTLD_LAZY | RTLD_GLOBAL);

    // 2. Check for errors
    const char* dl_error1 = dlerror();
    if (!handle)
    {
        throw std::runtime_error("Cannot open library: " + std::string(dl_error1));
    }

    // 3. Load actual queried `symbol`
    std::cout << "Loading symbol `" << symbol << "` ..." << std::endl;

    tSymbolSignature symbolFctn = nullptr;
    *(void**) (&symbolFctn) = dlsym(handle, symbol.c_str());

    // 4. Check for errors
    const char* dl_error2 = dlerror();
    if (dl_error2)
    {
        dlclose(handle);
        throw std::runtime_error("Cannot load symbol '" + symbol + "': " + std::string(dl_error2));
    }

    return symbolFctn;
#else  // on windows
    throw std::runtime_error(
        "`tSymbolSignature getTrtLLMFunction(std::string, std::string)` is not implemented on Windows.");
    return nullptr;
#endif // !defined(_WIN32)
}

#include "tensorrt_llm_libutils.h"

int main(int argc, char* argv[])
{
    class TRTLogger : public nvinfer1::ILogger
    {
    public:
void log(nvinfer1::ILogger::Severity severity, const char* msg) noexcept override
{
            if (severity <= nvinfer1::ILogger::Severity::kERROR)
                std::cerr << "[TensorRT-LLM ERR]: " << msg << std::endl;
            else if (severity == nvinfer1::ILogger::Severity::kWARNING)
                std::cerr << "[TensorRT-LLM WARNING]: " << msg << std::endl;
            else
                std::cout << "[TensorRT-LLM LOG]: " << msg << std::endl;
        }
    };

    TRTLogger* trtLogger = new TRTLogger();

    std::string libname = "libtensorrt_llm_plugin.so";

    /* =============== initLibNvInferPlugins =============== */

typedef bool (*initLibNvInferPlugins_sig)(void*, const void*);

    auto initLibNvInferPlugins = getTrtLLMFunction<initLibNvInferPlugins_sig>(
        /*libFileSoName=*/libname,
        /*symbol=*/"initLibNvInferPlugins");

    std::cout << std::endl;

    std::string libNamespace = "tensorrt_llm";
    const char* libNamespace_cstr = libNamespace.data();

    bool status1 = initLibNvInferPlugins(trtLogger, libNamespace_cstr);
    std::cout << "Success Status: " << status1 << std::endl << std::endl;

    bool status2 = initLibNvInferPlugins(trtLogger, libNamespace_cstr);
    std::cout << "Success Status: " << status2 << std::endl;

    /* =============== getInferLibVersion =============== */

    std::cout << std::endl;
    std::cout << "--------------------------------------------------------------------" << std::endl;

typedef int32_t (*getInferLibVersion_sig)();

    auto getInferLibVersion = getTrtLLMFunction<getInferLibVersion_sig>(
        /*libFileSoName=*/libname,
        /*symbol=*/"getInferLibVersion");

    std::cout << std::endl;

    int32_t version = getInferLibVersion();
    std::cout << "Version: " << version << std::endl;

    return 0;
}

build.sh

#!/usr/bin/env bash

BUILD_DIR="build"
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR}

pushd ${BUILD_DIR}

cmake \
    -DCMAKE_BUILD_TYPE=Release \
    ..

make -j"$(grep -c ^processor /proc/cpuinfo)"

export LD_LIBRARY_PATH="${SCRIPT_DIR}:${LD_LIBRARY_PATH}"

# Test Lib
echo
echo "--------------------------------------------------------------------"
./trt_llm_plugins_cpp_load_example
echo "--------------------------------------------------------------------"
echo

popd