mirror of
https://github.com/DrHo1y/ezrknn-llm.git
synced 2026-11-23 02:22:49 +07:00
Merged v1.1.1
This commit is contained in:
@@ -1,21 +1,26 @@
|
||||
cmake_minimum_required(VERSION 3.10)
|
||||
project(llm_demo)
|
||||
project(rkllm_demo)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
set(SOURCE_FILES src/main.cpp)
|
||||
set(SOURCE_FILES_1 src/llm_demo.cpp)
|
||||
add_executable(llm_demo ${SOURCE_FILES_1})
|
||||
|
||||
add_executable(${PROJECT_NAME} ${SOURCE_FILES})
|
||||
set(SOURCE_FILES_2 src/multimodel_demo.cpp)
|
||||
add_executable(multimodel_demo ${SOURCE_FILES_2})
|
||||
|
||||
set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
|
||||
include_directories(${RKLLM_API_PATH}/include)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
|
||||
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
|
||||
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB} log)
|
||||
find_package(OpenMP REQUIRED)
|
||||
target_link_libraries(llm_demo ${RKLLM_RT_LIB} log OpenMP::OpenMP_CXX)
|
||||
target_link_libraries(multimodel_demo ${RKLLM_RT_LIB} log OpenMP::OpenMP_CXX)
|
||||
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
|
||||
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB})
|
||||
target_link_libraries(llm_demo ${RKLLM_RT_LIB})
|
||||
target_link_libraries(multimodel_demo ${RKLLM_RT_LIB})
|
||||
endif()
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
|
||||
|
||||
```bash
|
||||
cp build/build_linux_aarch64_Release/llm_demo /usr/bin
|
||||
cp ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/lib
|
||||
cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/lib
|
||||
```
|
||||
|
||||
To exit the LLM enter either `exit` or `quit`.
|
||||
|
||||
194
rkllm-runtime/examples/rkllm_api_demo/src/llm_demo.cpp
Normal file
194
rkllm-runtime/examples/rkllm_api_demo/src/llm_demo.cpp
Normal file
File diff suppressed because one or more lines are too long
191
rkllm-runtime/examples/rkllm_api_demo/src/multimodel_demo.cpp
Normal file
191
rkllm-runtime/examples/rkllm_api_demo/src/multimodel_demo.cpp
Normal file
@@ -0,0 +1,191 @@
|
||||
// Copyright (c) 2024 by Rockchip Electronics Co., Ltd. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <string>
|
||||
#include "rkllm.h"
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <csignal>
|
||||
#include <vector>
|
||||
|
||||
#define PROMPT_TEXT_PREFIX "<用户>"
|
||||
#define PROMPT_TEXT_POSTFIX "<AI>"
|
||||
|
||||
|
||||
using namespace std;
|
||||
LLMHandle llmHandle = nullptr;
|
||||
|
||||
void exit_handler(int signal)
|
||||
{
|
||||
if (llmHandle != nullptr)
|
||||
{
|
||||
{
|
||||
cout << "程序即将退出" << endl;
|
||||
LLMHandle _tmp = llmHandle;
|
||||
llmHandle = nullptr;
|
||||
rkllm_destroy(_tmp);
|
||||
}
|
||||
}
|
||||
exit(signal);
|
||||
}
|
||||
|
||||
void callback(RKLLMResult *result, void *userdata, LLMCallState state)
|
||||
{
|
||||
if (state == RKLLM_RUN_FINISH)
|
||||
{
|
||||
printf("\n");
|
||||
} else if (state == RKLLM_RUN_ERROR) {
|
||||
printf("\\run error\n");
|
||||
} else if (state == RKLLM_RUN_GET_LAST_HIDDEN_LAYER) {
|
||||
/* ================================================================================================================
|
||||
若使用GET_LAST_HIDDEN_LAYER功能,callback接口会回传内存指针:last_hidden_layer,token数量:num_tokens与隐藏层大小:embd_size
|
||||
通过这三个参数可以取得last_hidden_layer中的数据
|
||||
注:需要在当前callback中获取,若未及时获取,下一次callback会将该指针释放
|
||||
===============================================================================================================*/
|
||||
if (result->last_hidden_layer.embd_size != 0 && result->last_hidden_layer.num_tokens != 0) {
|
||||
int data_size = result->last_hidden_layer.embd_size * result->last_hidden_layer.num_tokens * sizeof(float);
|
||||
printf("\ndata_size:%d",data_size);
|
||||
std::ofstream outFile("last_hidden_layer.bin", std::ios::binary);
|
||||
if (outFile.is_open()) {
|
||||
outFile.write(reinterpret_cast<const char*>(result->last_hidden_layer.hidden_states), data_size);
|
||||
outFile.close();
|
||||
std::cout << "Data saved to output.bin successfully!" << std::endl;
|
||||
} else {
|
||||
std::cerr << "Failed to open the file for writing!" << std::endl;
|
||||
}
|
||||
}
|
||||
} else if (state == RKLLM_RUN_NORMAL) {
|
||||
printf("%s", result->text);
|
||||
// for(int i=0; i<result->num; i++)
|
||||
// {
|
||||
// printf("%d token_id: %d logprob: %f\n", i, result->tokens[i].id, result->tokens[i].logprob);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc < 4) {
|
||||
std::cerr << "Usage: " << argv[0] << " model_path max_new_tokens max_context_len\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
signal(SIGINT, exit_handler);
|
||||
printf("rkllm init start\n");
|
||||
|
||||
//设置参数及初始化
|
||||
RKLLMParam param = rkllm_createDefaultParam();
|
||||
param.model_path = argv[1];
|
||||
param.top_k = 1;
|
||||
param.max_new_tokens = std::atoi(argv[2]);
|
||||
param.max_context_len = std::atoi(argv[3]);
|
||||
|
||||
// if use multimodel mode, need to set img_start,img_end and img_content
|
||||
param.img_start = "<image>";
|
||||
param.img_end = "</image>\n";
|
||||
param.img_content = "<unk>";
|
||||
|
||||
param.skip_special_token = true;
|
||||
|
||||
int ret = rkllm_init(&llmHandle, ¶m, callback);
|
||||
if (ret == 0){
|
||||
printf("rkllm init success\n");
|
||||
} else {
|
||||
printf("rkllm init failed\n");
|
||||
exit_handler(-1);
|
||||
}
|
||||
|
||||
vector<string> pre_input;
|
||||
pre_input.push_back("<image>What is in the image?");
|
||||
cout << "\n**********************可输入以下问题对应序号获取回答/或自定义输入********************\n"
|
||||
<< endl;
|
||||
for (int i = 0; i < (int)pre_input.size(); i++)
|
||||
{
|
||||
cout << "[" << i << "] " << pre_input[i] << endl;
|
||||
}
|
||||
cout << "\n*************************************************************************\n"
|
||||
<< endl;
|
||||
|
||||
string text;
|
||||
RKLLMInput rkllm_input;
|
||||
|
||||
// 初始化 infer 参数结构体
|
||||
RKLLMInferParam rkllm_infer_params;
|
||||
memset(&rkllm_infer_params, 0, sizeof(RKLLMInferParam)); // 将所有内容初始化为 0
|
||||
|
||||
// // 1. 初始化并设置 LoRA 参数(如果需要使用 LoRA)
|
||||
// RKLLMLoraParam lora_params;
|
||||
// lora_params.lora_adapter_name = "test"; // 指定用于推理的 lora 名称
|
||||
|
||||
// // 2. 初始化并设置 Prompt Cache 参数(如果需要使用 prompt cache)
|
||||
// RKLLMPromptCacheParam prompt_cache_params;
|
||||
// prompt_cache_params.save_prompt_cache = true; // 是否保存 prompt cache
|
||||
// prompt_cache_params.prompt_cache_path = "./prompt_cache.bin"; // 若需要保存prompt cache, 指定 cache 文件路径
|
||||
|
||||
rkllm_infer_params.mode = RKLLM_INFER_GENERATE;
|
||||
// rkllm_infer_params.lora_params = &lora_params;
|
||||
// rkllm_infer_params.prompt_cache_params = &prompt_cache_params;
|
||||
|
||||
// rkllm_load_prompt_cache(llmHandle, "./prompt_cache.bin");
|
||||
while (true)
|
||||
{
|
||||
std::string input_str;
|
||||
printf("\n");
|
||||
printf("user: ");
|
||||
std::getline(std::cin, input_str);
|
||||
if (input_str == "exit")
|
||||
{
|
||||
break;
|
||||
}
|
||||
for (int i = 0; i < (int)pre_input.size(); i++)
|
||||
{
|
||||
if (input_str == to_string(i))
|
||||
{
|
||||
input_str = pre_input[i];
|
||||
cout << input_str << endl;
|
||||
}
|
||||
}
|
||||
|
||||
float * img_embed_data_ptr = (float *)malloc(64 * 2304 * sizeof(float));
|
||||
// 打开输入文件流(以二进制模式)
|
||||
std::ifstream inFile("./img_vec.bin", std::ios::binary);
|
||||
// 检查文件是否成功打开
|
||||
if (!inFile) {
|
||||
printf("Failed to open file for reading: ");
|
||||
}
|
||||
// 读取数据
|
||||
float temp_data;
|
||||
int idx = 0;
|
||||
while (inFile.read(reinterpret_cast<char*>(&temp_data), sizeof(float))){
|
||||
img_embed_data_ptr[idx] = temp_data;
|
||||
idx = idx + 1;
|
||||
}
|
||||
// 关闭文件
|
||||
inFile.close();
|
||||
|
||||
text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX;
|
||||
rkllm_input.input_type = RKLLM_INPUT_MULTIMODAL;
|
||||
rkllm_input.multimodal_input.prompt = (char *)text.c_str();
|
||||
rkllm_input.multimodal_input.image_embed = img_embed_data_ptr;
|
||||
rkllm_input.multimodal_input.n_image_tokens = 64;
|
||||
printf("robot: ");
|
||||
rkllm_run(llmHandle, &rkllm_input, &rkllm_infer_params, NULL);
|
||||
free(img_embed_data_ptr);
|
||||
}
|
||||
rkllm_destroy(llmHandle);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user