drholy/ezrknn-llm
1
0
Fork 0
mirror of https://github.com/DrHo1y/ezrknn-llm.git synced 2026-11-11 02:07:24 +07:00
Code Issues Packages Projects Releases Wiki Activity

Updated to release 1.0.1

Browse Source
This commit is contained in:
Pelochus
2024-05-09 16:55:24 +02:00
parent 973a16e14c 4a726e2667
commit de1f13006f
27 changed files with 1146 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt Normal file
View File

@@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 3.10)
project(llm_demo)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(SOURCE_FILES src/main.cpp)
add_executable(${PROJECT_NAME} ${SOURCE_FILES})
set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
include_directories(${RKLLM_API_PATH}/include)
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB} log)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB})
endif()

50
rkllm-runtime/examples/rkllm_api_demo/README.md Normal file
View File

@@ -0,0 +1,50 @@
# Aarch64 Linux Demo
## Build
To execute, run:
```bash
bash build-linux.sh
```
## Install
Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
```bash
cp build/build_linux_aarch64_Release/llm_demo /usr/bin
cp ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/lib
```
To exit the LLM enter either `exit` or `quit`.
## Run
```bash
adb shell
cd /userdata/llm
export LD_LIBRARY_PATH=./lib
taskset f0 ./llm_demo /path/to/your/rkllm/model
```
# Android Demo
## Build
Ensure the `ANDROID_NDK_PATH` option in the `build-android.sh` script is correctly configured:
```sh
ANDROID_NDK_PATH=/path/to/your/android-ndk-r18b
```
To execute, run:
```bash
bash build-android.sh
```
## Install
Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
```bash
adb push build/build_android_arm64-v8a_Release/llm_demo /userdata/llm
adb push ../../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib
```
## Run
```bash
adb shell
cd /userdata/llm
export LD_LIBRARY_PATH=./lib
taskset f0 ./llm_demo /path/to/your/rkllm/model
```

33
rkllm-runtime/examples/rkllm_api_demo/build-android.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Debug / Release / RelWithDebInfo
if [[ -z ${BUILD_TYPE} ]];then
BUILD_TYPE=Release
fi
ANDROID_NDK_PATH=~/android-ndk-r21e
TARGET_ARCH=arm64-v8a
TARGET_PLATFORM=android
if [[ -n ${TARGET_ARCH} ]];then
TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
fi
ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
if [[ ! -d "${BUILD_DIR}" ]]; then
mkdir -p ${BUILD_DIR}
fi
cd ${BUILD_DIR}
cmake ../.. \
-DCMAKE_SYSTEM_NAME=Android \
-DCMAKE_SYSTEM_VERSION=23 \
-DCMAKE_ANDROID_ARCH_ABI=${TARGET_ARCH} \
-DCMAKE_ANDROID_STL_TYPE=c++_static \
-DCMAKE_ANDROID_NDK=${ANDROID_NDK_PATH} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
make -j4

33
rkllm-runtime/examples/rkllm_api_demo/build-linux.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Debug / Release / RelWithDebInfo
if [[ -z ${BUILD_TYPE} ]];then
BUILD_TYPE=Release
fi
C_COMPILER=/usr/bin/gcc
CXX_COMPILER=/usr/bin/g++
STRIP_COMPILER=/usr/bin/strip
TARGET_ARCH=aarch64
TARGET_PLATFORM=linux
if [[ -n ${TARGET_ARCH} ]];then
TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
fi
ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
if [[ ! -d "${BUILD_DIR}" ]]; then
mkdir -p ${BUILD_DIR}
fi
cd ${BUILD_DIR}
cmake ../.. \
-DCMAKE_SYSTEM_PROCESSOR=${TARGET_ARCH} \
-DCMAKE_SYSTEM_NAME=Linux \
-DCMAKE_C_COMPILER=${C_COMPILER} \
-DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
make -j4

144
rkllm-runtime/examples/rkllm_api_demo/src/main.cpp Normal file
View File

@@ -0,0 +1,144 @@
// Copyright (c) 2024 by Rockchip Electronics Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by Pelochus
#include <string.h>
#include <unistd.h>
#include <string>
#include "rkllm.h"
#include <fstream>
#include <iostream>
#include <csignal>
#include <vector>
#define PROMPT_TEXT_PREFIX "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
#define PROMPT_TEXT_POSTFIX "<|im_end|><|im_start|>assistant"
using namespace std;
LLMHandle llmHandle = nullptr;
void exit_handler(int signal)
{
if (llmHandle != nullptr)
{
cout << "Catched exit signal. Exiting..." << endl;
LLMHandle _tmp = llmHandle;
llmHandle = nullptr;
rkllm_destroy(_tmp);
exit(signal);
}
}
void callback(RKLLMResult *result, void *userdata, LLMCallState state)
{
if (state == LLM_RUN_FINISH)
{
printf("\n");
}
else if (state == LLM_RUN_ERROR)
{
printf("\\LLM run error\n");
}
else
{
<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp
printf("%s", text);
=======
printf("%s", result->text);
>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
}
}
int main(int argc, char **argv)
{
if (argc != 2)
{
printf("Usage: %s [rkllm_model_path]\n", argv[0]);
return -1;
}
signal(SIGINT, exit_handler);
string rkllm_model(argv[1]);
printf("RKLLM starting, please wait...\n");
RKLLMParam param = rkllm_createDefaultParam();
param.model_path = rkllm_model.c_str();
param.num_npu_core = 2;
param.top_k = 1;
param.max_new_tokens = 256;
param.max_context_len = 512;
<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp
=======
param.logprobs = false;
param.top_logprobs = 5;
param.use_gpu = false;
>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
rkllm_init(&llmHandle, param, callback);
printf("RKLLM init success!\n");
vector<string> pre_input;
pre_input.push_back("Welcome to ezrkllm! This is an adaptation of Rockchip's rknn-llm repo (see github.com/airockchip/rknn-llm) for running LLMs on its SoCs' NPUs.\n");
pre_input.push_back("You are currently running the runtime for ");
pre_input.push_back(param.target_platform);
pre_input.push_back("\nTo exit the model, enter either exit or quit\n");
pre_input.push_back("\nMore information here: https://github.com/Pelochus/ezrknpu");
pre_input.push_back("\nDetailed information for devs here: https://github.com/Pelochus/ezrknn-llm");
cout << "\n*************************** Pelochus' ezrkllm runtime *************************\n" << endl;
for (int i = 0; i < (int)pre_input.size(); i++)
{
cout << pre_input[i];
}
cout << "\n*******************************************************************************\n" << endl;
string text;
while (true)
{
std::string input_str;
printf("\n");
printf("You: ");
std::getline(std::cin, input_str);
if (input_str == "exit" || input_str == "quit")
{
cout << "Quitting program..." << endl;
break;
}
for (int i = 0; i < (int)pre_input.size(); i++)
{
if (input_str == to_string(i))
{
input_str = pre_input[i];
cout << input_str << endl;
}
}
// string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX;
string text = input_str;
printf("LLM: ");
rkllm_run(llmHandle, text.c_str(), NULL);
}
rkllm_destroy(llmHandle);
return 0;
}

30
rkllm-runtime/examples/rkllm_server_demo/README.md Normal file
View File

@@ -0,0 +1,30 @@
# RKLLM-Server Demo
## Before Run
Before running the demo, you need to prepare the following files:
- The transformed RKLLM model file in board.
- check the IP address of the board with 'ifconfig' command.
## RKLLM-Server-Flask Demo
### Build
You can run the demo with the only command:
```bash
# ./build_rkllm_server_flask.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad]
./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
```
### Access with API
After building the RKLLM-Server-Flask, You can use ‘chat_api_flask.py’ to access the RKLLM-Server-Flask and get the answser of RKLLM models.
Attention: you should check the IP address of the board with 'ifconfig' command and replace the IP address in the ‘chat_api_flask.py’.
## RKLLM-Server-Gradio Demo
### Build
You can run the demo with the only command:
```bash
# ./build_rkllm_server_gradio.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad]
./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
```
### Access the Server
After running the demo, You can access the RKLLM-Server-Gradio with two ways:
1. Just Start your browser and access the URL: ‘http://[board_ip]:8080/’. You can chat with the RKLLM models in visual interface.
2. Use the 'chat_api_gradio.py'(you need fix the IP address in the code previously) and get the answser of RKLLM models.

61
rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_flask.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
#*****************************************************************************************#
# 该脚本为 RKLLM-Server-Flask 服务的一键设置脚本
# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Flask 服务的自动化部署。
# 使用说明: ./build_rkllm_server_flask.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径]
# example: ./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
#*****************************************************************************************#
#################### 检查板端是否已经安装了 pip/gradio 库 ####################
# 1.准备板端的gradio环境
adb shell << EOF
# 检查是否安装了 pip3
if ! command -v pip3 &> /dev/null; then
echo "-------- pip3 未安装，将进行安装... --------"
# 安装 pip3
sudo apt update
sudo apt install python3-pip -y
else
echo "-------- pip3 已经安装 --------"
fi
# 检查是否安装了 flask
if ! python3 -c "import flask" &> /dev/null; then
echo "-------- flask 未安装，将进行安装... --------"
# 安装 flask
pip install flask==2.2.2 Werkzeug==2.2.2 -i https://pypi.tuna.tsinghua.edu.cn/simple
else
echo "-------- flask 已经安装 --------"
fi
exit
EOF
#################### 推送 server 运行的相关文件进入板端 ####################
# 2.检查需要推送进板端的路径是否存在
adb shell ls $2 > /dev/null 2>&1
if [ $? -ne 0 ]; then
# 如果路径不存在，则创建路径
adb shell mkdir -p $2
echo "-------- rkllm_server 工作目录不存在，已创建目录 --------"
else
echo "-------- rkllm_server 工作目录已存在 --------"
fi
# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件
cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/
# 4.推送文件到 Linux 板端
adb push ./rkllm_server $2
#################### 进入板端并启动 server 服务 ####################
# 5.进入板端启动 server 服务
adb shell << EOF
cd $2/rkllm_server/
python3 flask_server.py --target_platform $1 --rkllm_model_path $3
EOF

61
rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_gradio.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
#*****************************************************************************************#
# 该脚本为 RKLLM-Server-Gradio 服务的一键设置脚本
# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Gradio 服务的自动化部署。
# 使用说明: ./build_rkllm_server_gradio.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径]
# example: ./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
#*****************************************************************************************#
#################### 检查板端是否已经安装了 pip/gradio 库 ####################
# 1.准备板端的gradio环境
adb shell << EOF
# 检查是否安装了 pip3
if ! command -v pip3 &> /dev/null; then
echo "-------- pip3 未安装，将进行安装... --------"
# 安装 pip3
sudo apt update
sudo apt install python3-pip -y
else
echo "-------- pip3 已经安装 --------"
fi
# 检查是否安装了 gradio
if ! python3 -c "import gradio" &> /dev/null; then
echo "-------- Gradio 未安装，将进行安装... --------"
# 安装 Gradio
pip3 install gradio>=4.24.0 -i https://pypi.tuna.tsinghua.edu.cn/simple/
else
echo "-------- Gradio 已经安装 --------"
fi
exit
EOF
#################### 推送 server 运行的相关文件进入板端 ####################
# 2.检查需要推送进板端的路径是否存在
adb shell ls $2 > /dev/null 2>&1
if [ $? -ne 0 ]; then
# 如果路径不存在，则创建路径
adb shell mkdir -p $2
echo "-------- rkllm_server 工作目录不存在，已创建目录 --------"
else
echo "-------- rkllm_server 工作目录已存在 --------"
fi
# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件
cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/
# 4.推送文件到 Linux 板端
adb push ./rkllm_server $2
#################### 进入板端并启动 server 服务 ####################
# 5.进入板端启动 server 服务
adb shell << EOF
cd $2/rkllm_server/
python3 gradio_server.py --target_platform $1 --rkllm_model_path $3
EOF

81
rkllm-runtime/examples/rkllm_server_demo/chat_api_flask.py Normal file
View File

@@ -0,0 +1,81 @@
import sys
import requests
import json
# 设置 Server 服务器的地址
server_url = 'http://172.16.10.102:8080/rkllm_chat'
# 设置是否开启流式对话
is_streaming = True
# 创建一个会话对象
session = requests.Session()
session.keep_alive = False # 关闭连接池，保持长连接
adapter = requests.adapters.HTTPAdapter(max_retries=5)
session.mount('https://', adapter)
session.mount('http://', adapter)
if __name__ == '__main__':
print("============================")
print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....")
print("============================")
# 进入循环，持续获取用户输入，并与RKLLM模型进行对话
while True:
try:
user_message = input("请输入您的问题：")
if user_message == "exit":
print("============================")
print("程序正在退出......")
print("============================")
break
else:
# 设置请求头，此处的请求头实际并无作用，仅为模拟OpenAI接口设计
headers = {
'Content-Type': 'application/json',
'Authorization': 'not_required'
}
# 准备要发送的数据
# model: 为用户在设置RKLLM-Server时定义的模型，此处并无作用
# messages: 用户输入的问题，RKLLM-Server将会把它作为输入，并返回模型的回复；支持在 messags 加入多个问题
# stream: 是否开启流式对话，与OpenAI接口相同
data = {
"model": 'your_model_deploy_with_RKLLM_Server',
"messages": [{"role": "user", "content": user_message}],
"stream": is_streaming
}
# 发送 POST 请求
responses = session.post(server_url, json=data, headers=headers, stream=is_streaming, verify=False)
if not is_streaming:
# 解析响应
if responses.status_code == 200:
print("Q:", data["messages"][-1]["content"])
print("A:", json.loads(responses.text)["choices"][-1]["message"]["content"])
else:
print("Error:", responses.text)
else:
if responses.status_code == 200:
print("Q:", data["messages"][-1]["content"])
print("A:", end="")
for line in responses.iter_lines():
if line:
line = json.loads(line.decode('utf-8'))
if line["choices"][-1]["finish_reason"] != "stop":
print(line["choices"][-1]["delta"]["content"], end="")
sys.stdout.flush()
else:
print('Error:', responses.text)
except KeyboardInterrupt:
# 捕获 Ctrl-C 信号，关闭会话
session.close()
print("\n")
print("============================")
print("程序正在退出......")
print("============================")
break

44
rkllm-runtime/examples/rkllm_server_demo/chat_api_gradio.py Normal file
View File

@@ -0,0 +1,44 @@
from gradio_client import Client
# 该函数通过调用Gradio Client API与RKLLM模型进行交互
def chat_with_rkllm(user_message, history=[]):
# 实例化Gradio Client，用户需要根据自己部署的具体网址进行修改
client = Client("http://172.16.10.102:8080/")
# 调用Gradio Client API进行交互，内部的API主要包括：
# /get_user_input：模型获取用户输入，并将输入添加至历史记录history
# /get_RKLLM_output：RKLLM利用已包含输入的历史记录history生成回复
_, history = client.predict(user_message=user_message, history=history, api_name="/get_user_input")
result_history = client.predict(history=history, api_name="/get_RKLLM_output")
return result_history
if __name__ == '__main__':
#初始化聊天记录
result_history = []
print("============================")
print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....")
print("============================")
# 进入循环，持续获取用户输入，并与RKLLM模型进行对话
while True:
try:
user_message = input("请输入您的问题：")
if user_message == "exit":
print("============================")
print("程序正在退出......")
print("============================")
break
else:
# 调用chat_with_rkllm函数，获取模型的回复
result_history = chat_with_rkllm(user_message, result_history)
# 打印模型输出
print("Q:", result_history[-1][0])
print("A:", result_history[-1][1])
except KeyboardInterrupt:
print("\n")
print("============================")
print("程序正在退出......")
print("============================")
break

9
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3576.sh Normal file
View File

@@ -0,0 +1,9 @@
#!/system/bin/sh
echo userspace > /sys/class/devfreq/27700000.npu/governor
echo 1000000000 > /sys/class/devfreq/27700000.npu/userspace/set_freq
echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor
echo 2208000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor
echo 2304000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed

15
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3588.sh Normal file
View File

@@ -0,0 +1,15 @@
echo userspace > /sys/class/devfreq/fdab0000.npu/governor
echo 1000000000 > /sys/class/devfreq/fdab0000.npu/userspace/set_freq
echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor
echo 1800000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor
echo 2400000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor
echo 2400000 > /sys/devices/system/cpu/cpufreq/policy6/scaling_setspeed
echo userspace > /sys/class/devfreq/dmc/governor
echo 2112000000 > /sys/class/devfreq/dmc/userspace/set_freq
echo userspace > /sys/class/devfreq/fb000000.gpu/governor
echo 1000000000 > /sys/class/devfreq/fb000000.gpu/userspace/set_freq

297
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/flask_server.py Normal file
View File

@@ -0,0 +1,297 @@
import ctypes
import sys
import os
import subprocess
import resource
import threading
import time
import argparse
import json
from flask import Flask, request, jsonify, Response
app = Flask(__name__)
# 创建一个锁，用于控制多人访问Server
lock = threading.Lock()
# 创建一个全局变量，用于标识服务器当前是否处于阻塞状态
is_blocking = False
# 设置动态库路径
rkllm_lib = ctypes.CDLL('lib/librkllmrt.so')
# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出
global_text = []
global_state = -1
split_byte_data = bytes(b"") # 用于保存分割的字节数据
# 定义动态库中的结构体
class Token(ctypes.Structure):
_fields_ = [
("logprob", ctypes.c_float),
("id", ctypes.c_int32)
]
class RKLLMResult(ctypes.Structure):
_fields_ = [
("text", ctypes.c_char_p),
("tokens", ctypes.POINTER(Token)),
("num", ctypes.c_int32)
]
# 定义回调函数
def callback(result, userdata, state):
global global_text, global_state, split_byte_data
if state == 0:
# 保存输出的token文本及RKLLM运行状态
global_state = state
# 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析
try:
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
split_byte_data = bytes(b"")
except:
split_byte_data += result.contents.text
sys.stdout.flush()
elif state == 1:
# 保存RKLLM运行状态
global_state = state
print("\n")
sys.stdout.flush()
else:
print("run error")
# Python端与C++端的回调函数连接
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
c_callback = callback_type(callback)
# 定义动态库中的结构体
class RKNNllmParam(ctypes.Structure):
_fields_ = [
("model_path", ctypes.c_char_p),
("num_npu_core", ctypes.c_int32),
("max_context_len", ctypes.c_int32),
("max_new_tokens", ctypes.c_int32),
("top_k", ctypes.c_int32),
("top_p", ctypes.c_float),
("temperature", ctypes.c_float),
("repeat_penalty", ctypes.c_float),
("frequency_penalty", ctypes.c_float),
("presence_penalty", ctypes.c_float),
("mirostat", ctypes.c_int32),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("logprobs", ctypes.c_bool),
("top_logprobs", ctypes.c_int32),
("use_gpu", ctypes.c_bool)
]
# 定义RKLLM_Handle_t和userdata
RKLLM_Handle_t = ctypes.c_void_p
userdata = ctypes.c_void_p(None)
# 设置提示文本
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作
class RKLLM(object):
def __init__(self, model_path, target_platform):
rknnllm_param = RKNNllmParam()
rknnllm_param.model_path = bytes(model_path, 'utf-8')
if target_platform == "rk3588":
rknnllm_param.num_npu_core = 3
elif target_platform == "rk3576":
rknnllm_param.num_npu_core = 1
rknnllm_param.max_context_len = 320
rknnllm_param.max_new_tokens = 512
rknnllm_param.top_k = 1
rknnllm_param.top_p = 0.9
rknnllm_param.temperature = 0.8
rknnllm_param.repeat_penalty = 1.1
rknnllm_param.frequency_penalty = 0.0
rknnllm_param.presence_penalty = 0.0
rknnllm_param.mirostat = 0
rknnllm_param.mirostat_tau = 5.0
rknnllm_param.mirostat_eta = 0.1
rknnllm_param.logprobs = False
rknnllm_param.top_logprobs = 5
rknnllm_param.use_gpu = True
self.handle = RKLLM_Handle_t()
self.rkllm_init = rkllm_lib.rkllm_init
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type]
self.rkllm_init.restype = ctypes.c_int
self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback)
self.rkllm_run = rkllm_lib.rkllm_run
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p]
self.rkllm_run.restype = ctypes.c_int
self.rkllm_destroy = rkllm_lib.rkllm_destroy
self.rkllm_destroy.argtypes = [RKLLM_Handle_t]
self.rkllm_destroy.restype = ctypes.c_int
def run(self, prompt):
prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8')
self.rkllm_run(self.handle, prompt, ctypes.byref(userdata))
return
def release(self):
self.rkllm_destroy(self.handle)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;')
parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径')
args = parser.parse_args()
if not (args.target_platform in ["rk3588", "rk3576"]):
print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======")
sys.stdout.flush()
exit()
if not os.path.exists(args.rkllm_model_path):
print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======")
sys.stdout.flush()
exit()
# 定频设置
command = "sudo bash fix_freq_{}.sh".format(args.target_platform)
subprocess.run(command, shell=True)
# 设置文件描述符限制
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
# 初始化RKLLM模型
print("=========init....===========")
sys.stdout.flush()
target_platform = args.target_platform
model_path = args.rkllm_model_path
rkllm_model = RKLLM(model_path, target_platform)
print("RKLLM初始化成功！")
print("==============================")
sys.stdout.flush()
# 创建一个函数用于接受用户使用 request 发送的数据
@app.route('/rkllm_chat', methods=['POST'])
def receive_message():
# 链接全局变量，获取回调函数的输出信息
global global_text, global_state
global is_blocking
# 如果服务器正在阻塞状态，则返回特定响应
if is_blocking or global_state==0:
return jsonify({'status': 'error', 'message': 'RKLLM_Server is busy! Maybe you can try again later.'}), 503
# 加锁
lock.acquire()
try:
# 设置服务器为阻塞状态
is_blocking = True
# 获取 POST 请求中的 JSON 数据
data = request.json
if data and 'messages' in data:
# 重置全局变量
global_text = []
global_state = -1
# 定义返回的结构体
rkllm_responses = {
"id": "rkllm_chat",
"object": "rkllm_chat",
"created": None,
"choices": [],
"usage": {
"prompt_tokens": None,
"completion_tokens": None,
"total_tokens": None
}
}
if not "stream" in data.keys() or data["stream"] == False:
# 在这里处理收到的数据
messages = data['messages']
print("Received messages:", messages)
for index, message in enumerate(messages):
input_prompt = message['content']
rkllm_output = ""
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
model_thread.start()
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
rkllm_output += global_text.pop(0)
time.sleep(0.005)
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
rkllm_responses["choices"].append(
{"index": index,
"message": {
"role": "assistant",
"content": rkllm_output,
},
"logprobs": None,
"finish_reason": "stop"
}
)
return jsonify(rkllm_responses), 200
else:
# 在这里处理收到的数据
messages = data['messages']
print("Received messages:", messages)
for index, message in enumerate(messages):
input_prompt = message['content']
rkllm_output = ""
def generate():
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
model_thread.start()
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
rkllm_output = global_text.pop(0)
rkllm_responses["choices"].append(
{"index": index,
"delta": {
"role": "assistant",
"content": rkllm_output,
},
"logprobs": None,
"finish_reason": "stop" if global_state == 1 else None,
}
)
yield f"{json.dumps(rkllm_responses)}\n\n"
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
return Response(generate(), content_type='text/plain')
else:
return jsonify({'status': 'error', 'message': 'Invalid JSON data!'}), 400
finally:
# 释放锁
lock.release()
# 将服务器状态设置为非阻塞
is_blocking = False
# 启动 Flask 应用程序
# app.run(host='0.0.0.0', port=8080)
app.run(host='0.0.0.0', port=8080, threaded=True, debug=False)
print("====================")
print("RKLLM模型推理结束, 释放RKLLM模型资源...")
rkllm_model.release()
print("====================")

228
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/gradio_server.py Normal file
View File

@@ -0,0 +1,228 @@
import ctypes
import sys
import os
import subprocess
import resource
import threading
import time
import gradio as gr
import argparse
# 设定环境变量
os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0"
os.environ["GRADIO_SERVER_PORT"] = "8080"
# 设置动态库路径
rkllm_lib = ctypes.CDLL('lib/librkllmrt.so')
# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出
global_text = []
global_state = -1
split_byte_data = bytes(b"") # 用于保存分割的字节数据
# 定义动态库中的结构体
class Token(ctypes.Structure):
_fields_ = [
("logprob", ctypes.c_float),
("id", ctypes.c_int32)
]
class RKLLMResult(ctypes.Structure):
_fields_ = [
("text", ctypes.c_char_p),
("tokens", ctypes.POINTER(Token)),
("num", ctypes.c_int32)
]
# 定义回调函数
def callback(result, userdata, state):
global global_text, global_state, split_byte_data
if state == 0:
# 保存输出的token文本及RKLLM运行状态
global_state = state
# 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析
try:
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
split_byte_data = bytes(b"")
except:
split_byte_data += result.contents.text
sys.stdout.flush()
elif state == 1:
# 保存RKLLM运行状态
global_state = state
print("\n")
sys.stdout.flush()
else:
print("run error")
# Python端与C++端的回调函数连接
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
c_callback = callback_type(callback)
# 定义动态库中的结构体
class RKNNllmParam(ctypes.Structure):
_fields_ = [
("model_path", ctypes.c_char_p),
("num_npu_core", ctypes.c_int32),
("max_context_len", ctypes.c_int32),
("max_new_tokens", ctypes.c_int32),
("top_k", ctypes.c_int32),
("top_p", ctypes.c_float),
("temperature", ctypes.c_float),
("repeat_penalty", ctypes.c_float),
("frequency_penalty", ctypes.c_float),
("presence_penalty", ctypes.c_float),
("mirostat", ctypes.c_int32),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("logprobs", ctypes.c_bool),
("top_logprobs", ctypes.c_int32),
("use_gpu", ctypes.c_bool)
]
# 定义RKLLM_Handle_t和userdata
RKLLM_Handle_t = ctypes.c_void_p
userdata = ctypes.c_void_p(None)
# 设置提示文本
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作
class RKLLM(object):
def __init__(self, model_path, target_platform):
rknnllm_param = RKNNllmParam()
rknnllm_param.model_path = bytes(model_path, 'utf-8')
if target_platform == "rk3588":
rknnllm_param.num_npu_core = 3
elif target_platform == "rk3576":
rknnllm_param.num_npu_core = 2
rknnllm_param.max_context_len = 320
rknnllm_param.max_new_tokens = 512
rknnllm_param.top_k = 1
rknnllm_param.top_p = 0.9
rknnllm_param.temperature = 0.8
rknnllm_param.repeat_penalty = 1.1
rknnllm_param.frequency_penalty = 0.0
rknnllm_param.presence_penalty = 0.0
rknnllm_param.mirostat = 0
rknnllm_param.mirostat_tau = 5.0
rknnllm_param.mirostat_eta = 0.1
rknnllm_param.logprobs = False
rknnllm_param.top_logprobs = 5
rknnllm_param.use_gpu = True
self.handle = RKLLM_Handle_t()
self.rkllm_init = rkllm_lib.rkllm_init
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type]
self.rkllm_init.restype = ctypes.c_int
self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback)
self.rkllm_run = rkllm_lib.rkllm_run
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p]
self.rkllm_run.restype = ctypes.c_int
self.rkllm_destroy = rkllm_lib.rkllm_destroy
self.rkllm_destroy.argtypes = [RKLLM_Handle_t]
self.rkllm_destroy.restype = ctypes.c_int
def run(self, prompt):
prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8')
self.rkllm_run(self.handle, prompt, ctypes.byref(userdata))
return
def release(self):
self.rkllm_destroy(self.handle)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;')
parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径')
args = parser.parse_args()
if not (args.target_platform in ["rk3588", "rk3576"]):
print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======")
sys.stdout.flush()
exit()
if not os.path.exists(args.rkllm_model_path):
print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======")
sys.stdout.flush()
exit()
# 定频设置
command = "sudo bash fix_freq_{}.sh".format(args.target_platform)
subprocess.run(command, shell=True)
# 设置文件描述符限制
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
# 初始化RKLLM模型
print("=========init....===========")
sys.stdout.flush()
target_platform = args.target_platform
model_path = args.rkllm_model_path
rkllm_model = RKLLM(model_path, target_platform)
print("RKLLM初始化成功！")
print("==============================")
sys.stdout.flush()
# 记录用户输入的prompt
def get_user_input(user_message, history):
history = history + [[user_message, None]]
return "", history
# 获取RKLLM模型的输出并进行流式打印
def get_RKLLM_output(history):
# 链接全局变量，获取回调函数的输出信息
global global_text, global_state
global_text = []
global_state = -1
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(history[-1][0],))
model_thread.start()
# history[-1][1]表示当前的输出对话
history[-1][1] = ""
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
history[-1][1] += global_text.pop(0)
time.sleep(0.005)
# gradio在调用then方法式自动将yield返回的结果推进行输出
yield history
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
# 创建gradio界面
with gr.Blocks(title="Chat with RKLLM") as chatRKLLM:
gr.Markdown("<div align='center'><font size='70'> Chat with RKLLM </font></div>")
gr.Markdown("### 在 inputTextBox 输入您的问题，按下 Enter 键，即可与 RKLLM 模型进行对话。")
# 创建一个Chatbot组件，用于显示对话历史
rkllmServer = gr.Chatbot(height=600)
# 创建一个Textbox组件，让用户输入消息
msg = gr.Textbox(placeholder="Please input your question here...", label="inputTextBox")
# 创建一个Button组件，用于清除聊天历史
clear = gr.Button("清除")
# 将用户输入的消息提交给get_user_input函数，并且立即更新聊天历史
# 然后调用get_RKLLM_output函数，进一步更新聊天历史
# queue=False参数确保这些更新不会被放入队列，而是立即执行
msg.submit(get_user_input, [msg, rkllmServer], [msg, rkllmServer], queue=False).then(get_RKLLM_output, rkllmServer, rkllmServer)
# 当点击清除按钮时，执行一个空操作（lambda: None），并且立即清除聊天历史
clear.click(lambda: None, None, rkllmServer, queue=False)
# 启用事件队列系统
chatRKLLM.queue()
# 启动Gradio应用程序
chatRKLLM.launch()
print("====================")
print("RKLLM模型推理结束, 释放RKLLM模型资源...")
rkllm_model.release()
print("====================")

0
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/lib/.gitkeep Normal file
View File