drholy/ezrknn-llm
1
0
Fork 0
mirror of https://github.com/DrHo1y/ezrknn-llm.git synced 2026-11-15 20:43:52 +07:00
Code Issues Packages Projects Releases Wiki Activity

release v1.0.1

Browse Source
This commit is contained in:
will.yang
2024-05-08 22:23:11 +08:00
parent b81deb2077
commit d59d017d4c
27 changed files with 1140 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt Normal file
View File

@@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 3.10)
project(llm_demo)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(SOURCE_FILES src/main.cpp)
add_executable(${PROJECT_NAME} ${SOURCE_FILES})
set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
include_directories(${RKLLM_API_PATH}/include)
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB} log)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB})
endif()

51
rkllm-runtime/examples/rkllm_api_demo/Readme.md Normal file
View File

@@ -0,0 +1,51 @@
# Aarch64 Linux Demo
## Build
Ensure the `GCC_COMPILER_PATH` option in the `build-linux.sh` script is correctly configured:
```sh
GCC_COMPILER_PATH=/path/to/your/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
```
To execute, run:
```bash
bash build-linux.sh
```
## Install
Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
```bash
adb push build/build_linux_aarch64_Release/llm_demo /userdata/llm
adb push ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /userdata/llm/lib
```
## Run
```bash
adb shell
cd /userdata/llm
export LD_LIBRARY_PATH=./lib
taskset f0 ./llm_demo /path/to/your/rkllm/model
```
# Android Demo
## Build
Ensure the `ANDROID_NDK_PATH` option in the `build-android.sh` script is correctly configured:
```sh
ANDROID_NDK_PATH=/path/to/your/android-ndk-r18b
```
To execute, run:
```bash
bash build-android.sh
```
## Install
Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
```bash
adb push build/build_android_arm64-v8a_Release/llm_demo /userdata/llm
adb push ../../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib
```
## Run
```bash
adb shell
cd /userdata/llm
export LD_LIBRARY_PATH=./lib
taskset f0 ./llm_demo /path/to/your/rkllm/model
```

33
rkllm-runtime/examples/rkllm_api_demo/build-android.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Debug / Release / RelWithDebInfo
if [[ -z ${BUILD_TYPE} ]];then
BUILD_TYPE=Release
fi
ANDROID_NDK_PATH=~/android-ndk-r21e
TARGET_ARCH=arm64-v8a
TARGET_PLATFORM=android
if [[ -n ${TARGET_ARCH} ]];then
TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
fi
ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
if [[ ! -d "${BUILD_DIR}" ]]; then
mkdir -p ${BUILD_DIR}
fi
cd ${BUILD_DIR}
cmake ../.. \
-DCMAKE_SYSTEM_NAME=Android \
-DCMAKE_SYSTEM_VERSION=23 \
-DCMAKE_ANDROID_ARCH_ABI=${TARGET_ARCH} \
-DCMAKE_ANDROID_STL_TYPE=c++_static \
-DCMAKE_ANDROID_NDK=${ANDROID_NDK_PATH} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
make -j4

34
rkllm-runtime/examples/rkllm_api_demo/build-linux.sh Normal file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
# Debug / Release / RelWithDebInfo
if [[ -z ${BUILD_TYPE} ]];then
BUILD_TYPE=Release
fi
GCC_COMPILER_PATH=~/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
C_COMPILER=${GCC_COMPILER_PATH}-gcc
CXX_COMPILER=${GCC_COMPILER_PATH}-g++
STRIP_COMPILER=${GCC_COMPILER_PATH}-strip
TARGET_ARCH=aarch64
TARGET_PLATFORM=linux
if [[ -n ${TARGET_ARCH} ]];then
TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
fi
ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
if [[ ! -d "${BUILD_DIR}" ]]; then
mkdir -p ${BUILD_DIR}
fi
cd ${BUILD_DIR}
cmake ../.. \
-DCMAKE_SYSTEM_PROCESSOR=${TARGET_ARCH} \
-DCMAKE_SYSTEM_NAME=Linux \
-DCMAKE_C_COMPILER=${C_COMPILER} \
-DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
make -j4

128
rkllm-runtime/examples/rkllm_api_demo/src/main.cpp Normal file
View File

@@ -0,0 +1,128 @@
// Copyright (c) 2024 by Rockchip Electronics Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string.h>
#include <unistd.h>
#include <string>
#include "rkllm.h"
#include <fstream>
#include <iostream>
#include <csignal>
#include <vector>
#define PROMPT_TEXT_PREFIX "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
#define PROMPT_TEXT_POSTFIX "<|im_end|><|im_start|>assistant"
using namespace std;
LLMHandle llmHandle = nullptr;
void exit_handler(int signal)
{
if (llmHandle != nullptr)
{
{
cout << "程序即将退出" << endl;
LLMHandle _tmp = llmHandle;
llmHandle = nullptr;
rkllm_destroy(_tmp);
}
exit(signal);
}
}
void callback(RKLLMResult *result, void *userdata, LLMCallState state)
{
if (state == LLM_RUN_FINISH)
{
printf("\n");
}
else if (state == LLM_RUN_ERROR)
{
printf("\\run error\n");
}
else
{
printf("%s", result->text);
}
}
int main(int argc, char **argv)
{
if (argc != 2)
{
printf("Usage:%s [rkllm_model_path]\n", argv[0]);
return -1;
}
signal(SIGINT, exit_handler);
string rkllm_model(argv[1]);
printf("rkllm init start\n");
//设置参数及初始化
RKLLMParam param = rkllm_createDefaultParam();
param.model_path = rkllm_model.c_str();
param.num_npu_core = 2;
param.top_k = 1;
param.max_new_tokens = 256;
param.max_context_len = 512;
param.logprobs = false;
param.top_logprobs = 5;
param.use_gpu = false;
rkllm_init(&llmHandle, param, callback);
printf("rkllm init success\n");
vector<string> pre_input;
pre_input.push_back("把下面的现代文翻译成文言文：到了春风和煦，阳光明媚的时候，湖面平静，没有惊涛骇浪，天色湖光相连，一片碧绿，广阔无际；沙洲上的鸥鸟，时而飞翔，时而停歇，美丽的鱼游来游去，岸上与小洲上的花草，青翠欲滴。");
pre_input.push_back("以咏梅为题目，帮我写一首古诗，要求包含梅花、白雪等元素。");
pre_input.push_back("上联: 江边惯看千帆过");
pre_input.push_back("把这句话翻译成中文：Knowledge can be acquired from many sources. These include books, teachers and practical experience, and each has its own advantages. The knowledge we gain from books and formal education enables us to learn about things that we have no opportunity to experience in daily life. We can also develop our analytical skills and learn how to view and interpret the world around us in different ways. Furthermore, we can learn from the past by reading books. In this way, we won't repeat the mistakes of others and can build on their achievements.");
pre_input.push_back("把这句话翻译成英文：RK3588是新一代高端处理器，具有高算力、低功耗、超强多媒体、丰富数据接口等特点");
cout << "\n**********************可输入以下问题对应序号获取回答/或自定义输入********************\n"
<< endl;
for (int i = 0; i < (int)pre_input.size(); i++)
{
cout << "[" << i << "] " << pre_input[i] << endl;
}
cout << "\n*************************************************************************\n"
<< endl;
string text;
while (true)
{
std::string input_str;
printf("\n");
printf("user: ");
std::getline(std::cin, input_str);
if (input_str == "exit")
{
break;
}
for (int i = 0; i < (int)pre_input.size(); i++)
{
if (input_str == to_string(i))
{
input_str = pre_input[i];
cout << input_str << endl;
}
}
// string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX;
string text = input_str;
printf("robot: ");
rkllm_run(llmHandle, text.c_str(), NULL);
}
rkllm_destroy(llmHandle);
return 0;
}

30
rkllm-runtime/examples/rkllm_server_demo/README.md Normal file
View File

@@ -0,0 +1,30 @@
# RKLLM-Server Demo
## Before Run
Before running the demo, you need to prepare the following files:
- The transformed RKLLM model file in board.
- check the IP address of the board with 'ifconfig' command.
## RKLLM-Server-Flask Demo
### Build
You can run the demo with the only command:
```bash
# ./build_rkllm_server_flask.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad]
./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
```
### Access with API
After building the RKLLM-Server-Flask, You can use ‘chat_api_flask.py’ to access the RKLLM-Server-Flask and get the answser of RKLLM models.
Attention: you should check the IP address of the board with 'ifconfig' command and replace the IP address in the ‘chat_api_flask.py’.
## RKLLM-Server-Gradio Demo
### Build
You can run the demo with the only command:
```bash
# ./build_rkllm_server_gradio.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad]
./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
```
### Access the Server
After running the demo, You can access the RKLLM-Server-Gradio with two ways:
1. Just Start your browser and access the URL: ‘http://[board_ip]:8080/’. You can chat with the RKLLM models in visual interface.
2. Use the 'chat_api_gradio.py'(you need fix the IP address in the code previously) and get the answser of RKLLM models.

61
rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_flask.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
#*****************************************************************************************#
# 该脚本为 RKLLM-Server-Flask 服务的一键设置脚本
# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Flask 服务的自动化部署。
# 使用说明: ./build_rkllm_server_flask.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径]
# example: ./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
#*****************************************************************************************#
#################### 检查板端是否已经安装了 pip/gradio 库 ####################
# 1.准备板端的gradio环境
adb shell << EOF
# 检查是否安装了 pip3
if ! command -v pip3 &> /dev/null; then
echo "-------- pip3 未安装，将进行安装... --------"
# 安装 pip3
sudo apt update
sudo apt install python3-pip -y
else
echo "-------- pip3 已经安装 --------"
fi
# 检查是否安装了 flask
if ! python3 -c "import flask" &> /dev/null; then
echo "-------- flask 未安装，将进行安装... --------"
# 安装 flask
pip install flask==2.2.2 Werkzeug==2.2.2 -i https://pypi.tuna.tsinghua.edu.cn/simple
else
echo "-------- flask 已经安装 --------"
fi
exit
EOF
#################### 推送 server 运行的相关文件进入板端 ####################
# 2.检查需要推送进板端的路径是否存在
adb shell ls $2 > /dev/null 2>&1
if [ $? -ne 0 ]; then
# 如果路径不存在，则创建路径
adb shell mkdir -p $2
echo "-------- rkllm_server 工作目录不存在，已创建目录 --------"
else
echo "-------- rkllm_server 工作目录已存在 --------"
fi
# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件
cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/
# 4.推送文件到 Linux 板端
adb push ./rkllm_server $2
#################### 进入板端并启动 server 服务 ####################
# 5.进入板端启动 server 服务
adb shell << EOF
cd $2/rkllm_server/
python3 flask_server.py --target_platform $1 --rkllm_model_path $3
EOF

61
rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_gradio.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
#*****************************************************************************************#
# 该脚本为 RKLLM-Server-Gradio 服务的一键设置脚本
# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Gradio 服务的自动化部署。
# 使用说明: ./build_rkllm_server_gradio.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径]
# example: ./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm
#*****************************************************************************************#
#################### 检查板端是否已经安装了 pip/gradio 库 ####################
# 1.准备板端的gradio环境
adb shell << EOF
# 检查是否安装了 pip3
if ! command -v pip3 &> /dev/null; then
echo "-------- pip3 未安装，将进行安装... --------"
# 安装 pip3
sudo apt update
sudo apt install python3-pip -y
else
echo "-------- pip3 已经安装 --------"
fi
# 检查是否安装了 gradio
if ! python3 -c "import gradio" &> /dev/null; then
echo "-------- Gradio 未安装，将进行安装... --------"
# 安装 Gradio
pip3 install gradio>=4.24.0 -i https://pypi.tuna.tsinghua.edu.cn/simple/
else
echo "-------- Gradio 已经安装 --------"
fi
exit
EOF
#################### 推送 server 运行的相关文件进入板端 ####################
# 2.检查需要推送进板端的路径是否存在
adb shell ls $2 > /dev/null 2>&1
if [ $? -ne 0 ]; then
# 如果路径不存在，则创建路径
adb shell mkdir -p $2
echo "-------- rkllm_server 工作目录不存在，已创建目录 --------"
else
echo "-------- rkllm_server 工作目录已存在 --------"
fi
# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件
cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/
# 4.推送文件到 Linux 板端
adb push ./rkllm_server $2
#################### 进入板端并启动 server 服务 ####################
# 5.进入板端启动 server 服务
adb shell << EOF
cd $2/rkllm_server/
python3 gradio_server.py --target_platform $1 --rkllm_model_path $3
EOF

81
rkllm-runtime/examples/rkllm_server_demo/chat_api_flask.py Normal file
View File

@@ -0,0 +1,81 @@
import sys
import requests
import json
# 设置 Server 服务器的地址
server_url = 'http://172.16.10.102:8080/rkllm_chat'
# 设置是否开启流式对话
is_streaming = True
# 创建一个会话对象
session = requests.Session()
session.keep_alive = False # 关闭连接池，保持长连接
adapter = requests.adapters.HTTPAdapter(max_retries=5)
session.mount('https://', adapter)
session.mount('http://', adapter)
if __name__ == '__main__':
print("============================")
print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....")
print("============================")
# 进入循环，持续获取用户输入，并与RKLLM模型进行对话
while True:
try:
user_message = input("请输入您的问题：")
if user_message == "exit":
print("============================")
print("程序正在退出......")
print("============================")
break
else:
# 设置请求头，此处的请求头实际并无作用，仅为模拟OpenAI接口设计
headers = {
'Content-Type': 'application/json',
'Authorization': 'not_required'
}
# 准备要发送的数据
# model: 为用户在设置RKLLM-Server时定义的模型，此处并无作用
# messages: 用户输入的问题，RKLLM-Server将会把它作为输入，并返回模型的回复；支持在 messags 加入多个问题
# stream: 是否开启流式对话，与OpenAI接口相同
data = {
"model": 'your_model_deploy_with_RKLLM_Server',
"messages": [{"role": "user", "content": user_message}],
"stream": is_streaming
}
# 发送 POST 请求
responses = session.post(server_url, json=data, headers=headers, stream=is_streaming, verify=False)
if not is_streaming:
# 解析响应
if responses.status_code == 200:
print("Q:", data["messages"][-1]["content"])
print("A:", json.loads(responses.text)["choices"][-1]["message"]["content"])
else:
print("Error:", responses.text)
else:
if responses.status_code == 200:
print("Q:", data["messages"][-1]["content"])
print("A:", end="")
for line in responses.iter_lines():
if line:
line = json.loads(line.decode('utf-8'))
if line["choices"][-1]["finish_reason"] != "stop":
print(line["choices"][-1]["delta"]["content"], end="")
sys.stdout.flush()
else:
print('Error:', responses.text)
except KeyboardInterrupt:
# 捕获 Ctrl-C 信号，关闭会话
session.close()
print("\n")
print("============================")
print("程序正在退出......")
print("============================")
break

44
rkllm-runtime/examples/rkllm_server_demo/chat_api_gradio.py Normal file
View File

@@ -0,0 +1,44 @@
from gradio_client import Client
# 该函数通过调用Gradio Client API与RKLLM模型进行交互
def chat_with_rkllm(user_message, history=[]):
# 实例化Gradio Client，用户需要根据自己部署的具体网址进行修改
client = Client("http://172.16.10.102:8080/")
# 调用Gradio Client API进行交互，内部的API主要包括：
# /get_user_input：模型获取用户输入，并将输入添加至历史记录history
# /get_RKLLM_output：RKLLM利用已包含输入的历史记录history生成回复
_, history = client.predict(user_message=user_message, history=history, api_name="/get_user_input")
result_history = client.predict(history=history, api_name="/get_RKLLM_output")
return result_history
if __name__ == '__main__':
#初始化聊天记录
result_history = []
print("============================")
print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....")
print("============================")
# 进入循环，持续获取用户输入，并与RKLLM模型进行对话
while True:
try:
user_message = input("请输入您的问题：")
if user_message == "exit":
print("============================")
print("程序正在退出......")
print("============================")
break
else:
# 调用chat_with_rkllm函数，获取模型的回复
result_history = chat_with_rkllm(user_message, result_history)
# 打印模型输出
print("Q:", result_history[-1][0])
print("A:", result_history[-1][1])
except KeyboardInterrupt:
print("\n")
print("============================")
print("程序正在退出......")
print("============================")
break

9
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3576.sh Normal file
View File

@@ -0,0 +1,9 @@
#!/system/bin/sh
echo userspace > /sys/class/devfreq/27700000.npu/governor
echo 1000000000 > /sys/class/devfreq/27700000.npu/userspace/set_freq
echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor
echo 2208000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor
echo 2304000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed

15
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3588.sh Normal file
View File

@@ -0,0 +1,15 @@
echo userspace > /sys/class/devfreq/fdab0000.npu/governor
echo 1000000000 > /sys/class/devfreq/fdab0000.npu/userspace/set_freq
echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor
echo 1800000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor
echo 2400000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed
echo userspace > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor
echo 2400000 > /sys/devices/system/cpu/cpufreq/policy6/scaling_setspeed
echo userspace > /sys/class/devfreq/dmc/governor
echo 2112000000 > /sys/class/devfreq/dmc/userspace/set_freq
echo userspace > /sys/class/devfreq/fb000000.gpu/governor
echo 1000000000 > /sys/class/devfreq/fb000000.gpu/userspace/set_freq

297
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/flask_server.py Normal file
View File

@@ -0,0 +1,297 @@
import ctypes
import sys
import os
import subprocess
import resource
import threading
import time
import argparse
import json
from flask import Flask, request, jsonify, Response
app = Flask(__name__)
# 创建一个锁，用于控制多人访问Server
lock = threading.Lock()
# 创建一个全局变量，用于标识服务器当前是否处于阻塞状态
is_blocking = False
# 设置动态库路径
rkllm_lib = ctypes.CDLL('lib/librkllmrt.so')
# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出
global_text = []
global_state = -1
split_byte_data = bytes(b"") # 用于保存分割的字节数据
# 定义动态库中的结构体
class Token(ctypes.Structure):
_fields_ = [
("logprob", ctypes.c_float),
("id", ctypes.c_int32)
]
class RKLLMResult(ctypes.Structure):
_fields_ = [
("text", ctypes.c_char_p),
("tokens", ctypes.POINTER(Token)),
("num", ctypes.c_int32)
]
# 定义回调函数
def callback(result, userdata, state):
global global_text, global_state, split_byte_data
if state == 0:
# 保存输出的token文本及RKLLM运行状态
global_state = state
# 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析
try:
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
split_byte_data = bytes(b"")
except:
split_byte_data += result.contents.text
sys.stdout.flush()
elif state == 1:
# 保存RKLLM运行状态
global_state = state
print("\n")
sys.stdout.flush()
else:
print("run error")
# Python端与C++端的回调函数连接
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
c_callback = callback_type(callback)
# 定义动态库中的结构体
class RKNNllmParam(ctypes.Structure):
_fields_ = [
("model_path", ctypes.c_char_p),
("num_npu_core", ctypes.c_int32),
("max_context_len", ctypes.c_int32),
("max_new_tokens", ctypes.c_int32),
("top_k", ctypes.c_int32),
("top_p", ctypes.c_float),
("temperature", ctypes.c_float),
("repeat_penalty", ctypes.c_float),
("frequency_penalty", ctypes.c_float),
("presence_penalty", ctypes.c_float),
("mirostat", ctypes.c_int32),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("logprobs", ctypes.c_bool),
("top_logprobs", ctypes.c_int32),
("use_gpu", ctypes.c_bool)
]
# 定义RKLLM_Handle_t和userdata
RKLLM_Handle_t = ctypes.c_void_p
userdata = ctypes.c_void_p(None)
# 设置提示文本
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作
class RKLLM(object):
def __init__(self, model_path, target_platform):
rknnllm_param = RKNNllmParam()
rknnllm_param.model_path = bytes(model_path, 'utf-8')
if target_platform == "rk3588":
rknnllm_param.num_npu_core = 3
elif target_platform == "rk3576":
rknnllm_param.num_npu_core = 1
rknnllm_param.max_context_len = 320
rknnllm_param.max_new_tokens = 512
rknnllm_param.top_k = 1
rknnllm_param.top_p = 0.9
rknnllm_param.temperature = 0.8
rknnllm_param.repeat_penalty = 1.1
rknnllm_param.frequency_penalty = 0.0
rknnllm_param.presence_penalty = 0.0
rknnllm_param.mirostat = 0
rknnllm_param.mirostat_tau = 5.0
rknnllm_param.mirostat_eta = 0.1
rknnllm_param.logprobs = False
rknnllm_param.top_logprobs = 5
rknnllm_param.use_gpu = True
self.handle = RKLLM_Handle_t()
self.rkllm_init = rkllm_lib.rkllm_init
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type]
self.rkllm_init.restype = ctypes.c_int
self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback)
self.rkllm_run = rkllm_lib.rkllm_run
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p]
self.rkllm_run.restype = ctypes.c_int
self.rkllm_destroy = rkllm_lib.rkllm_destroy
self.rkllm_destroy.argtypes = [RKLLM_Handle_t]
self.rkllm_destroy.restype = ctypes.c_int
def run(self, prompt):
prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8')
self.rkllm_run(self.handle, prompt, ctypes.byref(userdata))
return
def release(self):
self.rkllm_destroy(self.handle)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;')
parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径')
args = parser.parse_args()
if not (args.target_platform in ["rk3588", "rk3576"]):
print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======")
sys.stdout.flush()
exit()
if not os.path.exists(args.rkllm_model_path):
print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======")
sys.stdout.flush()
exit()
# 定频设置
command = "sudo bash fix_freq_{}.sh".format(args.target_platform)
subprocess.run(command, shell=True)
# 设置文件描述符限制
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
# 初始化RKLLM模型
print("=========init....===========")
sys.stdout.flush()
target_platform = args.target_platform
model_path = args.rkllm_model_path
rkllm_model = RKLLM(model_path, target_platform)
print("RKLLM初始化成功！")
print("==============================")
sys.stdout.flush()
# 创建一个函数用于接受用户使用 request 发送的数据
@app.route('/rkllm_chat', methods=['POST'])
def receive_message():
# 链接全局变量，获取回调函数的输出信息
global global_text, global_state
global is_blocking
# 如果服务器正在阻塞状态，则返回特定响应
if is_blocking or global_state==0:
return jsonify({'status': 'error', 'message': 'RKLLM_Server is busy! Maybe you can try again later.'}), 503
# 加锁
lock.acquire()
try:
# 设置服务器为阻塞状态
is_blocking = True
# 获取 POST 请求中的 JSON 数据
data = request.json
if data and 'messages' in data:
# 重置全局变量
global_text = []
global_state = -1
# 定义返回的结构体
rkllm_responses = {
"id": "rkllm_chat",
"object": "rkllm_chat",
"created": None,
"choices": [],
"usage": {
"prompt_tokens": None,
"completion_tokens": None,
"total_tokens": None
}
}
if not "stream" in data.keys() or data["stream"] == False:
# 在这里处理收到的数据
messages = data['messages']
print("Received messages:", messages)
for index, message in enumerate(messages):
input_prompt = message['content']
rkllm_output = ""
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
model_thread.start()
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
rkllm_output += global_text.pop(0)
time.sleep(0.005)
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
rkllm_responses["choices"].append(
{"index": index,
"message": {
"role": "assistant",
"content": rkllm_output,
},
"logprobs": None,
"finish_reason": "stop"
}
)
return jsonify(rkllm_responses), 200
else:
# 在这里处理收到的数据
messages = data['messages']
print("Received messages:", messages)
for index, message in enumerate(messages):
input_prompt = message['content']
rkllm_output = ""
def generate():
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
model_thread.start()
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
rkllm_output = global_text.pop(0)
rkllm_responses["choices"].append(
{"index": index,
"delta": {
"role": "assistant",
"content": rkllm_output,
},
"logprobs": None,
"finish_reason": "stop" if global_state == 1 else None,
}
)
yield f"{json.dumps(rkllm_responses)}\n\n"
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
return Response(generate(), content_type='text/plain')
else:
return jsonify({'status': 'error', 'message': 'Invalid JSON data!'}), 400
finally:
# 释放锁
lock.release()
# 将服务器状态设置为非阻塞
is_blocking = False
# 启动 Flask 应用程序
# app.run(host='0.0.0.0', port=8080)
app.run(host='0.0.0.0', port=8080, threaded=True, debug=False)
print("====================")
print("RKLLM模型推理结束, 释放RKLLM模型资源...")
rkllm_model.release()
print("====================")

228
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/gradio_server.py Normal file
View File

@@ -0,0 +1,228 @@
import ctypes
import sys
import os
import subprocess
import resource
import threading
import time
import gradio as gr
import argparse
# 设定环境变量
os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0"
os.environ["GRADIO_SERVER_PORT"] = "8080"
# 设置动态库路径
rkllm_lib = ctypes.CDLL('lib/librkllmrt.so')
# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出
global_text = []
global_state = -1
split_byte_data = bytes(b"") # 用于保存分割的字节数据
# 定义动态库中的结构体
class Token(ctypes.Structure):
_fields_ = [
("logprob", ctypes.c_float),
("id", ctypes.c_int32)
]
class RKLLMResult(ctypes.Structure):
_fields_ = [
("text", ctypes.c_char_p),
("tokens", ctypes.POINTER(Token)),
("num", ctypes.c_int32)
]
# 定义回调函数
def callback(result, userdata, state):
global global_text, global_state, split_byte_data
if state == 0:
# 保存输出的token文本及RKLLM运行状态
global_state = state
# 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析
try:
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
split_byte_data = bytes(b"")
except:
split_byte_data += result.contents.text
sys.stdout.flush()
elif state == 1:
# 保存RKLLM运行状态
global_state = state
print("\n")
sys.stdout.flush()
else:
print("run error")
# Python端与C++端的回调函数连接
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
c_callback = callback_type(callback)
# 定义动态库中的结构体
class RKNNllmParam(ctypes.Structure):
_fields_ = [
("model_path", ctypes.c_char_p),
("num_npu_core", ctypes.c_int32),
("max_context_len", ctypes.c_int32),
("max_new_tokens", ctypes.c_int32),
("top_k", ctypes.c_int32),
("top_p", ctypes.c_float),
("temperature", ctypes.c_float),
("repeat_penalty", ctypes.c_float),
("frequency_penalty", ctypes.c_float),
("presence_penalty", ctypes.c_float),
("mirostat", ctypes.c_int32),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("logprobs", ctypes.c_bool),
("top_logprobs", ctypes.c_int32),
("use_gpu", ctypes.c_bool)
]
# 定义RKLLM_Handle_t和userdata
RKLLM_Handle_t = ctypes.c_void_p
userdata = ctypes.c_void_p(None)
# 设置提示文本
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作
class RKLLM(object):
def __init__(self, model_path, target_platform):
rknnllm_param = RKNNllmParam()
rknnllm_param.model_path = bytes(model_path, 'utf-8')
if target_platform == "rk3588":
rknnllm_param.num_npu_core = 3
elif target_platform == "rk3576":
rknnllm_param.num_npu_core = 2
rknnllm_param.max_context_len = 320
rknnllm_param.max_new_tokens = 512
rknnllm_param.top_k = 1
rknnllm_param.top_p = 0.9
rknnllm_param.temperature = 0.8
rknnllm_param.repeat_penalty = 1.1
rknnllm_param.frequency_penalty = 0.0
rknnllm_param.presence_penalty = 0.0
rknnllm_param.mirostat = 0
rknnllm_param.mirostat_tau = 5.0
rknnllm_param.mirostat_eta = 0.1
rknnllm_param.logprobs = False
rknnllm_param.top_logprobs = 5
rknnllm_param.use_gpu = True
self.handle = RKLLM_Handle_t()
self.rkllm_init = rkllm_lib.rkllm_init
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type]
self.rkllm_init.restype = ctypes.c_int
self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback)
self.rkllm_run = rkllm_lib.rkllm_run
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p]
self.rkllm_run.restype = ctypes.c_int
self.rkllm_destroy = rkllm_lib.rkllm_destroy
self.rkllm_destroy.argtypes = [RKLLM_Handle_t]
self.rkllm_destroy.restype = ctypes.c_int
def run(self, prompt):
prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8')
self.rkllm_run(self.handle, prompt, ctypes.byref(userdata))
return
def release(self):
self.rkllm_destroy(self.handle)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;')
parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径')
args = parser.parse_args()
if not (args.target_platform in ["rk3588", "rk3576"]):
print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======")
sys.stdout.flush()
exit()
if not os.path.exists(args.rkllm_model_path):
print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======")
sys.stdout.flush()
exit()
# 定频设置
command = "sudo bash fix_freq_{}.sh".format(args.target_platform)
subprocess.run(command, shell=True)
# 设置文件描述符限制
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
# 初始化RKLLM模型
print("=========init....===========")
sys.stdout.flush()
target_platform = args.target_platform
model_path = args.rkllm_model_path
rkllm_model = RKLLM(model_path, target_platform)
print("RKLLM初始化成功！")
print("==============================")
sys.stdout.flush()
# 记录用户输入的prompt
def get_user_input(user_message, history):
history = history + [[user_message, None]]
return "", history
# 获取RKLLM模型的输出并进行流式打印
def get_RKLLM_output(history):
# 链接全局变量，获取回调函数的输出信息
global global_text, global_state
global_text = []
global_state = -1
# 创建模型推理的线程
model_thread = threading.Thread(target=rkllm_model.run, args=(history[-1][0],))
model_thread.start()
# history[-1][1]表示当前的输出对话
history[-1][1] = ""
# 等待模型运行完成，定时检查模型的推理线程
model_thread_finished = False
while not model_thread_finished:
while len(global_text) > 0:
history[-1][1] += global_text.pop(0)
time.sleep(0.005)
# gradio在调用then方法式自动将yield返回的结果推进行输出
yield history
model_thread.join(timeout=0.005)
model_thread_finished = not model_thread.is_alive()
# 创建gradio界面
with gr.Blocks(title="Chat with RKLLM") as chatRKLLM:
gr.Markdown("<div align='center'><font size='70'> Chat with RKLLM </font></div>")
gr.Markdown("### 在 inputTextBox 输入您的问题，按下 Enter 键，即可与 RKLLM 模型进行对话。")
# 创建一个Chatbot组件，用于显示对话历史
rkllmServer = gr.Chatbot(height=600)
# 创建一个Textbox组件，让用户输入消息
msg = gr.Textbox(placeholder="Please input your question here...", label="inputTextBox")
# 创建一个Button组件，用于清除聊天历史
clear = gr.Button("清除")
# 将用户输入的消息提交给get_user_input函数，并且立即更新聊天历史
# 然后调用get_RKLLM_output函数，进一步更新聊天历史
# queue=False参数确保这些更新不会被放入队列，而是立即执行
msg.submit(get_user_input, [msg, rkllmServer], [msg, rkllmServer], queue=False).then(get_RKLLM_output, rkllmServer, rkllmServer)
# 当点击清除按钮时，执行一个空操作（lambda: None），并且立即清除聊天历史
clear.click(lambda: None, None, rkllmServer, queue=False)
# 启用事件队列系统
chatRKLLM.queue()
# 启动Gradio应用程序
chatRKLLM.launch()
print("====================")
print("RKLLM模型推理结束, 释放RKLLM模型资源...")
rkllm_model.release()
print("====================")

0
rkllm-runtime/examples/rkllm_server_demo/rkllm_server/lib/.gitkeep Normal file
View File