diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 0000000..00369d9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# CHANGELOG +## v1.0.1 + - Optimize model conversion memory occupation + - Optimize inference memory occupation + - Increase prefill speed + - Reduce initialization time + - Improve quantization accuracy + - Add support for Gemma, ChatGLM3, MiniCPM, InternLM2, and Phi-3 + - Add Server invocation + - Add inference interruption interface + - Add logprob and token_id to the return value + +## v1.0.0 + - Supports the conversion and deployment of LLM models on RK3588/RK3576 platforms + - Compatible with Hugging Face model architectures + - Currently supports the models Llama, Qwen, Qwen2, and Phi-2 + - Supports quantization with w8a8 and w4a16 precision \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0bc8afc --- /dev/null +++ b/LICENSE @@ -0,0 +1,64 @@ +Copyright (c) Rockchip Electronics Co., Ltd. +All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// 4. This Software may contain some Open Source Software. You may not redistribute +// and/or modify such Open Source Software except in compliance with the applicable +// Open Source License. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +The following Open Source Software have been modified by Rockchip Electronics Co., Ltd. +---------------------------------------------------------------------------------------- +1. ggml master +Copyright (c) 2023-2024 The ggml authors +All rights reserved. +Licensed under the terms of the MIT License + +2. llama.cpp master +Copyright (c) 2023-2024 The ggml authors +All rights reserved. +Licensed under the terms of the MIT License + +The terms of the MIT License: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 1be408a..b6bf8df 100644 --- a/README.md +++ b/README.md @@ -74,23 +74,35 @@ To convert the model, run: - RK3588 Series - RK3576 Series +# Support Models + - [X] [TinyLLAMA 1.1B](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0/tree/fe8a4ea1ffedaf415f4da2f062534de366a451e6) + - [X] [Qwen 1.8B](https://huggingface.co/Qwen/Qwen-1_8B-Chat/tree/1d0f68de57b88cfde81f3c3e537f24464d889081) + - [X] [Qwen2 0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B/tree/8f445e3628f3500ee69f24e1303c9f10f5342a39) + - [X] [Phi-2 2.7B](https://hf-mirror.com/microsoft/phi-2/tree/834565c23f9b28b96ccbeabe614dd906b6db551a) + - [X] [Phi-3 3.8B](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/tree/291e9e30e38030c23497afa30f3af1f104837aa6) + - [X] [ChatGLM3 6B](https://huggingface.co/THUDM/chatglm3-6b/tree/103caa40027ebfd8450289ca2f278eac4ff26405) + - [X] [Gemma 2B](https://huggingface.co/google/gemma-2b-it/tree/de144fb2268dee1066f515465df532c05e699d48) + - [X] [InternLM2 1.8B](https://huggingface.co/internlm/internlm2-chat-1_8b/tree/ecccbb5c87079ad84e5788baa55dd6e21a9c614d) + - [X] [MiniCPM 2B](https://huggingface.co/openbmb/MiniCPM-2B-sft-bf16/tree/79fbb1db171e6d8bf77cdb0a94076a43003abd9e) + # Download - You can also download all packages, docker image, examples, docs and platform-tools from [RKLLM_SDK](https://console.zbox.filez.com/l/RJJDmB), fetch code: rkllm # RKNN Toolkit2 -If you want to deploy additional AI model, we have introduced a new SDK called RKNN-Toolkit2. For details, please refer to: +If you want to deploy additional AI model, we have introduced a SDK called RKNN-Toolkit2. For details, please refer to: https://github.com/airockchip/rknn-toolkit2 -# Notes - -Due to recent updates to the Phi2 model, the current version of the RKLLM SDK does not yet support these changes. -Please ensure to download a version of the [Phi2](https://hf-mirror.com/microsoft/phi-2/tree/834565c23f9b28b96ccbeabe614dd906b6db551a) model that is supported. - # CHANGELOG +## v1.0.1 + - Optimize model conversion memory occupation + - Optimize inference memory occupation + - Increase prefill speed + - Reduce initialization time + - Improve quantization accuracy + - Add support for Gemma, ChatGLM3, MiniCPM, InternLM2, and Phi-3 + - Add Server invocation + - Add inference interruption interface + - Add logprob and token_id to the return value -## v1.0.0-beta - - Supports the conversion and deployment of LLM models on RK3588/RK3576 platforms - - Compatible with Hugging Face model architectures - - Currently supports the models LLaMA, Qwen, Qwen2, and Phi-2 - - Supports quantization with w8a8 and w4a16 precision +For older version, please refer [CHANGELOG](CHANGELOG.md) \ No newline at end of file diff --git a/doc/Rockchip_RKLLM_SDK_CN.pdf b/doc/Rockchip_RKLLM_SDK_CN.pdf index 122bf80..42ca2b2 100644 Binary files a/doc/Rockchip_RKLLM_SDK_CN.pdf and b/doc/Rockchip_RKLLM_SDK_CN.pdf differ diff --git a/doc/Rockchip_RKLLM_SDK_EN.pdf b/doc/Rockchip_RKLLM_SDK_EN.pdf new file mode 100644 index 0000000..4ef3cbc Binary files /dev/null and b/doc/Rockchip_RKLLM_SDK_EN.pdf differ diff --git a/rkllm-runtime/example/CMakeLists.txt b/rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt similarity index 69% rename from rkllm-runtime/example/CMakeLists.txt rename to rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt index 618b031..0c0ccad 100644 --- a/rkllm-runtime/example/CMakeLists.txt +++ b/rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt @@ -8,13 +8,14 @@ set(SOURCE_FILES src/main.cpp) add_executable(${PROJECT_NAME} ${SOURCE_FILES}) -set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api") +set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api") include_directories(${RKLLM_API_PATH}/include) if(CMAKE_SYSTEM_NAME STREQUAL "Android") set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so) + target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB} log) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so) + target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB}) endif() -target_link_libraries(${PROJECT_NAME} ${RKLLM_RT_LIB}) diff --git a/rkllm-runtime/example/README.md b/rkllm-runtime/examples/rkllm_api_demo/README.md similarity index 91% rename from rkllm-runtime/example/README.md rename to rkllm-runtime/examples/rkllm_api_demo/README.md index 144fbec..b514c08 100644 --- a/rkllm-runtime/example/README.md +++ b/rkllm-runtime/examples/rkllm_api_demo/README.md @@ -7,9 +7,10 @@ bash build-linux.sh ## Install Push the compiled `llm_demo` file and `librkllmrt.so` file to the device: + ```bash cp build/build_linux_aarch64_Release/llm_demo /usr/bin -cp ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/include +cp ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/lib ``` To exit the LLM enter either `exit` or `quit`. @@ -37,7 +38,7 @@ bash build-android.sh Push the compiled `llm_demo` file and `librkllmrt.so` file to the device: ```bash adb push build/build_android_arm64-v8a_Release/llm_demo /userdata/llm -adb push ../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib +adb push ../../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib ``` ## Run diff --git a/rkllm-runtime/example/build-android.sh b/rkllm-runtime/examples/rkllm_api_demo/build-android.sh similarity index 95% rename from rkllm-runtime/example/build-android.sh rename to rkllm-runtime/examples/rkllm_api_demo/build-android.sh index 4c8812d..50f74bf 100755 --- a/rkllm-runtime/example/build-android.sh +++ b/rkllm-runtime/examples/rkllm_api_demo/build-android.sh @@ -4,7 +4,7 @@ if [[ -z ${BUILD_TYPE} ]];then BUILD_TYPE=Release fi -ANDROID_NDK_PATH=~/android-ndk-r18b +ANDROID_NDK_PATH=~/android-ndk-r21e TARGET_ARCH=arm64-v8a TARGET_PLATFORM=android diff --git a/rkllm-runtime/example/build-linux.sh b/rkllm-runtime/examples/rkllm_api_demo/build-linux.sh similarity index 100% rename from rkllm-runtime/example/build-linux.sh rename to rkllm-runtime/examples/rkllm_api_demo/build-linux.sh diff --git a/rkllm-runtime/example/src/main.cpp b/rkllm-runtime/examples/rkllm_api_demo/src/main.cpp similarity index 85% rename from rkllm-runtime/example/src/main.cpp rename to rkllm-runtime/examples/rkllm_api_demo/src/main.cpp index 5762aef..84487a5 100644 --- a/rkllm-runtime/example/src/main.cpp +++ b/rkllm-runtime/examples/rkllm_api_demo/src/main.cpp @@ -42,7 +42,7 @@ void exit_handler(int signal) } } -void callback(const char *text, void *userdata, LLMCallState state) +void callback(RKLLMResult *result, void *userdata, LLMCallState state) { if (state == LLM_RUN_FINISH) { @@ -54,7 +54,11 @@ void callback(const char *text, void *userdata, LLMCallState state) } else { +<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp printf("%s", text); +======= + printf("%s", result->text); +>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp } } @@ -71,13 +75,19 @@ int main(int argc, char **argv) printf("RKLLM starting, please wait...\n"); RKLLMParam param = rkllm_createDefaultParam(); - param.modelPath = rkllm_model.c_str(); - param.target_platform = "rk3588"; + param.model_path = rkllm_model.c_str(); param.num_npu_core = 2; param.top_k = 1; param.max_new_tokens = 256; param.max_context_len = 512; +<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp +======= + param.logprobs = false; + param.top_logprobs = 5; + param.use_gpu = false; + +>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp rkllm_init(&llmHandle, param, callback); printf("RKLLM init success!\n"); @@ -121,7 +131,9 @@ int main(int argc, char **argv) } } - string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX; + // string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX; + string text = input_str; + printf("LLM: "); rkllm_run(llmHandle, text.c_str(), NULL); } diff --git a/rkllm-runtime/examples/rkllm_server_demo/README.md b/rkllm-runtime/examples/rkllm_server_demo/README.md new file mode 100644 index 0000000..75f46c8 --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/README.md @@ -0,0 +1,30 @@ +# RKLLM-Server Demo +## Before Run +Before running the demo, you need to prepare the following files: +- The transformed RKLLM model file in board. +- check the IP address of the board with 'ifconfig' command. + +## RKLLM-Server-Flask Demo +### Build +You can run the demo with the only command: +```bash +# ./build_rkllm_server_flask.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad] +./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm +``` +### Access with API +After building the RKLLM-Server-Flask, You can use ‘chat_api_flask.py’ to access the RKLLM-Server-Flask and get the answser of RKLLM models. + +Attention: you should check the IP address of the board with 'ifconfig' command and replace the IP address in the ‘chat_api_flask.py’. + +## RKLLM-Server-Gradio Demo +### Build +You can run the demo with the only command: +```bash +# ./build_rkllm_server_gradio.sh [target_platform:rk3588/rk3576] [RKLLM-Server workshop] [transformed_rkllm_model_path in borad] +./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm +``` +### Access the Server +After running the demo, You can access the RKLLM-Server-Gradio with two ways: +1. Just Start your browser and access the URL: ‘http://[board_ip]:8080/’. You can chat with the RKLLM models in visual interface. +2. Use the 'chat_api_gradio.py'(you need fix the IP address in the code previously) and get the answser of RKLLM models. + \ No newline at end of file diff --git a/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_flask.sh b/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_flask.sh new file mode 100755 index 0000000..7f4c9ce --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_flask.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +#*****************************************************************************************# +# 该脚本为 RKLLM-Server-Flask 服务的一键设置脚本 +# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Flask 服务的自动化部署。 +# 使用说明: ./build_rkllm_server_flask.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径] +# example: ./build_rkllm_server_flask.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm +#*****************************************************************************************# + +#################### 检查板端是否已经安装了 pip/gradio 库 #################### +# 1.准备板端的gradio环境 +adb shell << EOF + +# 检查是否安装了 pip3 +if ! command -v pip3 &> /dev/null; then + echo "-------- pip3 未安装，将进行安装... --------" + # 安装 pip3 + sudo apt update + sudo apt install python3-pip -y +else + echo "-------- pip3 已经安装 --------" +fi + +# 检查是否安装了 flask +if ! python3 -c "import flask" &> /dev/null; then + echo "-------- flask 未安装，将进行安装... --------" + # 安装 flask + pip install flask==2.2.2 Werkzeug==2.2.2 -i https://pypi.tuna.tsinghua.edu.cn/simple +else + echo "-------- flask 已经安装 --------" +fi + +exit + +EOF + +#################### 推送 server 运行的相关文件进入板端 #################### +# 2.检查需要推送进板端的路径是否存在 +adb shell ls $2 > /dev/null 2>&1 +if [ $? -ne 0 ]; then + # 如果路径不存在，则创建路径 + adb shell mkdir -p $2 + echo "-------- rkllm_server 工作目录不存在，已创建目录 --------" +else + echo "-------- rkllm_server 工作目录已存在 --------" +fi + +# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件 +cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/ + +# 4.推送文件到 Linux 板端 +adb push ./rkllm_server $2 + +#################### 进入板端并启动 server 服务 #################### +# 5.进入板端启动 server 服务 +adb shell << EOF + +cd $2/rkllm_server/ +python3 flask_server.py --target_platform $1 --rkllm_model_path $3 + +EOF diff --git a/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_gradio.sh b/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_gradio.sh new file mode 100755 index 0000000..356968d --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/build_rkllm_server_gradio.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +#*****************************************************************************************# +# 该脚本为 RKLLM-Server-Gradio 服务的一键设置脚本 +# 用户可以运行该脚本实现Linux板端的 RKLLM-Server-Gradio 服务的自动化部署。 +# 使用说明: ./build_rkllm_server_gradio.sh [目标平台:rk3588/rk3576] [RKLLM-Server工作路径] [已转换的rkllm模型在板端的绝对路径] +# example: ./build_rkllm_server_gradio.sh rk3588 /user/data/rkllm_server /user/data/rkllm_server/model.rkllm +#*****************************************************************************************# + +#################### 检查板端是否已经安装了 pip/gradio 库 #################### +# 1.准备板端的gradio环境 +adb shell << EOF + +# 检查是否安装了 pip3 +if ! command -v pip3 &> /dev/null; then + echo "-------- pip3 未安装，将进行安装... --------" + # 安装 pip3 + sudo apt update + sudo apt install python3-pip -y +else + echo "-------- pip3 已经安装 --------" +fi + +# 检查是否安装了 gradio +if ! python3 -c "import gradio" &> /dev/null; then + echo "-------- Gradio 未安装，将进行安装... --------" + # 安装 Gradio + pip3 install gradio>=4.24.0 -i https://pypi.tuna.tsinghua.edu.cn/simple/ +else + echo "-------- Gradio 已经安装 --------" +fi + +exit + +EOF + +#################### 推送 server 运行的相关文件进入板端 #################### +# 2.检查需要推送进板端的路径是否存在 +adb shell ls $2 > /dev/null 2>&1 +if [ $? -ne 0 ]; then + # 如果路径不存在，则创建路径 + adb shell mkdir -p $2 + echo "-------- rkllm_server 工作目录不存在，已创建目录 --------" +else + echo "-------- rkllm_server 工作目录已存在 --------" +fi + +# 3.更新 ./rkllm_server/lib 中的 librkllmrt.so 文件 +cp ../../runtime/Linux/librkllm_api/aarch64/librkllmrt.so ./rkllm_server/lib/ + +# 4.推送文件到 Linux 板端 +adb push ./rkllm_server $2 + +#################### 进入板端并启动 server 服务 #################### +# 5.进入板端启动 server 服务 +adb shell << EOF + +cd $2/rkllm_server/ +python3 gradio_server.py --target_platform $1 --rkllm_model_path $3 + +EOF diff --git a/rkllm-runtime/examples/rkllm_server_demo/chat_api_flask.py b/rkllm-runtime/examples/rkllm_server_demo/chat_api_flask.py new file mode 100644 index 0000000..f6d6d06 --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/chat_api_flask.py @@ -0,0 +1,81 @@ +import sys +import requests +import json + +# 设置 Server 服务器的地址 +server_url = 'http://172.16.10.102:8080/rkllm_chat' +# 设置是否开启流式对话 +is_streaming = True + +# 创建一个会话对象 +session = requests.Session() +session.keep_alive = False # 关闭连接池，保持长连接 +adapter = requests.adapters.HTTPAdapter(max_retries=5) +session.mount('https://', adapter) +session.mount('http://', adapter) + +if __name__ == '__main__': + print("============================") + print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....") + print("============================") + # 进入循环，持续获取用户输入，并与RKLLM模型进行对话 + while True: + try: + user_message = input("请输入您的问题：") + if user_message == "exit": + print("============================") + print("程序正在退出......") + print("============================") + break + else: + # 设置请求头，此处的请求头实际并无作用，仅为模拟OpenAI接口设计 + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'not_required' + } + + # 准备要发送的数据 + # model: 为用户在设置RKLLM-Server时定义的模型，此处并无作用 + # messages: 用户输入的问题，RKLLM-Server将会把它作为输入，并返回模型的回复；支持在 messags 加入多个问题 + # stream: 是否开启流式对话，与OpenAI接口相同 + data = { + "model": 'your_model_deploy_with_RKLLM_Server', + "messages": [{"role": "user", "content": user_message}], + "stream": is_streaming + } + + # 发送 POST 请求 + responses = session.post(server_url, json=data, headers=headers, stream=is_streaming, verify=False) + + if not is_streaming: + # 解析响应 + if responses.status_code == 200: + print("Q:", data["messages"][-1]["content"]) + print("A:", json.loads(responses.text)["choices"][-1]["message"]["content"]) + else: + print("Error:", responses.text) + else: + if responses.status_code == 200: + print("Q:", data["messages"][-1]["content"]) + print("A:", end="") + for line in responses.iter_lines(): + if line: + line = json.loads(line.decode('utf-8')) + if line["choices"][-1]["finish_reason"] != "stop": + print(line["choices"][-1]["delta"]["content"], end="") + sys.stdout.flush() + else: + print('Error:', responses.text) + + + + + except KeyboardInterrupt: + # 捕获 Ctrl-C 信号，关闭会话 + session.close() + + print("\n") + print("============================") + print("程序正在退出......") + print("============================") + break diff --git a/rkllm-runtime/examples/rkllm_server_demo/chat_api_gradio.py b/rkllm-runtime/examples/rkllm_server_demo/chat_api_gradio.py new file mode 100644 index 0000000..1b8fdfb --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/chat_api_gradio.py @@ -0,0 +1,44 @@ +from gradio_client import Client + +# 该函数通过调用Gradio Client API与RKLLM模型进行交互 +def chat_with_rkllm(user_message, history=[]): + # 实例化Gradio Client，用户需要根据自己部署的具体网址进行修改 + client = Client("http://172.16.10.102:8080/") + + # 调用Gradio Client API进行交互，内部的API主要包括： + # /get_user_input：模型获取用户输入，并将输入添加至历史记录history + # /get_RKLLM_output：RKLLM利用已包含输入的历史记录history生成回复 + _, history = client.predict(user_message=user_message, history=history, api_name="/get_user_input") + result_history = client.predict(history=history, api_name="/get_RKLLM_output") + return result_history + +if __name__ == '__main__': + #初始化聊天记录 + result_history = [] + + print("============================") + print("在终端中输入您的问题，即可与 RKLLM 模型进行对话....") + print("============================") + # 进入循环，持续获取用户输入，并与RKLLM模型进行对话 + while True: + try: + user_message = input("请输入您的问题：") + if user_message == "exit": + print("============================") + print("程序正在退出......") + print("============================") + break + else: + # 调用chat_with_rkllm函数，获取模型的回复 + result_history = chat_with_rkllm(user_message, result_history) + + # 打印模型输出 + print("Q:", result_history[-1][0]) + print("A:", result_history[-1][1]) + except KeyboardInterrupt: + print("\n") + print("============================") + print("程序正在退出......") + print("============================") + break + \ No newline at end of file diff --git a/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3576.sh b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3576.sh new file mode 100644 index 0000000..15deba6 --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3576.sh @@ -0,0 +1,9 @@ +#!/system/bin/sh + +echo userspace > /sys/class/devfreq/27700000.npu/governor +echo 1000000000 > /sys/class/devfreq/27700000.npu/userspace/set_freq + +echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +echo 2208000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed +echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor +echo 2304000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed diff --git a/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3588.sh b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3588.sh new file mode 100644 index 0000000..89b9e1f --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/fix_freq_rk3588.sh @@ -0,0 +1,15 @@ +echo userspace > /sys/class/devfreq/fdab0000.npu/governor +echo 1000000000 > /sys/class/devfreq/fdab0000.npu/userspace/set_freq + +echo userspace > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +echo 1800000 > /sys/devices/system/cpu/cpufreq/policy0/scaling_setspeed +echo userspace > /sys/devices/system/cpu/cpufreq/policy4/scaling_governor +echo 2400000 > /sys/devices/system/cpu/cpufreq/policy4/scaling_setspeed +echo userspace > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor +echo 2400000 > /sys/devices/system/cpu/cpufreq/policy6/scaling_setspeed + +echo userspace > /sys/class/devfreq/dmc/governor +echo 2112000000 > /sys/class/devfreq/dmc/userspace/set_freq + +echo userspace > /sys/class/devfreq/fb000000.gpu/governor +echo 1000000000 > /sys/class/devfreq/fb000000.gpu/userspace/set_freq diff --git a/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/flask_server.py b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/flask_server.py new file mode 100644 index 0000000..2ceab37 --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/flask_server.py @@ -0,0 +1,297 @@ +import ctypes +import sys +import os +import subprocess +import resource +import threading +import time +import argparse +import json +from flask import Flask, request, jsonify, Response + +app = Flask(__name__) + +# 创建一个锁，用于控制多人访问Server +lock = threading.Lock() + +# 创建一个全局变量，用于标识服务器当前是否处于阻塞状态 +is_blocking = False + +# 设置动态库路径 +rkllm_lib = ctypes.CDLL('lib/librkllmrt.so') + +# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出 +global_text = [] +global_state = -1 +split_byte_data = bytes(b"") # 用于保存分割的字节数据 + +# 定义动态库中的结构体 +class Token(ctypes.Structure): + _fields_ = [ + ("logprob", ctypes.c_float), + ("id", ctypes.c_int32) + ] + +class RKLLMResult(ctypes.Structure): + _fields_ = [ + ("text", ctypes.c_char_p), + ("tokens", ctypes.POINTER(Token)), + ("num", ctypes.c_int32) + ] + + +# 定义回调函数 +def callback(result, userdata, state): + global global_text, global_state, split_byte_data + if state == 0: + # 保存输出的token文本及RKLLM运行状态 + global_state = state + # 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析 + try: + global_text.append((split_byte_data + result.contents.text).decode('utf-8')) + print((split_byte_data + result.contents.text).decode('utf-8'), end='') + split_byte_data = bytes(b"") + except: + split_byte_data += result.contents.text + sys.stdout.flush() + elif state == 1: + # 保存RKLLM运行状态 + global_state = state + print("\n") + sys.stdout.flush() + else: + print("run error") + +# Python端与C++端的回调函数连接 +callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int) +c_callback = callback_type(callback) + +# 定义动态库中的结构体 +class RKNNllmParam(ctypes.Structure): + _fields_ = [ + ("model_path", ctypes.c_char_p), + ("num_npu_core", ctypes.c_int32), + ("max_context_len", ctypes.c_int32), + ("max_new_tokens", ctypes.c_int32), + ("top_k", ctypes.c_int32), + ("top_p", ctypes.c_float), + ("temperature", ctypes.c_float), + ("repeat_penalty", ctypes.c_float), + ("frequency_penalty", ctypes.c_float), + ("presence_penalty", ctypes.c_float), + ("mirostat", ctypes.c_int32), + ("mirostat_tau", ctypes.c_float), + ("mirostat_eta", ctypes.c_float), + ("logprobs", ctypes.c_bool), + ("top_logprobs", ctypes.c_int32), + ("use_gpu", ctypes.c_bool) + ] + +# 定义RKLLM_Handle_t和userdata +RKLLM_Handle_t = ctypes.c_void_p +userdata = ctypes.c_void_p(None) + +# 设置提示文本 +PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user" +PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant" + +# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作 +class RKLLM(object): + def __init__(self, model_path, target_platform): + rknnllm_param = RKNNllmParam() + rknnllm_param.model_path = bytes(model_path, 'utf-8') + if target_platform == "rk3588": + rknnllm_param.num_npu_core = 3 + elif target_platform == "rk3576": + rknnllm_param.num_npu_core = 1 + rknnllm_param.max_context_len = 320 + rknnllm_param.max_new_tokens = 512 + rknnllm_param.top_k = 1 + rknnllm_param.top_p = 0.9 + rknnllm_param.temperature = 0.8 + rknnllm_param.repeat_penalty = 1.1 + rknnllm_param.frequency_penalty = 0.0 + rknnllm_param.presence_penalty = 0.0 + rknnllm_param.mirostat = 0 + rknnllm_param.mirostat_tau = 5.0 + rknnllm_param.mirostat_eta = 0.1 + rknnllm_param.logprobs = False + rknnllm_param.top_logprobs = 5 + rknnllm_param.use_gpu = True + self.handle = RKLLM_Handle_t() + + self.rkllm_init = rkllm_lib.rkllm_init + self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type] + self.rkllm_init.restype = ctypes.c_int + self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback) + + self.rkllm_run = rkllm_lib.rkllm_run + self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p] + self.rkllm_run.restype = ctypes.c_int + + self.rkllm_destroy = rkllm_lib.rkllm_destroy + self.rkllm_destroy.argtypes = [RKLLM_Handle_t] + self.rkllm_destroy.restype = ctypes.c_int + + def run(self, prompt): + prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8') + self.rkllm_run(self.handle, prompt, ctypes.byref(userdata)) + return + + def release(self): + self.rkllm_destroy(self.handle) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;') + parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径') + args = parser.parse_args() + + if not (args.target_platform in ["rk3588", "rk3576"]): + print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======") + sys.stdout.flush() + exit() + + if not os.path.exists(args.rkllm_model_path): + print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======") + sys.stdout.flush() + exit() + + # 定频设置 + command = "sudo bash fix_freq_{}.sh".format(args.target_platform) + subprocess.run(command, shell=True) + + # 设置文件描述符限制 + resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400)) + + # 初始化RKLLM模型 + print("=========init....===========") + sys.stdout.flush() + target_platform = args.target_platform + model_path = args.rkllm_model_path + rkllm_model = RKLLM(model_path, target_platform) + print("RKLLM初始化成功！") + print("==============================") + sys.stdout.flush() + + # 创建一个函数用于接受用户使用 request 发送的数据 + @app.route('/rkllm_chat', methods=['POST']) + def receive_message(): + # 链接全局变量，获取回调函数的输出信息 + global global_text, global_state + global is_blocking + + # 如果服务器正在阻塞状态，则返回特定响应 + if is_blocking or global_state==0: + return jsonify({'status': 'error', 'message': 'RKLLM_Server is busy! Maybe you can try again later.'}), 503 + + # 加锁 + lock.acquire() + try: + # 设置服务器为阻塞状态 + is_blocking = True + + # 获取 POST 请求中的 JSON 数据 + data = request.json + if data and 'messages' in data: + # 重置全局变量 + global_text = [] + global_state = -1 + + # 定义返回的结构体 + rkllm_responses = { + "id": "rkllm_chat", + "object": "rkllm_chat", + "created": None, + "choices": [], + "usage": { + "prompt_tokens": None, + "completion_tokens": None, + "total_tokens": None + } + } + + if not "stream" in data.keys() or data["stream"] == False: + # 在这里处理收到的数据 + messages = data['messages'] + print("Received messages:", messages) + for index, message in enumerate(messages): + input_prompt = message['content'] + rkllm_output = "" + + # 创建模型推理的线程 + model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,)) + model_thread.start() + + # 等待模型运行完成，定时检查模型的推理线程 + model_thread_finished = False + while not model_thread_finished: + while len(global_text) > 0: + rkllm_output += global_text.pop(0) + time.sleep(0.005) + + model_thread.join(timeout=0.005) + model_thread_finished = not model_thread.is_alive() + + rkllm_responses["choices"].append( + {"index": index, + "message": { + "role": "assistant", + "content": rkllm_output, + }, + "logprobs": None, + "finish_reason": "stop" + } + ) + return jsonify(rkllm_responses), 200 + else: + # 在这里处理收到的数据 + messages = data['messages'] + print("Received messages:", messages) + for index, message in enumerate(messages): + input_prompt = message['content'] + rkllm_output = "" + + def generate(): + # 创建模型推理的线程 + model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,)) + model_thread.start() + + # 等待模型运行完成，定时检查模型的推理线程 + model_thread_finished = False + while not model_thread_finished: + while len(global_text) > 0: + rkllm_output = global_text.pop(0) + + rkllm_responses["choices"].append( + {"index": index, + "delta": { + "role": "assistant", + "content": rkllm_output, + }, + "logprobs": None, + "finish_reason": "stop" if global_state == 1 else None, + } + ) + yield f"{json.dumps(rkllm_responses)}\n\n" + + model_thread.join(timeout=0.005) + model_thread_finished = not model_thread.is_alive() + + return Response(generate(), content_type='text/plain') + else: + return jsonify({'status': 'error', 'message': 'Invalid JSON data!'}), 400 + finally: + # 释放锁 + lock.release() + # 将服务器状态设置为非阻塞 + is_blocking = False + + # 启动 Flask 应用程序 + # app.run(host='0.0.0.0', port=8080) + app.run(host='0.0.0.0', port=8080, threaded=True, debug=False) + + print("====================") + print("RKLLM模型推理结束, 释放RKLLM模型资源...") + rkllm_model.release() + print("====================") diff --git a/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/gradio_server.py b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/gradio_server.py new file mode 100644 index 0000000..9e56446 --- /dev/null +++ b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/gradio_server.py @@ -0,0 +1,228 @@ +import ctypes +import sys +import os +import subprocess +import resource +import threading +import time +import gradio as gr +import argparse + +# 设定环境变量 +os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0" +os.environ["GRADIO_SERVER_PORT"] = "8080" + +# 设置动态库路径 +rkllm_lib = ctypes.CDLL('lib/librkllmrt.so') + +# 定义全局变量，用于保存回调函数的输出，便于在gradio界面中输出 +global_text = [] +global_state = -1 +split_byte_data = bytes(b"") # 用于保存分割的字节数据 + +# 定义动态库中的结构体 +class Token(ctypes.Structure): + _fields_ = [ + ("logprob", ctypes.c_float), + ("id", ctypes.c_int32) + ] + +class RKLLMResult(ctypes.Structure): + _fields_ = [ + ("text", ctypes.c_char_p), + ("tokens", ctypes.POINTER(Token)), + ("num", ctypes.c_int32) + ] + +# 定义回调函数 +def callback(result, userdata, state): + global global_text, global_state, split_byte_data + if state == 0: + # 保存输出的token文本及RKLLM运行状态 + global_state = state + # 需要监控当前的字节数据是否完整，不完整则进行记录，后续进行解析 + try: + global_text.append((split_byte_data + result.contents.text).decode('utf-8')) + print((split_byte_data + result.contents.text).decode('utf-8'), end='') + split_byte_data = bytes(b"") + except: + split_byte_data += result.contents.text + sys.stdout.flush() + elif state == 1: + # 保存RKLLM运行状态 + global_state = state + print("\n") + sys.stdout.flush() + else: + print("run error") + +# Python端与C++端的回调函数连接 +callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int) +c_callback = callback_type(callback) + +# 定义动态库中的结构体 +class RKNNllmParam(ctypes.Structure): + _fields_ = [ + ("model_path", ctypes.c_char_p), + ("num_npu_core", ctypes.c_int32), + ("max_context_len", ctypes.c_int32), + ("max_new_tokens", ctypes.c_int32), + ("top_k", ctypes.c_int32), + ("top_p", ctypes.c_float), + ("temperature", ctypes.c_float), + ("repeat_penalty", ctypes.c_float), + ("frequency_penalty", ctypes.c_float), + ("presence_penalty", ctypes.c_float), + ("mirostat", ctypes.c_int32), + ("mirostat_tau", ctypes.c_float), + ("mirostat_eta", ctypes.c_float), + ("logprobs", ctypes.c_bool), + ("top_logprobs", ctypes.c_int32), + ("use_gpu", ctypes.c_bool) + ] + +# 定义RKLLM_Handle_t和userdata +RKLLM_Handle_t = ctypes.c_void_p +userdata = ctypes.c_void_p(None) + +# 设置提示文本 +PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user" +PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant" + +# 定义Python端的RKLLM类，其中包括了对动态库中RKLLM模型的初始化、推理及释放操作 +class RKLLM(object): + def __init__(self, model_path, target_platform): + rknnllm_param = RKNNllmParam() + rknnllm_param.model_path = bytes(model_path, 'utf-8') + if target_platform == "rk3588": + rknnllm_param.num_npu_core = 3 + elif target_platform == "rk3576": + rknnllm_param.num_npu_core = 2 + rknnllm_param.max_context_len = 320 + rknnllm_param.max_new_tokens = 512 + rknnllm_param.top_k = 1 + rknnllm_param.top_p = 0.9 + rknnllm_param.temperature = 0.8 + rknnllm_param.repeat_penalty = 1.1 + rknnllm_param.frequency_penalty = 0.0 + rknnllm_param.presence_penalty = 0.0 + rknnllm_param.mirostat = 0 + rknnllm_param.mirostat_tau = 5.0 + rknnllm_param.mirostat_eta = 0.1 + rknnllm_param.logprobs = False + rknnllm_param.top_logprobs = 5 + rknnllm_param.use_gpu = True + self.handle = RKLLM_Handle_t() + + self.rkllm_init = rkllm_lib.rkllm_init + self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type] + self.rkllm_init.restype = ctypes.c_int + self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback) + + self.rkllm_run = rkllm_lib.rkllm_run + self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p] + self.rkllm_run.restype = ctypes.c_int + + self.rkllm_destroy = rkllm_lib.rkllm_destroy + self.rkllm_destroy.argtypes = [RKLLM_Handle_t] + self.rkllm_destroy.restype = ctypes.c_int + + def run(self, prompt): + prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8') + self.rkllm_run(self.handle, prompt, ctypes.byref(userdata)) + return + + def release(self): + self.rkllm_destroy(self.handle) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;') + parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径') + args = parser.parse_args() + + if not (args.target_platform in ["rk3588", "rk3576"]): + print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======") + sys.stdout.flush() + exit() + + if not os.path.exists(args.rkllm_model_path): + print("====== Error: 请给出准确的rkllm模型路径，需注意是板端的绝对路径 ======") + sys.stdout.flush() + exit() + + # 定频设置 + command = "sudo bash fix_freq_{}.sh".format(args.target_platform) + subprocess.run(command, shell=True) + + # 设置文件描述符限制 + resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400)) + + # 初始化RKLLM模型 + print("=========init....===========") + sys.stdout.flush() + target_platform = args.target_platform + model_path = args.rkllm_model_path + rkllm_model = RKLLM(model_path, target_platform) + print("RKLLM初始化成功！") + print("==============================") + sys.stdout.flush() + + # 记录用户输入的prompt + def get_user_input(user_message, history): + history = history + [[user_message, None]] + return "", history + + # 获取RKLLM模型的输出并进行流式打印 + def get_RKLLM_output(history): + # 链接全局变量，获取回调函数的输出信息 + global global_text, global_state + global_text = [] + global_state = -1 + + # 创建模型推理的线程 + model_thread = threading.Thread(target=rkllm_model.run, args=(history[-1][0],)) + model_thread.start() + + # history[-1][1]表示当前的输出对话 + history[-1][1] = "" + + # 等待模型运行完成，定时检查模型的推理线程 + model_thread_finished = False + while not model_thread_finished: + while len(global_text) > 0: + history[-1][1] += global_text.pop(0) + time.sleep(0.005) + # gradio在调用then方法式自动将yield返回的结果推进行输出 + yield history + + model_thread.join(timeout=0.005) + model_thread_finished = not model_thread.is_alive() + + # 创建gradio界面 + with gr.Blocks(title="Chat with RKLLM") as chatRKLLM: + gr.Markdown("
Chat with RKLLM
") + gr.Markdown("### 在 inputTextBox 输入您的问题，按下 Enter 键，即可与 RKLLM 模型进行对话。") + # 创建一个Chatbot组件，用于显示对话历史 + rkllmServer = gr.Chatbot(height=600) + # 创建一个Textbox组件，让用户输入消息 + msg = gr.Textbox(placeholder="Please input your question here...", label="inputTextBox") + # 创建一个Button组件，用于清除聊天历史 + clear = gr.Button("清除") + + # 将用户输入的消息提交给get_user_input函数，并且立即更新聊天历史 + # 然后调用get_RKLLM_output函数，进一步更新聊天历史 + # queue=False参数确保这些更新不会被放入队列，而是立即执行 + msg.submit(get_user_input, [msg, rkllmServer], [msg, rkllmServer], queue=False).then(get_RKLLM_output, rkllmServer, rkllmServer) + # 当点击清除按钮时，执行一个空操作（lambda: None），并且立即清除聊天历史 + clear.click(lambda: None, None, rkllmServer, queue=False) + + # 启用事件队列系统 + chatRKLLM.queue() + # 启动Gradio应用程序 + chatRKLLM.launch() + + print("====================") + print("RKLLM模型推理结束, 释放RKLLM模型资源...") + rkllm_model.release() + print("====================") \ No newline at end of file diff --git a/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/lib/.gitkeep b/rkllm-runtime/examples/rkllm_server_demo/rkllm_server/lib/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/rkllm-runtime/runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so b/rkllm-runtime/runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so index 0b93495..ba8f076 100755 Binary files a/rkllm-runtime/runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so and b/rkllm-runtime/runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so differ diff --git a/rkllm-runtime/runtime/Android/librkllm_api/include/rkllm.h b/rkllm-runtime/runtime/Android/librkllm_api/include/rkllm.h index 0321e29..ce1b2ff 100644 --- a/rkllm-runtime/runtime/Android/librkllm_api/include/rkllm.h +++ b/rkllm-runtime/runtime/Android/librkllm_api/include/rkllm.h @@ -5,45 +5,115 @@ extern "C" { #endif -typedef void* LLMHandle; +typedef void* LLMHandle; /* Handle for an instance of a language model. */ +/** + * @brief Structure for possible states of an inference call. + * + */ typedef enum { - LLM_RUN_NORMAL = 0, /*推理状态正常，推理尚未结束*/ - LLM_RUN_FINISH = 1, /*推理状态正常，推理结束*/ - LLM_RUN_ERROR = 2 /*推理状态异常*/ + LLM_RUN_NORMAL = 0, /* Inference status is normal and inference has not yet finished. */ + LLM_RUN_FINISH = 1, /* Inference status is normal and inference has finished. */ + LLM_RUN_ERROR = 2 /* Inference status is abnormal. */ } LLMCallState; +/** + * @brief Structure for setting up parameters for the language model + * + */ typedef struct { - const char* modelPath; /*模型文件的存放路径*/ - const char* target_platform; /*模型运行的硬件平台*/ - int32_t num_npu_core; /*模型推理时使用的 NPU 核心数量*/ - int32_t max_context_len; /*设置提示上下文的大小*/ - int32_t max_new_tokens; /*用于设置模型推理时生成 Token 的数量上限*/ - - int32_t top_k; - float top_p; - float temperature; - float repeat_penalty; - float frequency_penalty; - float presence_penalty; - int32_t mirostat; - float mirostat_tau; - float mirostat_eta; - + const char* model_path; /* Path where the model file is located. */ + int32_t num_npu_core; /* Number of NPU cores used for model inference. */ + int32_t max_context_len; /* Maximum size of the context. */ + int32_t max_new_tokens; /* Maximum number of tokens to generate during model inference. */ + int32_t top_k; /* The number of highest probability tokens to consider for generation. */ + float top_p; /* Nucleus sampling: cumulative probability cutoff to use for token selection. */ + float temperature; /* Hyperparameter to control the randomness of predictions by scaling the logits before applying softmax. */ + float repeat_penalty; /* Penalty applied to the logits of previously generated tokens, helps prevent repetitive or monotonic text. */ + float frequency_penalty; /* Penalty for repeating the same word or phrase, reducing the likelihood of repeated content. */ + float presence_penalty; /* Penalty or reward for introducing new tokens into the generated text. */ + int32_t mirostat; /* Enables mirostat algorithm, where 0 = off, 1 = use mirostat algorithm, 2 = use mirostat 2.0 algorithm. */ + float mirostat_tau; /* Target entropy (perplexity) for mirostat algorithm, setting the desired complexity of the generated text. */ + float mirostat_eta; /* Learning rate for the mirostat algorithm. */ + bool logprobs; /* Whether to return the log probabilities for each output token along with their token ids. */ + int32_t top_logprobs; /* The number of top tokens for which to return log probabilities, along with their token ids. */ + bool use_gpu; /* Flag to indicate whether to use GPU for inference. */ } RKLLMParam; -typedef void(*LLMResultCallback)(const char* result, void* userdata, LLMCallState state); +/** + * @brief Structure representing a token with its associated log probability. + * + */ +typedef struct { + float logprob; /* Log probability corresponding to the token ID. */ + int id; /* Token ID. */ +} Token; -RKLLMParam rkllm_createDefaultParam(); /*初始化RKLLMParam并设置默认参数*/ +/** + * @brief Structure to hold the results from the language model inference, including text and token details. + * + */ +typedef struct { + const char* text; /* Decoded text from the inference output. */ + Token* tokens; /* Array of Token structures, each containing a log probability and a token ID. */ + int num; /* Number of top tokens returned, typically those with the highest probabilities. */ +} RKLLMResult; -int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback); /*模型初始化*/ +/** + * @brief Callback function for handling inference results. + * + * @param result A pointer to an RKLLMResult struct containing the inference results. + * @param userdata A pointer to user-defined function or null if no user function was provided. + * @param state The state of the inference process, indicating success, failure, or completion. + */ +typedef void(*LLMResultCallback)(RKLLMResult* result, void* userdata, LLMCallState state); -int rkllm_run(LLMHandle handle, const char* prompt, void* userdata); /*模型推理*/ +/** + * @brief Initializes RKLLMParam with default settings. + * + * @return RKLLMParam An RKLLMParam struct with default values set. + */ +RKLLMParam rkllm_createDefaultParam(); -int rkllm_destroy(LLMHandle handle); /*模型释放*/ +/** + * @brief Initializes the model with specified parameters. + * + * @param handle Pointer to a handle for the language model, which will be initialized by this function. + * @param param An RKLLMParam struct containing all the parameters needed for the model. + * @param callback A function pointer to the callback that handles the results of the inference. + * @return int Returns 0 on success, or a negative error code on failure. + */ +int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback); + +/** + * @brief Releases the model resources. + * + * @param handle The handle to the language model to be destroyed. + * @return int Returns 0 on successful release, or a negative error code if an error occurs. + */ +int rkllm_destroy(LLMHandle handle); + +/** + * @brief Runs model inference on the given prompt. + * + * @param handle The handle to the initialized language model. + * @param prompt The text prompt on which to perform inference. + * @param userdata Optional user-defined function that will be passed to the callback. + * @return int Returns 0 on success, or a negative error code if an error occurs during inference. + */ +int rkllm_run(LLMHandle handle, const char* prompt, void* userdata); + +/** + * @brief Aborts the current inference process. + * + * @param handle The handle to the language model whose inference is to be aborted. + * @return int Returns 0 if the process is successfully aborted, or a negative error code + * if no process was running or if the abort fails. + */ +int rkllm_abort(LLMHandle handle); #ifdef __cplusplus -} +} //extern "C" #endif #endif \ No newline at end of file diff --git a/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/librkllmrt.so b/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/librkllmrt.so index 53e0099..a026cbe 100755 Binary files a/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/librkllmrt.so and b/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/librkllmrt.so differ diff --git a/rkllm-runtime/runtime/Linux/librkllm_api/include/rkllm.h b/rkllm-runtime/runtime/Linux/librkllm_api/include/rkllm.h index 0321e29..ce1b2ff 100644 --- a/rkllm-runtime/runtime/Linux/librkllm_api/include/rkllm.h +++ b/rkllm-runtime/runtime/Linux/librkllm_api/include/rkllm.h @@ -5,45 +5,115 @@ extern "C" { #endif -typedef void* LLMHandle; +typedef void* LLMHandle; /* Handle for an instance of a language model. */ +/** + * @brief Structure for possible states of an inference call. + * + */ typedef enum { - LLM_RUN_NORMAL = 0, /*推理状态正常，推理尚未结束*/ - LLM_RUN_FINISH = 1, /*推理状态正常，推理结束*/ - LLM_RUN_ERROR = 2 /*推理状态异常*/ + LLM_RUN_NORMAL = 0, /* Inference status is normal and inference has not yet finished. */ + LLM_RUN_FINISH = 1, /* Inference status is normal and inference has finished. */ + LLM_RUN_ERROR = 2 /* Inference status is abnormal. */ } LLMCallState; +/** + * @brief Structure for setting up parameters for the language model + * + */ typedef struct { - const char* modelPath; /*模型文件的存放路径*/ - const char* target_platform; /*模型运行的硬件平台*/ - int32_t num_npu_core; /*模型推理时使用的 NPU 核心数量*/ - int32_t max_context_len; /*设置提示上下文的大小*/ - int32_t max_new_tokens; /*用于设置模型推理时生成 Token 的数量上限*/ - - int32_t top_k; - float top_p; - float temperature; - float repeat_penalty; - float frequency_penalty; - float presence_penalty; - int32_t mirostat; - float mirostat_tau; - float mirostat_eta; - + const char* model_path; /* Path where the model file is located. */ + int32_t num_npu_core; /* Number of NPU cores used for model inference. */ + int32_t max_context_len; /* Maximum size of the context. */ + int32_t max_new_tokens; /* Maximum number of tokens to generate during model inference. */ + int32_t top_k; /* The number of highest probability tokens to consider for generation. */ + float top_p; /* Nucleus sampling: cumulative probability cutoff to use for token selection. */ + float temperature; /* Hyperparameter to control the randomness of predictions by scaling the logits before applying softmax. */ + float repeat_penalty; /* Penalty applied to the logits of previously generated tokens, helps prevent repetitive or monotonic text. */ + float frequency_penalty; /* Penalty for repeating the same word or phrase, reducing the likelihood of repeated content. */ + float presence_penalty; /* Penalty or reward for introducing new tokens into the generated text. */ + int32_t mirostat; /* Enables mirostat algorithm, where 0 = off, 1 = use mirostat algorithm, 2 = use mirostat 2.0 algorithm. */ + float mirostat_tau; /* Target entropy (perplexity) for mirostat algorithm, setting the desired complexity of the generated text. */ + float mirostat_eta; /* Learning rate for the mirostat algorithm. */ + bool logprobs; /* Whether to return the log probabilities for each output token along with their token ids. */ + int32_t top_logprobs; /* The number of top tokens for which to return log probabilities, along with their token ids. */ + bool use_gpu; /* Flag to indicate whether to use GPU for inference. */ } RKLLMParam; -typedef void(*LLMResultCallback)(const char* result, void* userdata, LLMCallState state); +/** + * @brief Structure representing a token with its associated log probability. + * + */ +typedef struct { + float logprob; /* Log probability corresponding to the token ID. */ + int id; /* Token ID. */ +} Token; -RKLLMParam rkllm_createDefaultParam(); /*初始化RKLLMParam并设置默认参数*/ +/** + * @brief Structure to hold the results from the language model inference, including text and token details. + * + */ +typedef struct { + const char* text; /* Decoded text from the inference output. */ + Token* tokens; /* Array of Token structures, each containing a log probability and a token ID. */ + int num; /* Number of top tokens returned, typically those with the highest probabilities. */ +} RKLLMResult; -int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback); /*模型初始化*/ +/** + * @brief Callback function for handling inference results. + * + * @param result A pointer to an RKLLMResult struct containing the inference results. + * @param userdata A pointer to user-defined function or null if no user function was provided. + * @param state The state of the inference process, indicating success, failure, or completion. + */ +typedef void(*LLMResultCallback)(RKLLMResult* result, void* userdata, LLMCallState state); -int rkllm_run(LLMHandle handle, const char* prompt, void* userdata); /*模型推理*/ +/** + * @brief Initializes RKLLMParam with default settings. + * + * @return RKLLMParam An RKLLMParam struct with default values set. + */ +RKLLMParam rkllm_createDefaultParam(); -int rkllm_destroy(LLMHandle handle); /*模型释放*/ +/** + * @brief Initializes the model with specified parameters. + * + * @param handle Pointer to a handle for the language model, which will be initialized by this function. + * @param param An RKLLMParam struct containing all the parameters needed for the model. + * @param callback A function pointer to the callback that handles the results of the inference. + * @return int Returns 0 on success, or a negative error code on failure. + */ +int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback); + +/** + * @brief Releases the model resources. + * + * @param handle The handle to the language model to be destroyed. + * @return int Returns 0 on successful release, or a negative error code if an error occurs. + */ +int rkllm_destroy(LLMHandle handle); + +/** + * @brief Runs model inference on the given prompt. + * + * @param handle The handle to the initialized language model. + * @param prompt The text prompt on which to perform inference. + * @param userdata Optional user-defined function that will be passed to the callback. + * @return int Returns 0 on success, or a negative error code if an error occurs during inference. + */ +int rkllm_run(LLMHandle handle, const char* prompt, void* userdata); + +/** + * @brief Aborts the current inference process. + * + * @param handle The handle to the language model whose inference is to be aborted. + * @return int Returns 0 if the process is successfully aborted, or a negative error code + * if no process was running or if the abort fails. + */ +int rkllm_abort(LLMHandle handle); #ifdef __cplusplus -} +} //extern "C" #endif #endif \ No newline at end of file diff --git a/rkllm-toolkit/packages/md5sum.txt b/rkllm-toolkit/packages/md5sum.txt index 125ed2d..d115b4f 100644 --- a/rkllm-toolkit/packages/md5sum.txt +++ b/rkllm-toolkit/packages/md5sum.txt @@ -1 +1 @@ -0bd11fa988ada8949fa4c1c1daf465d1 rkllm_toolkit-1.0.0-cp38-cp38-linux_x86_64.whl +cf82f9756844793bbcd132aae58d0b81 rkllm_toolkit-1.0.1-cp38-cp38-linux_x86_64.whl diff --git a/rkllm-toolkit/packages/rkllm_toolkit-1.0.0-cp38-cp38-linux_x86_64.whl b/rkllm-toolkit/packages/rkllm_toolkit-1.0.0-cp38-cp38-linux_x86_64.whl deleted file mode 100644 index 87e3b71..0000000 Binary files a/rkllm-toolkit/packages/rkllm_toolkit-1.0.0-cp38-cp38-linux_x86_64.whl and /dev/null differ diff --git a/rkllm-toolkit/packages/rkllm_toolkit-1.0.1-cp38-cp38-linux_x86_64.whl b/rkllm-toolkit/packages/rkllm_toolkit-1.0.1-cp38-cp38-linux_x86_64.whl new file mode 100644 index 0000000..0f6cd35 Binary files /dev/null and b/rkllm-toolkit/packages/rkllm_toolkit-1.0.1-cp38-cp38-linux_x86_64.whl differ