release v1.0.0BETA

README.md

@@ -0,0 +1,39 @@
+# Description
+  RKLLM software stack can help users to quickly deploy AI models to Rockchip chips. The overall framework is as follows:
+    <center class="half">
+        <div style="background-color:#ffffff;">
+        <img src="res/framework.jpg" title="RKLLM"/>
+    </center>
+
+  In order to use RKNPU, users need to first run the RKLLM-Toolkit tool on the computer, convert the trained model into an RKLLM format model, and then inference on the development board using the RKLLM C API.
+
+- RKLLM-Toolkit is a software development kit for users to perform model conversionand quantization on PC.
+
+- RKLLM Runtime provides C/C++ programming interfaces for Rockchip NPU platform to help users deploy RKLLM models and accelerate the implementation of LLM applications.
+
+- RKNPU kernel driver is responsible for interacting with NPU hardware. It has been open source and can be found in the Rockchip kernel code.
+
+# Support Platform
+  - RK3588 Series
+  - RK3576 Series
+
+# Download
+- You can also download all packages, docker image, examples, docs and platform-tools from [RKLLM_SDK](https://console.zbox.filez.com/l/RJJDmB), fetch code: rkllm
+
+# RKNN Toolkit2
+If you want to deploy additional AI model, we have introduced a new SDK called RKNN-Toolkit2. For details, please refer to:
+
+https://github.com/airockchip/rknn-toolkit2
+
+# Notes
+
+Due to recent updates to the Phi2 model, the current version of the RKLLM SDK does not yet support these changes. 
+Please ensure to download a version of the [Phi2](https://hf-mirror.com/microsoft/phi-2/tree/834565c23f9b28b96ccbeabe614dd906b6db551a) model that is supported. 
+
+# CHANGELOG
+
+## v1.0.0-beta
+ - Supports the conversion and deployment of LLM models on RK3588/RK3576 platforms
+ - Compatible with Hugging Face model architectures
+ - Currently supports the models LLaMA, Qwen, Qwen2, and Phi-2
+ - Supports quantization with w8a8 and w4a16 precision

rkllm-runtime/example/CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.10)
+project(llm_demo)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(SOURCE_FILES src/main.cpp)
+
+add_executable(${PROJECT_NAME} ${SOURCE_FILES})
+
+set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
+include_directories(${RKLLM_API_PATH}/include)
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
+endif()
+
+
+target_link_libraries(${PROJECT_NAME}  ${RKLLM_RT_LIB})

rkllm-runtime/example/Readme.md

@@ -0,0 +1,51 @@
+# Aarch64 Linux Demo
+## Build
+Ensure the `GCC_COMPILER_PATH` option in the `build-linux.sh` script is correctly configured:
+```sh
+GCC_COMPILER_PATH=/path/to/your/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
+```
+To execute, run:
+```bash
+bash build-linux.sh
+```
+
+## Install
+Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
+```bash
+adb push build/build_linux_aarch64_Release/llm_demo /userdata/llm
+adb push ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /userdata/llm/lib
+```
+
+## Run
+```bash
+adb shell
+cd /userdata/llm
+export LD_LIBRARY_PATH=./lib
+taskset f0 ./llm_demo /path/to/your/rkllm/model
+```
+
+# Android Demo
+## Build
+Ensure the `ANDROID_NDK_PATH` option in the `build-android.sh` script is correctly configured:
+```sh
+ANDROID_NDK_PATH=/path/to/your/android-ndk-r18b
+```
+To execute, run:
+```bash
+bash build-android.sh
+```
+
+## Install
+Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
+```bash
+adb push build/build_android_arm64-v8a_Release/llm_demo /userdata/llm
+adb push ../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib
+```
+
+## Run
+```bash
+adb shell
+cd /userdata/llm
+export LD_LIBRARY_PATH=./lib
+taskset f0 ./llm_demo /path/to/your/rkllm/model
+```

rkllm-runtime/example/build-android.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Debug / Release / RelWithDebInfo
+if [[ -z ${BUILD_TYPE} ]];then
+    BUILD_TYPE=Release
+fi
+
+ANDROID_NDK_PATH=~/android-ndk-r18b
+TARGET_ARCH=arm64-v8a
+
+TARGET_PLATFORM=android
+if [[ -n ${TARGET_ARCH} ]];then
+TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
+fi
+
+ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
+BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
+
+if [[ ! -d "${BUILD_DIR}" ]]; then
+  mkdir -p ${BUILD_DIR}
+fi
+
+cd ${BUILD_DIR}
+cmake ../.. \
+    -DCMAKE_SYSTEM_NAME=Android \
+    -DCMAKE_SYSTEM_VERSION=23 \
+    -DCMAKE_ANDROID_ARCH_ABI=${TARGET_ARCH} \
+    -DCMAKE_ANDROID_STL_TYPE=c++_static \
+    -DCMAKE_ANDROID_NDK=${ANDROID_NDK_PATH} \
+    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+    -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+
+make -j4

rkllm-runtime/example/build-linux.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Debug / Release / RelWithDebInfo
+if [[ -z ${BUILD_TYPE} ]];then
+    BUILD_TYPE=Release
+fi
+
+GCC_COMPILER_PATH=~/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
+C_COMPILER=${GCC_COMPILER_PATH}-gcc
+CXX_COMPILER=${GCC_COMPILER_PATH}-g++
+STRIP_COMPILER=${GCC_COMPILER_PATH}-strip
+
+TARGET_ARCH=aarch64
+TARGET_PLATFORM=linux
+if [[ -n ${TARGET_ARCH} ]];then
+TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
+fi
+
+ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
+BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
+
+if [[ ! -d "${BUILD_DIR}" ]]; then
+  mkdir -p ${BUILD_DIR}
+fi
+
+cd ${BUILD_DIR}
+cmake ../.. \
+    -DCMAKE_SYSTEM_PROCESSOR=${TARGET_ARCH} \
+    -DCMAKE_SYSTEM_NAME=Linux \
+    -DCMAKE_C_COMPILER=${C_COMPILER} \
+    -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
+    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+
+make -j4

rkllm-runtime/example/src/main.cpp

@@ -0,0 +1,124 @@
+// Copyright (c) 2024 by Rockchip Electronics Co., Ltd. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string.h>
+#include <unistd.h>
+#include <string>
+#include "rkllm.h"
+#include <fstream>
+#include <iostream>
+#include <csignal>
+#include <vector>
+
+#define PROMPT_TEXT_PREFIX "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
+#define PROMPT_TEXT_POSTFIX "<|im_end|><|im_start|>assistant"
+
+using namespace std;
+LLMHandle llmHandle = nullptr;
+
+void exit_handler(int signal)
+{
+    if (llmHandle != nullptr)
+    {
+        {
+            cout << "程序即将退出" << endl;
+            LLMHandle _tmp = llmHandle;
+            llmHandle = nullptr;
+            rkllm_destroy(_tmp);
+        }
+        exit(signal);
+    }
+}
+
+void callback(const char *text, void *userdata, LLMCallState state)
+{
+    
+    if (state == LLM_RUN_FINISH)
+    {
+        printf("\n");
+    }
+    else if (state == LLM_RUN_ERROR)
+    {
+        printf("\\run error\n");
+    }
+    else{
+        printf("%s", text);
+    }
+}
+
+int main(int argc, char **argv)
+{
+    if (argc != 2)
+    {
+        printf("Usage:%s [rkllm_model_path]\n", argv[0]);
+        return -1;
+    }
+    signal(SIGINT, exit_handler);
+    string rkllm_model(argv[1]);
+    printf("rkllm init start\n");
+
+    //设置参数及初始化
+    RKLLMParam param = rkllm_createDefaultParam();
+    param.modelPath = rkllm_model.c_str();
+    param.target_platform = "rk3588";
+    param.num_npu_core = 2;
+    param.top_k = 1;
+    param.max_new_tokens = 256;
+    param.max_context_len = 512;
+    rkllm_init(&llmHandle, param, callback);
+    printf("rkllm init success\n");
+    
+    vector<string> pre_input;
+    pre_input.push_back("把下面的现代文翻译成文言文：到了春风和煦，阳光明媚的时候，湖面平静，没有惊涛骇浪，天色湖光相连，一片碧绿，广阔无际；沙洲上的鸥鸟，时而飞翔，时而停歇，美丽的鱼游来游去，岸上与小洲上的花草，青翠欲滴。");
+    pre_input.push_back("以咏梅为题目，帮我写一首古诗，要求包含梅花、白雪等元素。");
+    pre_input.push_back("上联: 江边惯看千帆过");
+    pre_input.push_back("把这句话翻译成中文：Knowledge can be acquired from many sources. These include books, teachers and practical experience, and each has its own advantages. The knowledge we gain from books and formal education enables us to learn about things that we have no opportunity to experience in daily life. We can also develop our analytical skills and learn how to view and interpret the world around us in different ways. Furthermore, we can learn from the past by reading books. In this way, we won't repeat the mistakes of others and can build on their achievements.");
+    pre_input.push_back("把这句话翻译成英文：RK3588是新一代高端处理器，具有高算力、低功耗、超强多媒体、丰富数据接口等特点");
+    cout << "\n**********************可输入以下问题对应序号获取回答/或自定义输入********************\n"
+         << endl;
+    for (int i = 0; i < (int)pre_input.size(); i++)
+    {
+        cout << "[" << i << "] " << pre_input[i] << endl;
+    }
+    cout << "\n*************************************************************************\n"
+         << endl;
+
+    string text;
+    while (true)
+    {
+        std::string input_str;
+        printf("\n");
+        printf("user: ");
+        std::getline(std::cin, input_str);
+        if (input_str == "exit")
+        {
+            break;
+        }
+        for (int i = 0; i < (int)pre_input.size(); i++)
+        {
+            if (input_str == to_string(i))
+            {
+                input_str = pre_input[i];
+                cout << input_str << endl;
+            }
+        }
+        string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX;
+        printf("robot: ");
+        rkllm_run(llmHandle, text.c_str(), NULL);
+    }
+
+    rkllm_destroy(llmHandle);
+
+    return 0;
+}

rkllm-runtime/runtime/Android/librkllm_api/include/rkllm.h

@@ -0,0 +1,49 @@
+#ifndef _LLM_H_
+#define _LLM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* LLMHandle;
+
+typedef enum {
+    LLM_RUN_NORMAL = 0,  /*推理状态正常，推理尚未结束*/
+    LLM_RUN_FINISH = 1,  /*推理状态正常，推理结束*/
+    LLM_RUN_ERROR = 2    /*推理状态异常*/
+} LLMCallState;
+
+typedef struct {
+    const char* modelPath;          /*模型文件的存放路径*/
+    const char* target_platform;    /*模型运行的硬件平台*/
+    int32_t num_npu_core;           /*模型推理时使用的 NPU 核心数量*/
+    int32_t max_context_len;        /*设置提示上下文的大小*/
+    int32_t max_new_tokens;         /*用于设置模型推理时生成 Token 的数量上限*/
+
+    int32_t top_k;                  
+    float top_p;
+    float temperature;
+    float repeat_penalty;
+    float frequency_penalty;
+    float presence_penalty;
+    int32_t mirostat;
+    float mirostat_tau;
+    float mirostat_eta;
+
+} RKLLMParam;
+
+typedef void(*LLMResultCallback)(const char* result, void* userdata, LLMCallState state);
+
+RKLLMParam rkllm_createDefaultParam();                                              /*初始化RKLLMParam并设置默认参数*/
+
+int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback);    /*模型初始化*/
+
+int rkllm_run(LLMHandle handle, const char* prompt, void* userdata);                /*模型推理*/
+
+int rkllm_destroy(LLMHandle handle);                                                /*模型释放*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

rkllm-runtime/runtime/Linux/librkllm_api/include/rkllm.h

@@ -0,0 +1,49 @@
+#ifndef _LLM_H_
+#define _LLM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* LLMHandle;
+
+typedef enum {
+    LLM_RUN_NORMAL = 0,  /*推理状态正常，推理尚未结束*/
+    LLM_RUN_FINISH = 1,  /*推理状态正常，推理结束*/
+    LLM_RUN_ERROR = 2    /*推理状态异常*/
+} LLMCallState;
+
+typedef struct {
+    const char* modelPath;          /*模型文件的存放路径*/
+    const char* target_platform;    /*模型运行的硬件平台*/
+    int32_t num_npu_core;           /*模型推理时使用的 NPU 核心数量*/
+    int32_t max_context_len;        /*设置提示上下文的大小*/
+    int32_t max_new_tokens;         /*用于设置模型推理时生成 Token 的数量上限*/
+
+    int32_t top_k;                  
+    float top_p;
+    float temperature;
+    float repeat_penalty;
+    float frequency_penalty;
+    float presence_penalty;
+    int32_t mirostat;
+    float mirostat_tau;
+    float mirostat_eta;
+
+} RKLLMParam;
+
+typedef void(*LLMResultCallback)(const char* result, void* userdata, LLMCallState state);
+
+RKLLMParam rkllm_createDefaultParam();                                              /*初始化RKLLMParam并设置默认参数*/
+
+int rkllm_init(LLMHandle* handle, RKLLMParam param, LLMResultCallback callback);    /*模型初始化*/
+
+int rkllm_run(LLMHandle handle, const char* prompt, void* userdata);                /*模型推理*/
+
+int rkllm_destroy(LLMHandle handle);                                                /*模型释放*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

rkllm-toolkit/examples/huggingface/test.py

@@ -0,0 +1,29 @@
+from rkllm.api import RKLLM
+
+'''
+https://huggingface.co/Qwen/Qwen-1_8B-Chat
+Download the Qwen model from the above website.
+'''
+
+modelpath = '/path/to/your/model'
+llm = RKLLM()
+
+# Load model
+ret = llm.load_huggingface(model = modelpath)
+if ret != 0:
+    print('Load model failed!')
+    exit(ret)
+
+# Build model
+ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8', target_platform='rk3588')
+if ret != 0:
+    print('Build model failed!')
+    exit(ret)
+
+# Export rknn model
+ret = llm.export_rkllm("./qwen.rkllm")
+if ret != 0:
+    print('Export model failed!')
+    exit(ret)
+
+

rkllm-toolkit/packages/md5sum.txt

@@ -0,0 +1 @@
+0bd11fa988ada8949fa4c1c1daf465d1 rkllm_toolkit-1.0.0-cp38-cp38-linux_x86_64.whl