Updated to release 1.0.1

2026-11-15 20:43:52 +07:00 · 2024-05-09 16:55:24 +02:00
parent 973a16e14c 4a726e2667
commit de1f13006f
27 changed files with 1146 additions and 73 deletions
--- a/rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt
+++ b/rkllm-runtime/examples/rkllm_api_demo/CMakeLists.txt
@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.10)
+project(llm_demo)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(SOURCE_FILES src/main.cpp)
+
+add_executable(${PROJECT_NAME} ${SOURCE_FILES})
+
+set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
+include_directories(${RKLLM_API_PATH}/include)
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
+    target_link_libraries(${PROJECT_NAME}  ${RKLLM_RT_LIB} log)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
+    target_link_libraries(${PROJECT_NAME}  ${RKLLM_RT_LIB})
+endif()
+
+
--- a/rkllm-runtime/examples/rkllm_api_demo/README.md
+++ b/rkllm-runtime/examples/rkllm_api_demo/README.md
@@ -0,0 +1,50 @@
+# Aarch64 Linux Demo
+## Build
+To execute, run:
+```bash
+bash build-linux.sh
+```
+
+## Install
+Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
+
+```bash
+cp build/build_linux_aarch64_Release/llm_demo /usr/bin
+cp ../runtime/Linux/librkllm_api/aarch64/librkllmrt.so /usr/local/lib
+```
+
+To exit the LLM enter either `exit` or `quit`.
+
+## Run
+```bash
+adb shell
+cd /userdata/llm
+export LD_LIBRARY_PATH=./lib
+taskset f0 ./llm_demo /path/to/your/rkllm/model
+```
+
+# Android Demo
+## Build
+Ensure the `ANDROID_NDK_PATH` option in the `build-android.sh` script is correctly configured:
+```sh
+ANDROID_NDK_PATH=/path/to/your/android-ndk-r18b
+```
+To execute, run:
+```bash
+bash build-android.sh
+```
+
+## Install
+Push the compiled `llm_demo` file and `librkllmrt.so` file to the device:
+```bash
+adb push build/build_android_arm64-v8a_Release/llm_demo /userdata/llm
+adb push ../../runtime/Android/librkllm_api/arm64-v8a/librkllmrt.so /userdata/llm/lib
+```
+
+## Run
+```bash
+adb shell
+cd /userdata/llm
+export LD_LIBRARY_PATH=./lib
+taskset f0 ./llm_demo /path/to/your/rkllm/model
+```
--- a/rkllm-runtime/examples/rkllm_api_demo/build-android.sh
+++ b/rkllm-runtime/examples/rkllm_api_demo/build-android.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Debug / Release / RelWithDebInfo
+if [[ -z ${BUILD_TYPE} ]];then
+    BUILD_TYPE=Release
+fi
+
+ANDROID_NDK_PATH=~/android-ndk-r21e
+TARGET_ARCH=arm64-v8a
+
+TARGET_PLATFORM=android
+if [[ -n ${TARGET_ARCH} ]];then
+TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
+fi
+
+ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
+BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
+
+if [[ ! -d "${BUILD_DIR}" ]]; then
+  mkdir -p ${BUILD_DIR}
+fi
+
+cd ${BUILD_DIR}
+cmake ../.. \
+    -DCMAKE_SYSTEM_NAME=Android \
+    -DCMAKE_SYSTEM_VERSION=23 \
+    -DCMAKE_ANDROID_ARCH_ABI=${TARGET_ARCH} \
+    -DCMAKE_ANDROID_STL_TYPE=c++_static \
+    -DCMAKE_ANDROID_NDK=${ANDROID_NDK_PATH} \
+    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+    -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+
+make -j4
--- a/rkllm-runtime/examples/rkllm_api_demo/build-linux.sh
+++ b/rkllm-runtime/examples/rkllm_api_demo/build-linux.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Debug / Release / RelWithDebInfo
+if [[ -z ${BUILD_TYPE} ]];then
+    BUILD_TYPE=Release
+fi
+
+C_COMPILER=/usr/bin/gcc
+CXX_COMPILER=/usr/bin/g++
+STRIP_COMPILER=/usr/bin/strip
+
+TARGET_ARCH=aarch64
+TARGET_PLATFORM=linux
+if [[ -n ${TARGET_ARCH} ]];then
+TARGET_PLATFORM=${TARGET_PLATFORM}_${TARGET_ARCH}
+fi
+
+ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd )
+BUILD_DIR=${ROOT_PWD}/build/build_${TARGET_PLATFORM}_${BUILD_TYPE}
+
+if [[ ! -d "${BUILD_DIR}" ]]; then
+  mkdir -p ${BUILD_DIR}
+fi
+
+cd ${BUILD_DIR}
+cmake ../.. \
+    -DCMAKE_SYSTEM_PROCESSOR=${TARGET_ARCH} \
+    -DCMAKE_SYSTEM_NAME=Linux \
+    -DCMAKE_C_COMPILER=${C_COMPILER} \
+    -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
+    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+
+make -j4
--- a/rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
+++ b/rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
@@ -0,0 +1,144 @@
+// Copyright (c) 2024 by Rockchip Electronics Co., Ltd. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Modified by Pelochus
+
+#include <string.h>
+#include <unistd.h>
+#include <string>
+#include "rkllm.h"
+#include <fstream>
+#include <iostream>
+#include <csignal>
+#include <vector>
+
+#define PROMPT_TEXT_PREFIX "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
+#define PROMPT_TEXT_POSTFIX "<|im_end|><|im_start|>assistant"
+
+using namespace std;
+
+LLMHandle llmHandle = nullptr;
+
+void exit_handler(int signal)
+{
+    if (llmHandle != nullptr)
+    {
+        cout << "Catched exit signal. Exiting..." << endl;
+        LLMHandle _tmp = llmHandle;
+        llmHandle = nullptr;
+        rkllm_destroy(_tmp);
+        exit(signal);
+    }
+}
+
+void callback(RKLLMResult *result, void *userdata, LLMCallState state)
+{
+    if (state == LLM_RUN_FINISH)
+    {
+        printf("\n");
+    }
+    else if (state == LLM_RUN_ERROR)
+    {
+        printf("\\LLM run error\n");
+    }
+    else
+    {
+<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp
+        printf("%s", text);
+=======
+        printf("%s", result->text);
+>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
+    }
+}
+
+int main(int argc, char **argv)
+{
+    if (argc != 2)
+    {
+        printf("Usage: %s [rkllm_model_path]\n", argv[0]);
+        return -1;
+    }
+    
+    signal(SIGINT, exit_handler);
+    string rkllm_model(argv[1]);
+    printf("RKLLM starting, please wait...\n");
+
+    RKLLMParam param = rkllm_createDefaultParam();
+    param.model_path = rkllm_model.c_str();
+    param.num_npu_core = 2;
+    param.top_k = 1;
+    param.max_new_tokens = 256;
+    param.max_context_len = 512;
+<<<<<<< HEAD:rkllm-runtime/example/src/main.cpp
+    
+=======
+    param.logprobs = false;
+    param.top_logprobs = 5;
+    param.use_gpu = false;
+
+>>>>>>> airockchip-main:rkllm-runtime/examples/rkllm_api_demo/src/main.cpp
+    rkllm_init(&llmHandle, param, callback);
+    printf("RKLLM init success!\n");
+    
+    vector<string> pre_input;
+    pre_input.push_back("Welcome to ezrkllm! This is an adaptation of Rockchip's rknn-llm repo (see github.com/airockchip/rknn-llm) for running LLMs on its SoCs' NPUs.\n");
+    pre_input.push_back("You are currently running the runtime for ");
+    pre_input.push_back(param.target_platform);
+    pre_input.push_back("\nTo exit the model, enter either exit or quit\n");
+    pre_input.push_back("\nMore information here: https://github.com/Pelochus/ezrknpu");
+    pre_input.push_back("\nDetailed information for devs here: https://github.com/Pelochus/ezrknn-llm");
+    
+    cout << "\n*************************** Pelochus' ezrkllm runtime *************************\n" << endl;
+    
+    for (int i = 0; i < (int)pre_input.size(); i++)
+    {
+        cout << pre_input[i];
+    }
+    
+    cout << "\n*******************************************************************************\n" << endl;
+
+    string text;
+    while (true)
+    {
+        std::string input_str;
+        printf("\n");
+        printf("You: ");
+        std::getline(std::cin, input_str);
+        
+        if (input_str == "exit" || input_str == "quit")
+        {
+            cout << "Quitting program..." << endl;
+            break;
+        }
+        
+        for (int i = 0; i < (int)pre_input.size(); i++)
+        {
+            if (input_str == to_string(i))
+            {
+                input_str = pre_input[i];
+                cout << input_str << endl;
+            }
+        }
+        
+        // string text = PROMPT_TEXT_PREFIX + input_str + PROMPT_TEXT_POSTFIX;
+        string text = input_str;
+
+        printf("LLM: ");
+        rkllm_run(llmHandle, text.c_str(), NULL);
+    }
+
+    rkllm_destroy(llmHandle);
+
+    return 0;
+}