mirror of
https://github.com/DrHo1y/ezrknn-llm.git
synced 2026-11-17 05:06:23 +07:00
release v1.1.0
This commit is contained in:
@@ -11,68 +11,41 @@ from flask import Flask, request, jsonify, Response
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# 创建一个锁,用于控制多人访问Server
|
||||
lock = threading.Lock()
|
||||
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
|
||||
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
|
||||
|
||||
# 创建一个全局变量,用于标识服务器当前是否处于阻塞状态
|
||||
is_blocking = False
|
||||
|
||||
# 设置动态库路径
|
||||
# Set the dynamic library path
|
||||
rkllm_lib = ctypes.CDLL('lib/librkllmrt.so')
|
||||
|
||||
# 定义全局变量,用于保存回调函数的输出,便于在gradio界面中输出
|
||||
global_text = []
|
||||
global_state = -1
|
||||
split_byte_data = bytes(b"") # 用于保存分割的字节数据
|
||||
# Define the structures from the library
|
||||
RKLLM_Handle_t = ctypes.c_void_p
|
||||
userdata = ctypes.c_void_p(None)
|
||||
|
||||
# 定义动态库中的结构体
|
||||
class Token(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("logprob", ctypes.c_float),
|
||||
("id", ctypes.c_int32)
|
||||
]
|
||||
LLMCallState = ctypes.c_int
|
||||
LLMCallState.RKLLM_RUN_NORMAL = 0
|
||||
LLMCallState.RKLLM_RUN_FINISH = 1
|
||||
LLMCallState.RKLLM_RUN_ERROR = 2
|
||||
LLMCallState.RKLLM_RUN_GET_LAST_HIDDEN_LAYER = 3
|
||||
|
||||
class RKLLMResult(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("text", ctypes.c_char_p),
|
||||
("tokens", ctypes.POINTER(Token)),
|
||||
("num", ctypes.c_int32)
|
||||
]
|
||||
RKLLMInputMode = ctypes.c_int
|
||||
RKLLMInputMode.RKLLM_INPUT_PROMPT = 0
|
||||
RKLLMInputMode.RKLLM_INPUT_TOKEN = 1
|
||||
RKLLMInputMode.RKLLM_INPUT_EMBED = 2
|
||||
RKLLMInputMode.RKLLM_INPUT_MULTIMODAL = 3
|
||||
|
||||
RKLLMInferMode = ctypes.c_int
|
||||
RKLLMInferMode.RKLLM_INFER_GENERATE = 0
|
||||
RKLLMInferMode.RKLLM_INFER_GET_LAST_HIDDEN_LAYER = 1
|
||||
|
||||
# 定义回调函数
|
||||
def callback(result, userdata, state):
|
||||
global global_text, global_state, split_byte_data
|
||||
if state == 0:
|
||||
# 保存输出的token文本及RKLLM运行状态
|
||||
global_state = state
|
||||
# 需要监控当前的字节数据是否完整,不完整则进行记录,后续进行解析
|
||||
try:
|
||||
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
|
||||
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
|
||||
split_byte_data = bytes(b"")
|
||||
except:
|
||||
split_byte_data += result.contents.text
|
||||
sys.stdout.flush()
|
||||
elif state == 1:
|
||||
# 保存RKLLM运行状态
|
||||
global_state = state
|
||||
print("\n")
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
print("run error")
|
||||
|
||||
# Python端与C++端的回调函数连接
|
||||
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
|
||||
c_callback = callback_type(callback)
|
||||
|
||||
# 定义动态库中的结构体
|
||||
class RKNNllmParam(ctypes.Structure):
|
||||
class RKLLMParam(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("model_path", ctypes.c_char_p),
|
||||
("license_path", ctypes.c_char_p),
|
||||
("num_npu_core", ctypes.c_int32),
|
||||
("max_context_len", ctypes.c_int32),
|
||||
("n_prefill_batch", ctypes.c_int32),
|
||||
("max_new_tokens", ctypes.c_int32),
|
||||
("skip_special_token", ctypes.c_bool),
|
||||
("top_k", ctypes.c_int32),
|
||||
("top_p", ctypes.c_float),
|
||||
("temperature", ctypes.c_float),
|
||||
@@ -84,58 +57,234 @@ class RKNNllmParam(ctypes.Structure):
|
||||
("mirostat_eta", ctypes.c_float),
|
||||
("logprobs", ctypes.c_bool),
|
||||
("top_logprobs", ctypes.c_int32),
|
||||
("use_gpu", ctypes.c_bool)
|
||||
("is_async", ctypes.c_bool),
|
||||
("img_start", ctypes.c_char_p),
|
||||
("img_end", ctypes.c_char_p),
|
||||
("img_content", ctypes.c_char_p),
|
||||
]
|
||||
|
||||
# 定义RKLLM_Handle_t和userdata
|
||||
RKLLM_Handle_t = ctypes.c_void_p
|
||||
userdata = ctypes.c_void_p(None)
|
||||
class RKLLMLoraAdapter(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("lora_adapter_path", ctypes.c_char_p),
|
||||
("lora_adapter_name", ctypes.c_char_p),
|
||||
("scale", ctypes.c_float)
|
||||
]
|
||||
|
||||
# 设置提示文本
|
||||
PROMPT_TEXT_PREFIX = "<|im_start|>system You are a helpful assistant. <|im_end|> <|im_start|>user"
|
||||
PROMPT_TEXT_POSTFIX = "<|im_end|><|im_start|>assistant"
|
||||
class RKLLMLoraParam(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("lora_adapter_name", ctypes.c_char_p)
|
||||
]
|
||||
|
||||
# 定义Python端的RKLLM类,其中包括了对动态库中RKLLM模型的初始化、推理及释放操作
|
||||
class RKLLMPromptCacheParam(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("save_prompt_cache", ctypes.c_int),
|
||||
("prompt_cache_path", ctypes.c_char_p)
|
||||
]
|
||||
|
||||
class RKLLMEmbedInput(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("embed", ctypes.POINTER(ctypes.c_float)),
|
||||
("n_tokens", ctypes.c_size_t)
|
||||
]
|
||||
|
||||
class RKLLMTokenInput(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("input_ids", ctypes.POINTER(ctypes.c_int32)),
|
||||
("n_tokens", ctypes.c_size_t)
|
||||
]
|
||||
|
||||
class RKLLMMultiModelInput(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("prompt", ctypes.c_char_p),
|
||||
("image_embed", ctypes.POINTER(ctypes.c_float)),
|
||||
("n_image_tokens", ctypes.c_size_t)
|
||||
]
|
||||
|
||||
class RKLLMInputUnion(ctypes.Union):
|
||||
_fields_ = [
|
||||
("prompt_input", ctypes.c_char_p),
|
||||
("embed_input", RKLLMEmbedInput),
|
||||
("token_input", RKLLMTokenInput),
|
||||
("multimodal_input", RKLLMMultiModelInput)
|
||||
]
|
||||
|
||||
class RKLLMInput(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("input_mode", ctypes.c_int),
|
||||
("input_data", RKLLMInputUnion)
|
||||
]
|
||||
class RKLLMInferParam(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("mode", RKLLMInferMode),
|
||||
("lora_params", ctypes.POINTER(RKLLMLoraParam)),
|
||||
("prompt_cache_params", ctypes.POINTER(RKLLMPromptCacheParam))
|
||||
]
|
||||
|
||||
class RKLLMResultLastHiddenLayer(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("hidden_states", ctypes.POINTER(ctypes.c_float)),
|
||||
("embd_size", ctypes.c_int),
|
||||
("num_tokens", ctypes.c_int)
|
||||
]
|
||||
|
||||
class RKLLMResult(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("text", ctypes.c_char_p),
|
||||
("size", ctypes.c_int),
|
||||
("last_hidden_layer", RKLLMResultLastHiddenLayer)
|
||||
]
|
||||
|
||||
|
||||
# Create a lock to control multi-user access to the server.
|
||||
lock = threading.Lock()
|
||||
|
||||
# Create a global variable to indicate whether the server is currently in a blocked state.
|
||||
is_blocking = False
|
||||
|
||||
# Define global variables to store the callback function output for displaying in the Gradio interface
|
||||
global_text = []
|
||||
global_state = -1
|
||||
split_byte_data = bytes(b"") # Used to store the segmented byte data
|
||||
|
||||
# Define the callback function
|
||||
def callback_impl(result, userdata, state):
|
||||
global global_text, global_state, split_byte_data
|
||||
if state == LLMCallState.RKLLM_RUN_FINISH:
|
||||
global_state = state
|
||||
print("\n")
|
||||
sys.stdout.flush()
|
||||
elif state == LLMCallState.RKLLM_RUN_ERROR:
|
||||
global_state = state
|
||||
print("run error")
|
||||
sys.stdout.flush()
|
||||
elif state == LLMCallState.RKLLM_RUN_GET_LAST_HIDDEN_LAYER:
|
||||
'''
|
||||
If using the GET_LAST_HIDDEN_LAYER function, the callback interface will return the memory pointer: last_hidden_layer, the number of tokens: num_tokens, and the size of the hidden layer: embd_size.
|
||||
With these three parameters, you can retrieve the data from last_hidden_layer.
|
||||
Note: The data needs to be retrieved during the current callback; if not obtained in time, the pointer will be released by the next callback.
|
||||
'''
|
||||
if result.last_hidden_layer.embd_size != 0 and result.last_hidden_layer.num_tokens != 0:
|
||||
data_size = result.last_hidden_layer.embd_size * result.last_hidden_layer.num_tokens * ctypes.sizeof(ctypes.c_float)
|
||||
print(f"data_size: {data_size}")
|
||||
global_text.append(f"data_size: {data_size}\n")
|
||||
output_path = os.getcwd() + "/last_hidden_layer.bin"
|
||||
with open(output_path, "wb") as outFile:
|
||||
data = ctypes.cast(result.last_hidden_layer.hidden_states, ctypes.POINTER(ctypes.c_float))
|
||||
float_array_type = ctypes.c_float * (data_size // ctypes.sizeof(ctypes.c_float))
|
||||
float_array = float_array_type.from_address(ctypes.addressof(data.contents))
|
||||
outFile.write(bytearray(float_array))
|
||||
print(f"Data saved to {output_path} successfully!")
|
||||
global_text.append(f"Data saved to {output_path} successfully!")
|
||||
else:
|
||||
print("Invalid hidden layer data.")
|
||||
global_text.append("Invalid hidden layer data.")
|
||||
global_state = state
|
||||
time.sleep(0.05) # Delay for 0.05 seconds to wait for the output result
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
# Save the output token text and the RKLLM running state
|
||||
global_state = state
|
||||
# Monitor if the current byte data is complete; if incomplete, record it for later parsing
|
||||
try:
|
||||
global_text.append((split_byte_data + result.contents.text).decode('utf-8'))
|
||||
print((split_byte_data + result.contents.text).decode('utf-8'), end='')
|
||||
split_byte_data = bytes(b"")
|
||||
except:
|
||||
split_byte_data += result.contents.text
|
||||
sys.stdout.flush()
|
||||
|
||||
# Connect the callback function between the Python side and the C++ side
|
||||
callback_type = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
|
||||
callback = callback_type(callback_impl)
|
||||
|
||||
# Define the RKLLM class, which includes initialization, inference, and release operations for the RKLLM model in the dynamic library
|
||||
class RKLLM(object):
|
||||
def __init__(self, model_path, target_platform):
|
||||
rknnllm_param = RKNNllmParam()
|
||||
rknnllm_param.model_path = bytes(model_path, 'utf-8')
|
||||
if target_platform == "rk3588":
|
||||
rknnllm_param.num_npu_core = 3
|
||||
elif target_platform == "rk3576":
|
||||
rknnllm_param.num_npu_core = 1
|
||||
rknnllm_param.max_context_len = 320
|
||||
rknnllm_param.max_new_tokens = 512
|
||||
rknnllm_param.top_k = 1
|
||||
rknnllm_param.top_p = 0.9
|
||||
rknnllm_param.temperature = 0.8
|
||||
rknnllm_param.repeat_penalty = 1.1
|
||||
rknnllm_param.frequency_penalty = 0.0
|
||||
rknnllm_param.presence_penalty = 0.0
|
||||
rknnllm_param.mirostat = 0
|
||||
rknnllm_param.mirostat_tau = 5.0
|
||||
rknnllm_param.mirostat_eta = 0.1
|
||||
rknnllm_param.logprobs = False
|
||||
rknnllm_param.top_logprobs = 5
|
||||
rknnllm_param.use_gpu = True
|
||||
def __init__(self, model_path, num_npu_core, lora_model_path = None, prompt_cache_path = None):
|
||||
rkllm_param = RKLLMParam()
|
||||
rkllm_param.model_path = bytes(model_path, 'utf-8')
|
||||
rkllm_param.license_path = None
|
||||
rkllm_param.num_npu_core = num_npu_core
|
||||
|
||||
rkllm_param.max_context_len = 512
|
||||
rkllm_param.n_prefill_batch = 512
|
||||
rkllm_param.max_new_tokens = -1
|
||||
rkllm_param.skip_special_token = True
|
||||
|
||||
rkllm_param.top_k = 1
|
||||
rkllm_param.top_p = 0.9
|
||||
rkllm_param.temperature = 0.8
|
||||
rkllm_param.repeat_penalty = 1.1
|
||||
rkllm_param.frequency_penalty = 0.0
|
||||
rkllm_param.presence_penalty = 0.0
|
||||
|
||||
rkllm_param.mirostat = 0
|
||||
rkllm_param.mirostat_tau = 5.0
|
||||
rkllm_param.mirostat_eta = 0.1
|
||||
|
||||
rkllm_param.logprobs = False
|
||||
rkllm_param.top_logprobs = 5
|
||||
rkllm_param.is_async = False
|
||||
|
||||
rkllm_param.img_start = "".encode('utf-8')
|
||||
rkllm_param.img_end = "".encode('utf-8')
|
||||
rkllm_param.img_content = "".encode('utf-8')
|
||||
|
||||
self.handle = RKLLM_Handle_t()
|
||||
|
||||
self.rkllm_init = rkllm_lib.rkllm_init
|
||||
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), ctypes.POINTER(RKNNllmParam), callback_type]
|
||||
self.rkllm_init.argtypes = [ctypes.POINTER(RKLLM_Handle_t), RKLLMParam, callback_type]
|
||||
self.rkllm_init.restype = ctypes.c_int
|
||||
self.rkllm_init(ctypes.byref(self.handle), rknnllm_param, c_callback)
|
||||
self.rkllm_init(ctypes.byref(self.handle), rkllm_param, callback)
|
||||
|
||||
self.rkllm_run = rkllm_lib.rkllm_run
|
||||
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(ctypes.c_char), ctypes.c_void_p]
|
||||
self.rkllm_run.argtypes = [RKLLM_Handle_t, ctypes.POINTER(RKLLMInput), ctypes.POINTER(RKLLMInferParam), ctypes.c_void_p]
|
||||
self.rkllm_run.restype = ctypes.c_int
|
||||
|
||||
self.rkllm_destroy = rkllm_lib.rkllm_destroy
|
||||
self.rkllm_destroy.argtypes = [RKLLM_Handle_t]
|
||||
self.rkllm_destroy.restype = ctypes.c_int
|
||||
|
||||
self.lora_adapter_path = None
|
||||
self.lora_model_name = None
|
||||
if lora_model_path:
|
||||
self.lora_adapter_path = lora_model_path
|
||||
self.lora_adapter_name = "test"
|
||||
|
||||
lora_adapter = RKLLMLoraAdapter()
|
||||
ctypes.memset(ctypes.byref(lora_adapter), 0, ctypes.sizeof(RKLLMLoraAdapter))
|
||||
lora_adapter.lora_adapter_path = ctypes.c_char_p((self.lora_adapter_path).encode('utf-8'))
|
||||
lora_adapter.lora_adapter_name = ctypes.c_char_p((self.lora_adapter_name).encode('utf-8'))
|
||||
lora_adapter.scale = 1.0
|
||||
|
||||
rkllm_load_lora = rkllm_lib.rkllm_load_lora
|
||||
rkllm_load_lora.argtypes = [RKLLM_Handle_t, ctypes.POINTER(RKLLMLoraAdapter)]
|
||||
rkllm_load_lora.restype = ctypes.c_int
|
||||
rkllm_load_lora(self.handle, ctypes.byref(lora_adapter))
|
||||
|
||||
self.prompt_cache_path = None
|
||||
if prompt_cache_path:
|
||||
self.prompt_cache_path = prompt_cache_path
|
||||
|
||||
rkllm_load_prompt_cache = rkllm_lib.rkllm_load_prompt_cache
|
||||
rkllm_load_prompt_cache.argtypes = [RKLLM_Handle_t, ctypes.c_char_p]
|
||||
rkllm_load_prompt_cache.restype = ctypes.c_int
|
||||
rkllm_load_prompt_cache(self.handle, ctypes.c_char_p((prompt_cache_path).encode('utf-8')))
|
||||
|
||||
def run(self, prompt):
|
||||
prompt = bytes(PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX, 'utf-8')
|
||||
self.rkllm_run(self.handle, prompt, ctypes.byref(userdata))
|
||||
rkllm_lora_params = None
|
||||
if self.lora_model_name:
|
||||
rkllm_lora_params = RKLLMLoraParam()
|
||||
rkllm_lora_params.lora_adapter_name = ctypes.c_char_p((self.lora_model_name).encode('utf-8'))
|
||||
|
||||
rkllm_infer_params = RKLLMInferParam()
|
||||
ctypes.memset(ctypes.byref(rkllm_infer_params), 0, ctypes.sizeof(RKLLMInferParam))
|
||||
rkllm_infer_params.mode = RKLLMInferMode.RKLLM_INFER_GENERATE
|
||||
rkllm_infer_params.lora_params = ctypes.byref(rkllm_lora_params) if rkllm_lora_params else None
|
||||
|
||||
rkllm_input = RKLLMInput()
|
||||
rkllm_input.input_mode = RKLLMInputMode.RKLLM_INPUT_PROMPT
|
||||
rkllm_input.input_data.prompt_input = ctypes.c_char_p((PROMPT_TEXT_PREFIX + prompt + PROMPT_TEXT_POSTFIX).encode('utf-8'))
|
||||
self.rkllm_run(self.handle, ctypes.byref(rkllm_input), ctypes.byref(rkllm_infer_params), None)
|
||||
return
|
||||
|
||||
def release(self):
|
||||
@@ -143,62 +292,81 @@ class RKLLM(object):
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--target_platform', help='目标平台: 如rk3588/rk3576;')
|
||||
parser.add_argument('--rkllm_model_path', help='Linux板端上已转换好的rkllm模型的绝对路径')
|
||||
parser.add_argument('--rkllm_model_path', type=str, required=True, help='Absolute path of the converted RKLLM model on the Linux board;')
|
||||
parser.add_argument('--target_platform', type=str, required=True, help='Target platform: e.g., rk3588/rk3576;')
|
||||
parser.add_argument('--num_npu_core', type=int, required=True, help='Target num_npu_core;')
|
||||
parser.add_argument('--lora_model_path', type=str, help='Absolute path of the lora_model on the Linux board;')
|
||||
parser.add_argument('--prompt_cache_path', type=str, help='Absolute path of the prompt_cache file on the Linux board;')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not (args.target_platform in ["rk3588", "rk3576"]):
|
||||
print("====== Error: 请指定正确的目标平台: rk3588/rk3576 ======")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
if not os.path.exists(args.rkllm_model_path):
|
||||
print("====== Error: 请给出准确的rkllm模型路径,需注意是板端的绝对路径 ======")
|
||||
print("Error: Please provide the correct rkllm model path, and ensure it is the absolute path on the board.")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
# 定频设置
|
||||
if not (args.target_platform in ["rk3588", "rk3576"]):
|
||||
print("Error: Please specify the correct target platform: rk3588/rk3576.")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
if not (args.num_npu_core in [1, 2, 3]):
|
||||
print("Error: rk3576 supports 1/2 cores, rk3588 supports 1/2/3 cores, please specify the correct number of cores.")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
if args.lora_model_path:
|
||||
if not os.path.exists(args.lora_model_path):
|
||||
print("Error: Please provide the correct lora_model path, and advise it is the absolute path on the board.")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
if args.prompt_cache_path:
|
||||
if not os.path.exists(args.prompt_cache_path):
|
||||
print("Error: Please provide the correct prompt_cache_file path, and advise it is the absolute path on the board.")
|
||||
sys.stdout.flush()
|
||||
exit()
|
||||
|
||||
# Fix frequency
|
||||
command = "sudo bash fix_freq_{}.sh".format(args.target_platform)
|
||||
subprocess.run(command, shell=True)
|
||||
|
||||
# 设置文件描述符限制
|
||||
# Set resource limit
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
|
||||
|
||||
# 初始化RKLLM模型
|
||||
# Initialize RKLLM model
|
||||
print("=========init....===========")
|
||||
sys.stdout.flush()
|
||||
target_platform = args.target_platform
|
||||
model_path = args.rkllm_model_path
|
||||
rkllm_model = RKLLM(model_path, target_platform)
|
||||
print("RKLLM初始化成功!")
|
||||
num_npu_core = args.num_npu_core
|
||||
rkllm_model = RKLLM(model_path, num_npu_core, args.lora_model_path, args.prompt_cache_path)
|
||||
print("RKLLM Model has been initialized successfully!")
|
||||
print("==============================")
|
||||
sys.stdout.flush()
|
||||
|
||||
# 创建一个函数用于接受用户使用 request 发送的数据
|
||||
# Create a function to receive data sent by the user using a request
|
||||
@app.route('/rkllm_chat', methods=['POST'])
|
||||
def receive_message():
|
||||
# 链接全局变量,获取回调函数的输出信息
|
||||
# Link global variables to retrieve the output information from the callback function
|
||||
global global_text, global_state
|
||||
global is_blocking
|
||||
|
||||
# 如果服务器正在阻塞状态,则返回特定响应
|
||||
# If the server is in a blocking state, return a specific response.
|
||||
if is_blocking or global_state==0:
|
||||
return jsonify({'status': 'error', 'message': 'RKLLM_Server is busy! Maybe you can try again later.'}), 503
|
||||
|
||||
# 加锁
|
||||
lock.acquire()
|
||||
try:
|
||||
# 设置服务器为阻塞状态
|
||||
# Set the server to a blocking state.
|
||||
is_blocking = True
|
||||
|
||||
# 获取 POST 请求中的 JSON 数据
|
||||
# Get JSON data from the POST request.
|
||||
data = request.json
|
||||
if data and 'messages' in data:
|
||||
# 重置全局变量
|
||||
# Reset global variables.
|
||||
global_text = []
|
||||
global_state = -1
|
||||
|
||||
# 定义返回的结构体
|
||||
# Define the structure for the returned response.
|
||||
rkllm_responses = {
|
||||
"id": "rkllm_chat",
|
||||
"object": "rkllm_chat",
|
||||
@@ -212,18 +380,18 @@ if __name__ == "__main__":
|
||||
}
|
||||
|
||||
if not "stream" in data.keys() or data["stream"] == False:
|
||||
# 在这里处理收到的数据
|
||||
# Process the received data here.
|
||||
messages = data['messages']
|
||||
print("Received messages:", messages)
|
||||
for index, message in enumerate(messages):
|
||||
input_prompt = message['content']
|
||||
rkllm_output = ""
|
||||
|
||||
# 创建模型推理的线程
|
||||
# Create a thread for model inference.
|
||||
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
|
||||
model_thread.start()
|
||||
|
||||
# 等待模型运行完成,定时检查模型的推理线程
|
||||
# Wait for the model to finish running and periodically check the inference thread of the model.
|
||||
model_thread_finished = False
|
||||
while not model_thread_finished:
|
||||
while len(global_text) > 0:
|
||||
@@ -245,7 +413,6 @@ if __name__ == "__main__":
|
||||
)
|
||||
return jsonify(rkllm_responses), 200
|
||||
else:
|
||||
# 在这里处理收到的数据
|
||||
messages = data['messages']
|
||||
print("Received messages:", messages)
|
||||
for index, message in enumerate(messages):
|
||||
@@ -253,11 +420,9 @@ if __name__ == "__main__":
|
||||
rkllm_output = ""
|
||||
|
||||
def generate():
|
||||
# 创建模型推理的线程
|
||||
model_thread = threading.Thread(target=rkllm_model.run, args=(input_prompt,))
|
||||
model_thread.start()
|
||||
|
||||
# 等待模型运行完成,定时检查模型的推理线程
|
||||
model_thread_finished = False
|
||||
while not model_thread_finished:
|
||||
while len(global_text) > 0:
|
||||
@@ -282,16 +447,14 @@ if __name__ == "__main__":
|
||||
else:
|
||||
return jsonify({'status': 'error', 'message': 'Invalid JSON data!'}), 400
|
||||
finally:
|
||||
# 释放锁
|
||||
lock.release()
|
||||
# 将服务器状态设置为非阻塞
|
||||
is_blocking = False
|
||||
|
||||
# 启动 Flask 应用程序
|
||||
# Start the Flask application.
|
||||
# app.run(host='0.0.0.0', port=8080)
|
||||
app.run(host='0.0.0.0', port=8080, threaded=True, debug=False)
|
||||
|
||||
print("====================")
|
||||
print("RKLLM模型推理结束, 释放RKLLM模型资源...")
|
||||
print("RKLLM model inference completed, releasing RKLLM model resources...")
|
||||
rkllm_model.release()
|
||||
print("====================")
|
||||
|
||||
Reference in New Issue
Block a user