mirror of
https://github.com/DrHo1y/ezrknn-llm.git
synced 2026-11-17 05:06:23 +07:00
release v1.1.0
This commit is contained in:
@@ -1,29 +0,0 @@
|
||||
from rkllm.api import RKLLM
|
||||
|
||||
'''
|
||||
https://huggingface.co/Qwen/Qwen-1_8B-Chat
|
||||
Download the Qwen model from the above website.
|
||||
'''
|
||||
|
||||
modelpath = '/path/to/your/model'
|
||||
llm = RKLLM()
|
||||
|
||||
# Load model
|
||||
ret = llm.load_huggingface(model = modelpath)
|
||||
if ret != 0:
|
||||
print('Load model failed!')
|
||||
exit(ret)
|
||||
|
||||
# Build model
|
||||
ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8', target_platform='rk3588')
|
||||
if ret != 0:
|
||||
print('Build model failed!')
|
||||
exit(ret)
|
||||
|
||||
# Export rknn model
|
||||
ret = llm.export_rkllm("./qwen.rkllm")
|
||||
if ret != 0:
|
||||
print('Export model failed!')
|
||||
exit(ret)
|
||||
|
||||
|
||||
86
rkllm-toolkit/examples/test.py
Executable file
86
rkllm-toolkit/examples/test.py
Executable file
@@ -0,0 +1,86 @@
|
||||
from rkllm.api import RKLLM
|
||||
from datasets import load_dataset
|
||||
from transformers import AutoTokenizer
|
||||
from tqdm import tqdm
|
||||
import torch
|
||||
from torch import nn
|
||||
import os
|
||||
# os.environ['CUDA_VISIBLE_DEVICES']='1'
|
||||
|
||||
'''
|
||||
https://huggingface.co/Qwen/Qwen-1_8B-Chat
|
||||
从上面网址中下载Qwen模型
|
||||
'''
|
||||
|
||||
modelpath = './path/to/model'
|
||||
# modelpath = "./path/to/Qwen-1.8B-F16.gguf"
|
||||
llm = RKLLM()
|
||||
|
||||
# Load model
|
||||
# Use 'export CUDA_VISIBLE_DEVICES=2' to specify GPU device
|
||||
# options ['cpu', 'cuda']
|
||||
ret = llm.load_huggingface(model=modelpath, model_lora = None, device='cpu')
|
||||
# ret = llm.load_gguf(model = modelpath)
|
||||
if ret != 0:
|
||||
print('Load model failed!')
|
||||
exit(ret)
|
||||
|
||||
# Build model
|
||||
dataset = "./data_quant.json"
|
||||
# Json file format, please note to add prompt in the input,like this:
|
||||
# [{"input":"Human: 你好!\nAssistant: ", "target": "你好!我是人工智能助手KK!"},...]
|
||||
|
||||
qparams = None
|
||||
# qparams = 'gdq.qparams' # Use extra_qparams
|
||||
ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8',
|
||||
quantized_algorithm='normal', target_platform='rk3588', num_npu_core=3, extra_qparams=qparams, dataset=dataset)
|
||||
|
||||
if ret != 0:
|
||||
print('Build model failed!')
|
||||
exit(ret)
|
||||
|
||||
# Evaluate Accuracy
|
||||
def eval_wikitext(llm):
|
||||
seqlen = 512
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
modelpath, trust_remote_code=True)
|
||||
# Dataset download link:
|
||||
# https://huggingface.co/datasets/Salesforce/wikitext/tree/main/wikitext-2-raw-v1
|
||||
testenc = load_dataset(
|
||||
"parquet", data_files='./wikitext/wikitext-2-raw-1/test-00000-of-00001.parquet', split='train')
|
||||
testenc = tokenizer("\n\n".join(
|
||||
testenc['text']), return_tensors="pt").input_ids
|
||||
nsamples = testenc.numel() // seqlen
|
||||
nlls = []
|
||||
for i in tqdm(range(nsamples), desc="eval_wikitext: "):
|
||||
batch = testenc[:, (i * seqlen): ((i + 1) * seqlen)]
|
||||
inputs = {"input_ids": batch}
|
||||
lm_logits = llm.get_logits(inputs)
|
||||
if lm_logits is None:
|
||||
print("get logits failed!")
|
||||
return
|
||||
shift_logits = lm_logits[:, :-1, :]
|
||||
shift_labels = batch[:, 1:].to(lm_logits.device)
|
||||
loss_fct = nn.CrossEntropyLoss().to(lm_logits.device)
|
||||
loss = loss_fct(
|
||||
shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
|
||||
neg_log_likelihood = loss.float() * seqlen
|
||||
nlls.append(neg_log_likelihood)
|
||||
ppl = torch.exp(torch.stack(nlls).sum() / (nsamples * seqlen))
|
||||
print(f'wikitext-2-raw-1-test ppl: {round(ppl.item(), 2)}')
|
||||
|
||||
# eval_wikitext(llm)
|
||||
|
||||
|
||||
# Chat with model
|
||||
messages = "<|im_start|>system You are a helpful assistant.<|im_end|><|im_start|>user你好!\n<|im_end|><|im_start|>assistant"
|
||||
kwargs = {"max_length": 128, "top_k": 1, "top_p": 0.8,
|
||||
"temperature": 0.8, "do_sample": True, "repetition_penalty": 1.1}
|
||||
# print(llm.chat_model(messages, kwargs))
|
||||
|
||||
|
||||
# Export rkllm model
|
||||
ret = llm.export_rkllm("./qwen.rkllm")
|
||||
if ret != 0:
|
||||
print('Export model failed!')
|
||||
exit(ret)
|
||||
Reference in New Issue
Block a user