fix code to work
This commit is contained in:
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
/models
|
||||||
|
/compiled-models
|
||||||
14
Dockerfile
14
Dockerfile
@@ -1,16 +1,24 @@
|
|||||||
FROM ubuntu:22.04 AS builder
|
FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS builder
|
||||||
|
|
||||||
# Установка зависимостей
|
# Установка зависимостей
|
||||||
|
|
||||||
RUN apt update && apt-get install -y \
|
RUN apt update && apt-get install -y \
|
||||||
python3 pip git curl wget nano sudo apt-utils && \
|
python3.10 pip git curl wget nano sudo apt-utils && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Установка RKLLM Toolkit
|
# Установка RKLLM Toolkit
|
||||||
RUN git clone https://github.com/airockchip/rknn-llm
|
RUN git clone https://github.com/airockchip/rknn-llm
|
||||||
|
|
||||||
|
# Удаляем защиту PEP 668
|
||||||
|
RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED || true
|
||||||
|
|
||||||
|
RUN pip3 install torch==2.6.0 numpy==1.26.4
|
||||||
|
|
||||||
|
# 2. Установите auto_gptq БЕЗ CUDA (игнорируя суффиксы версий)
|
||||||
|
RUN pip install auto-gptq==0.7.1 --no-build-isolation
|
||||||
|
|
||||||
# Установка Python-пакета
|
# Установка Python-пакета
|
||||||
RUN pip3 install /ezrknn-llm/rkllm-toolkit/packages/rkllm_toolkit-1.2.3-cp312-cp312-linux_x86_64.whl
|
RUN pip3 install /rknn-llm/rkllm-toolkit/packages/rkllm_toolkit-1.2.3-cp310-cp310-linux_x86_64.whl
|
||||||
|
|
||||||
# Клонирование модели
|
# Клонирование модели
|
||||||
WORKDIR /models
|
WORKDIR /models
|
||||||
|
|||||||
22
build.sh
22
build.sh
@@ -1,25 +1,31 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
MODELS_PATH="./models"
|
MODELS_PATH="models"
|
||||||
OUTPUT_DIR="./compiled-models"
|
OUTPUT_DIR="compiled-models"
|
||||||
IMAGE_NAME="rkllm-builder"
|
IMAGE_NAME="rkllm-builder"
|
||||||
CONTAINER_NAME="rkllm-compile-$$"
|
CONTAINER_NAME="rkllm-compile-$$"
|
||||||
|
|
||||||
mkdir -p "$OUTPUT_DIR"
|
mkdir -p "$MODELS_PATH" "$OUTPUT_DIR"
|
||||||
|
|
||||||
echo "🏗️ Сборка образа для компиляции..."
|
echo "🏗️ Сборка образа для компиляции..."
|
||||||
docker build -f Dockerfile -t "$IMAGE_NAME" .
|
docker build -f Dockerfile -t "$IMAGE_NAME" .
|
||||||
|
|
||||||
mkdir -p "$MODELS_PATH"
|
# Клонируем модель на хост (если ещё не клонирована)
|
||||||
cd "$MODELS_PATH"
|
# if [ ! -d "$MODELS_PATH/Qwen3-4B-Instruct-2507-gptq-w4a16-g128" ]; then
|
||||||
git clone https://huggingface.co/simaai/Qwen3-4B-Instruct-2507-a16w4 && \
|
# echo "📥 Клонирование модели из Hugging Face..."
|
||||||
cd Qwen3-4B-Instruct-2507-a16w4 && git lfs pull
|
# cd "$MODELS_PATH"
|
||||||
|
# git clone https://huggingface.co/kaitchup/Qwen3-4B-Instruct-2507-gptq-w4a16-g128
|
||||||
|
# cd "Qwen3-4B-Instruct-2507-gptq-w4a16-g128" && git lfs pull || true
|
||||||
|
# cd ../..
|
||||||
|
# else
|
||||||
|
# echo "✅ Модель уже клонирована в $MODELS_PATH/Qwen3-4B-Instruct-2507-gptq-w4a16-g128"
|
||||||
|
# fi
|
||||||
|
|
||||||
echo "⚙️ Запуск компиляции (может занять 30-60 минут)..."
|
echo "⚙️ Запуск компиляции (может занять 30-60 минут)..."
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
--name "$CONTAINER_NAME" \
|
--name "$CONTAINER_NAME" \
|
||||||
-v "$(pwd)/"$MODELS_PATH":/models" \
|
-v "$(pwd)/$MODELS_PATH:/models" \
|
||||||
-v "$(pwd)/$OUTPUT_DIR:/output" \
|
-v "$(pwd)/$OUTPUT_DIR:/output" \
|
||||||
"$IMAGE_NAME"
|
"$IMAGE_NAME"
|
||||||
|
|
||||||
|
|||||||
@@ -4,23 +4,25 @@ import torch
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
import os
|
import os
|
||||||
|
|
||||||
modelpath = '/models/Qwen3-4B-Instruct-2507-a16w4/devkit'
|
modelpath = '/models/Qwen3-4B-Instruct-2507-F16/Qwen3-4B-Instruct-2507-F16.gguf'
|
||||||
output = '/output/Qwen3-4B-Instruct-2507-a16w4.rkllm'
|
output = '/output/Qwen3-4B-Instruct-2507-F16-Q.rkllm'
|
||||||
os.makedirs("/output", exist_ok=True)
|
os.makedirs("/output", exist_ok=True)
|
||||||
|
|
||||||
print(f"Загрузка модели из: {modelpath}")
|
print(f"Загрузка модели из: {modelpath}")
|
||||||
|
|
||||||
llm = RKLLM()
|
llm = RKLLM()
|
||||||
ret = llm.load_huggingface(model=modelpath, model_lora = None, device='cpu')
|
ret = llm.load_gguf(model=modelpath)
|
||||||
|
# ret = llm.load_huggingface(model=modelpath, model_lora = None, device='cpu')
|
||||||
|
|
||||||
if ret != 0:
|
if ret != 0:
|
||||||
print(f"❌ Ошибка загрузки модели: {ret}")
|
print(f"❌ Ошибка загрузки модели: {ret}")
|
||||||
exit(ret)
|
exit(ret)
|
||||||
|
|
||||||
print("Компиляция для RK3588 (NPU)...")
|
print("Компиляция для RK3588 (NPU)...")
|
||||||
ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w4a16',
|
ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8',
|
||||||
quantized_algorithm='normal', target_platform='rk3588', num_npu_core=3, extra_qparams=None)
|
quantized_algorithm='normal', target_platform='rk3588', num_npu_core=3, extra_qparams=None)
|
||||||
|
|
||||||
|
|
||||||
if ret != 0:
|
if ret != 0:
|
||||||
print(f"❌ Ошибка компиляции: {ret}")
|
print(f"❌ Ошибка компиляции: {ret}")
|
||||||
exit(ret)
|
exit(ret)
|
||||||
|
|||||||
Reference in New Issue
Block a user