Compare commits
14 Commits
324ce0ddea
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 17cf7a6a55 | |||
| a66f257fed | |||
| 7a4470ab95 | |||
| e6f2ac2195 | |||
| 35d0a9db6d | |||
| d484043db5 | |||
| 18e2dcd592 | |||
| 5a779f44a9 | |||
| 8782400d7e | |||
| d9126af9d7 | |||
| f9a81c0752 | |||
| c45c367982 | |||
| 811e767bc5 | |||
| d736fe838a |
53
.gitea/workflows/docker.yml
Normal file
53
.gitea/workflows/docker.yml
Normal file
@@ -0,0 +1,53 @@
|
||||
name: Build, Push and Redeploy Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
REGISTRY_HOST: git.byte-mate.ru
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU (for multi-arch, optional)
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Gitea Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY_HOST }}
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.GIT_PASSWORD }}
|
||||
|
||||
# - name: Log in to Gitea Container Registry
|
||||
# run: |
|
||||
# echo "${{ secrets.GIT_PASSWORD }}" | docker login "${{ env.REGISTRY_HOST }}" -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Get short SHA
|
||||
id: vars
|
||||
run: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
docker buildx build \
|
||||
--no-cache \
|
||||
--push \
|
||||
--platform linux/arm64/v8 \
|
||||
--tag ${{ env.REGISTRY_HOST }}/drholy/rkllm-gradio-server:${{ env.sha_short }} \
|
||||
--tag ${{ env.REGISTRY_HOST }}/drholy/rkllm-gradio-server:latest \
|
||||
--push \
|
||||
.
|
||||
|
||||
- name: Redeploy stack in Portainer
|
||||
run: |
|
||||
curl -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
|
||||
41
Dockerfile
41
Dockerfile
@@ -2,7 +2,6 @@
|
||||
FROM python:3.11.11
|
||||
|
||||
# Установка метаданных
|
||||
LABEL maintainer="your-email@example.com"
|
||||
LABEL description="Docker image with Python 3.11.11 and required packages"
|
||||
|
||||
# Обновление системы и установка базовых зависимостей
|
||||
@@ -17,13 +16,39 @@ RUN apt-get update && \
|
||||
cmake \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Обновление pip и установка setuptools/wheel
|
||||
WORKDIR /app
|
||||
|
||||
# Кэшируем установку зависимостей Python
|
||||
COPY requirements.txt .
|
||||
RUN pip install --upgrade pip && \
|
||||
pip install setuptools wheel
|
||||
pip install setuptools wheel && \
|
||||
pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Проверка версии Python и pip
|
||||
RUN python --version && pip --version
|
||||
# Сборка RKLLM runtime
|
||||
RUN git clone --depth 1 https://git.byte-mate.ru/drholy/ezrknn-llm /tmp/rkllm-build && \
|
||||
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib && \
|
||||
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include && \
|
||||
cd /tmp/rkllm-build/rkllm-runtime/examples/rkllm_api_demo && \
|
||||
bash build-linux.sh && \
|
||||
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm && \
|
||||
# Очистка временных файлов сборки
|
||||
ldconfig && \
|
||||
rm -rf /tmp/rkllm-build && \
|
||||
apt-get purge -y git cmake build-essential libssl-dev && \
|
||||
apt-get autoremove -y && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
# Очистка кэша
|
||||
RUN apt-get clean && \
|
||||
rm -rf /tmp/* /var/tmp/*
|
||||
# Копируем приложение (исключая кэш и модели)
|
||||
COPY app/ .
|
||||
|
||||
# Исключаем модели из образа — монтируем как volume при запуске
|
||||
RUN rm -rf /app/models/* 2>/dev/null || true && \
|
||||
find /app -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
|
||||
chmod +x /app/entrypoint.sh
|
||||
|
||||
# Порт Gradio по умолчанию
|
||||
EXPOSE 8080
|
||||
|
||||
WORKDIR /app
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
@@ -53,9 +53,6 @@ http://localhost:8080
|
||||
Проект использует следующие виртуальные файловые системы:
|
||||
|
||||
- `/app` — для приложения.
|
||||
- `/cache/cache` — для кэша.
|
||||
- `/cache/site-packages` — для пакетов Python.
|
||||
- `/offline_packages` — для офлайн пакетов.
|
||||
- `/ezrknn-llm` — для дополнительных библиотек.
|
||||
|
||||
## Настройка и использование моделей
|
||||
|
||||
@@ -1,38 +1,18 @@
|
||||
#!/bin/bash
|
||||
message_print() {
|
||||
echo
|
||||
echo "#########################################"
|
||||
echo $1
|
||||
echo "#########################################"
|
||||
echo
|
||||
}
|
||||
set -e # Автоматический выход при ошибке
|
||||
|
||||
cd /
|
||||
echo
|
||||
echo "#########################################"
|
||||
echo "Starting RKLLM Gradio Server"
|
||||
echo "#########################################"
|
||||
echo
|
||||
|
||||
message_print "Changing to repository..."
|
||||
git clone https://github.com/DrHo1y/ezrknn-llm
|
||||
cd ezrknn-llm/
|
||||
cp ./rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib
|
||||
cp ./rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include
|
||||
# Проверка наличия моделей (монтируются через volume)
|
||||
if [ ! -d "/app/models" ] || [ -z "$(ls -A /app/models 2>/dev/null)" ]; then
|
||||
echo "⚠️ WARNING: Models directory is empty!"
|
||||
echo " Mount models volume: -v ./models:/app/models"
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
message_print "Compiling LLM runtime for Linux..."
|
||||
cd ./rkllm-runtime/examples/rkllm_api_demo/
|
||||
bash build-linux.sh
|
||||
|
||||
message_print "Moving rkllm to /usr/bin..."
|
||||
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm
|
||||
echo "* soft nofile 16384" >> /etc/security/limits.conf
|
||||
echo "* hard nofile 1048576" >> /etc/security/limits.conf
|
||||
|
||||
message_print "Increasing file limit for all users (needed for LLMs to run)..."
|
||||
echo "root soft nofile 16384" >> /etc/security/limits.conf
|
||||
echo "root hard nofile 1048576" >> /etc/security/limits.conf
|
||||
message_print "Done installing ezrknn-llm!"
|
||||
|
||||
message_print "Install python packages"
|
||||
cd /offline_packages
|
||||
python -m pip install --no-index --find-links . -r requirements.txt
|
||||
|
||||
message_print "Start Gradio server"
|
||||
cd /app
|
||||
python rkllm_server_gradio.py
|
||||
# Запуск сервера
|
||||
exec python rkllm_server_gradio.py
|
||||
@@ -15,36 +15,9 @@ model_configs = {
|
||||
"Qwen2.5-3B-Instruct-w8w8": {"filename": "Qwen2.5-3B-Instruct-w8w8.rkllm"}
|
||||
}
|
||||
},
|
||||
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": {
|
||||
"Qwen3-4B-Instruct-2507-F16": {
|
||||
"base_config": {
|
||||
"st_model_id": "Qwen/Qwen2.5-14B-Instruct",
|
||||
"max_context_len": 128000,
|
||||
"max_new_tokens": 8192,
|
||||
"top_k": 5,
|
||||
"top_p": 0.8,
|
||||
"temperature": 0.2,
|
||||
"repeat_penalty": 1.00,
|
||||
"frequency_penalty": 0.2,
|
||||
"system_prompt": """
|
||||
Ты — эксперт по программированию, и твоя задача — генерировать только рабочий код без каких-либо комментариев, объяснений или дополнительного текста. Вывод должен содержать исключительно код, который можно скопировать и использовать напрямую.
|
||||
|
||||
<описание задачи>
|
||||
|
||||
Пример:
|
||||
Если требуется написать функцию на Python, которая суммирует два числа, то ответ должен быть таким:
|
||||
```
|
||||
def sum(a, b):
|
||||
return a + b
|
||||
```
|
||||
"""
|
||||
},
|
||||
"models": {
|
||||
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"}
|
||||
}
|
||||
},
|
||||
"Qwen2.5-Coder-3B-Instruct-w8w8": {
|
||||
"base_config": {
|
||||
"st_model_id": "Qwen/Qwen2.5-14B-Instruct",
|
||||
"st_model_id": "unsloth/Qwen3-4B-Instruct-2507-GGUF",
|
||||
"max_context_len": 128000,
|
||||
"max_new_tokens": 8192,
|
||||
"top_k": 5,
|
||||
@@ -55,7 +28,7 @@ model_configs = {
|
||||
"system_prompt": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"
|
||||
},
|
||||
"models": {
|
||||
"Qwen2.5-Coder-3B-Instruct-w8w8": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"}
|
||||
"Qwen3-4B-Instruct-2507-F16": {"filename": "Qwen3-4B-Instruct-2507-F16.rkllm"}
|
||||
}
|
||||
}
|
||||
# "LLaMA-Mesh": {
|
||||
|
||||
@@ -6,8 +6,8 @@ from model_class import *
|
||||
from mesh_utils import *
|
||||
|
||||
# Set environment variables
|
||||
os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0"
|
||||
os.environ["GRADIO_SERVER_PORT"] = "8080"
|
||||
#os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0"
|
||||
#os.environ["GRADIO_SERVER_PORT"] = 8080
|
||||
os.environ["RKLLM_LOG_LEVEL"] = "1"
|
||||
# Set resource limit
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
|
||||
@@ -29,7 +29,7 @@ if __name__ == "__main__":
|
||||
print(init_msg)
|
||||
sys.stdout.flush()
|
||||
rkllm_model = RKLLMLoaderClass(model=model)
|
||||
model_init = f"RKLLM Model, {rkllm_model.model_name} has been initialized successfully!"
|
||||
model_init = f"RKLLM Model, {rkllm_model.model_name} has been initialized successfully!"
|
||||
print(model_init)
|
||||
complete_init = "=============================="
|
||||
print(complete_init)
|
||||
@@ -106,7 +106,13 @@ if __name__ == "__main__":
|
||||
# Enable the event queue system.
|
||||
chatRKLLM.queue()
|
||||
# Start the Gradio application.
|
||||
chatRKLLM.launch()
|
||||
chatRKLLM.launch(
|
||||
server_name="0.0.0.0",
|
||||
server_port=8080,
|
||||
share=False,
|
||||
show_error=True,
|
||||
quiet=False
|
||||
)
|
||||
|
||||
print("====================")
|
||||
print("RKLLM model inference completed, releasing RKLLM model resources...")
|
||||
|
||||
20
docker-compose-prod.yml
Normal file
20
docker-compose-prod.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
services:
|
||||
rkllm:
|
||||
image: git.home/drholy/rkllm-gradio-server:latest
|
||||
container_name: rkllm
|
||||
restart: always
|
||||
privileged: true
|
||||
volumes:
|
||||
- ./models:/app/models:ro
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 16384
|
||||
hard: 1048576
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- apps
|
||||
networks:
|
||||
apps:
|
||||
name: apps
|
||||
driver: bridge
|
||||
@@ -1,19 +1,16 @@
|
||||
services:
|
||||
rkllm:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
build: .
|
||||
platform: linux/arm64/v8
|
||||
container_name: rkllm
|
||||
restart: always
|
||||
privileged: true
|
||||
volumes:
|
||||
- ./app:/app
|
||||
- ./cache/cache:/root/.cache
|
||||
- ./cache/site-packages:/usr/local/lib/python3.11/site-packages
|
||||
- ./offline_packages:/offline_packages
|
||||
- ./ezrknn-llm:/ezrknn-llm
|
||||
entrypoint: ./app/entrypoint.sh
|
||||
- ./models:/app/models:ro
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 16384
|
||||
hard: 1048576
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
@@ -22,8 +19,4 @@ networks:
|
||||
apps:
|
||||
name: apps
|
||||
driver: bridge
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.100.0.0/24"
|
||||
gateway: "172.100.0.1"
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
gradio==5.14.0
|
||||
gradio_client==1.7.0
|
||||
huggingface-hub==0.26.2
|
||||
Jinja2==3.1.4
|
||||
numpy==2.1.3
|
||||
transformers==4.46.2
|
||||
trimesh==4.5.2
|
||||
sentencepiece==0.2.0
|
||||
8
requirements.txt
Executable file
8
requirements.txt
Executable file
@@ -0,0 +1,8 @@
|
||||
gradio==5.25.0
|
||||
gradio_client>=1.7.0
|
||||
huggingface-hub==0.28.1
|
||||
Jinja2==3.1.4
|
||||
numpy==2.1.3
|
||||
transformers==4.46.2
|
||||
trimesh==4.5.2
|
||||
sentencepiece==0.2.0
|
||||
Reference in New Issue
Block a user