Compare commits

..

14 Commits

Author SHA1 Message Date
17cf7a6a55 update models
All checks were successful
Build, Push and Redeploy Docker Image / build-and-push (push) Successful in 5m18s
2026-02-04 18:59:03 +07:00
a66f257fed fix model id
All checks were successful
Build, Push and Redeploy Docker Image / build-and-push (push) Successful in 5m18s
2026-02-04 05:22:34 +07:00
7a4470ab95 add new model in list
All checks were successful
Build, Push and Redeploy Docker Image / build-and-push (push) Successful in 4m43s
2026-02-04 04:37:56 +07:00
e6f2ac2195 add production docker compose 2026-02-03 21:35:33 +07:00
35d0a9db6d fix 3
All checks were successful
Build, Push and Redeploy Docker Image / build-and-push (push) Successful in 7m16s
2026-02-03 21:06:11 +07:00
d484043db5 fix 2
Some checks failed
Build, Push and Redeploy Docker Image / build-and-push (push) Failing after 7m2s
2026-02-03 20:56:06 +07:00
18e2dcd592 fix 1
Some checks failed
Build, Push and Redeploy Docker Image / build-and-push (push) Failing after 6m34s
2026-02-03 20:45:55 +07:00
5a779f44a9 upgrade dockerfile
Some checks failed
Build, Push and Redeploy Docker Image / build-and-push (push) Failing after 3m45s
2026-02-03 20:40:21 +07:00
8782400d7e edit workflow file
Some checks failed
Build, Push and Redeploy Docker Image / build-and-push (push) Failing after 3m36s
2026-02-03 20:13:14 +07:00
d9126af9d7 edit action
Some checks failed
Build, Push and Redeploy Docker Image / build-and-push (push) Failing after 4m3s
2026-02-03 19:14:17 +07:00
f9a81c0752 edit workflows 2026-02-03 19:07:41 +07:00
c45c367982 add action 2026-02-03 18:47:19 +07:00
811e767bc5 delete action 2026-02-03 18:44:25 +07:00
d736fe838a turn to work version 2026-02-03 18:31:42 +07:00
10 changed files with 148 additions and 101 deletions

View File

@@ -0,0 +1,53 @@
name: Build, Push and Redeploy Docker Image
on:
push:
branches:
- master
jobs:
build-and-push:
runs-on: ubuntu-latest
env:
REGISTRY_HOST: git.byte-mate.ru
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up QEMU (for multi-arch, optional)
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Gitea Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY_HOST }}
username: ${{ gitea.actor }}
password: ${{ secrets.GIT_PASSWORD }}
# - name: Log in to Gitea Container Registry
# run: |
# echo "${{ secrets.GIT_PASSWORD }}" | docker login "${{ env.REGISTRY_HOST }}" -u "${{ github.actor }}" --password-stdin
- name: Get short SHA
id: vars
run: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
- name: Build and push
run: |
docker buildx build \
--no-cache \
--push \
--platform linux/arm64/v8 \
--tag ${{ env.REGISTRY_HOST }}/drholy/rkllm-gradio-server:${{ env.sha_short }} \
--tag ${{ env.REGISTRY_HOST }}/drholy/rkllm-gradio-server:latest \
--push \
.
- name: Redeploy stack in Portainer
run: |
curl -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"

View File

@@ -2,7 +2,6 @@
FROM python:3.11.11 FROM python:3.11.11
# Установка метаданных # Установка метаданных
LABEL maintainer="your-email@example.com"
LABEL description="Docker image with Python 3.11.11 and required packages" LABEL description="Docker image with Python 3.11.11 and required packages"
# Обновление системы и установка базовых зависимостей # Обновление системы и установка базовых зависимостей
@@ -17,13 +16,39 @@ RUN apt-get update && \
cmake \ cmake \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Обновление pip и установка setuptools/wheel WORKDIR /app
# Кэшируем установку зависимостей Python
COPY requirements.txt .
RUN pip install --upgrade pip && \ RUN pip install --upgrade pip && \
pip install setuptools wheel pip install setuptools wheel && \
pip install --no-cache-dir -r requirements.txt
# Проверка версии Python и pip # Сборка RKLLM runtime
RUN python --version && pip --version RUN git clone --depth 1 https://git.byte-mate.ru/drholy/ezrknn-llm /tmp/rkllm-build && \
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib && \
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include && \
cd /tmp/rkllm-build/rkllm-runtime/examples/rkllm_api_demo && \
bash build-linux.sh && \
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm && \
# Очистка временных файлов сборки
ldconfig && \
rm -rf /tmp/rkllm-build && \
apt-get purge -y git cmake build-essential libssl-dev && \
apt-get autoremove -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Очистка кэша # Копируем приложение (исключая кэш и модели)
RUN apt-get clean && \ COPY app/ .
rm -rf /tmp/* /var/tmp/*
# Исключаем модели из образа — монтируем как volume при запуске
RUN rm -rf /app/models/* 2>/dev/null || true && \
find /app -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
chmod +x /app/entrypoint.sh
# Порт Gradio по умолчанию
EXPOSE 8080
WORKDIR /app
ENTRYPOINT ["/app/entrypoint.sh"]

View File

@@ -53,9 +53,6 @@ http://localhost:8080
Проект использует следующие виртуальные файловые системы: Проект использует следующие виртуальные файловые системы:
- `/app` — для приложения. - `/app` — для приложения.
- `/cache/cache` — для кэша.
- `/cache/site-packages` — для пакетов Python.
- `/offline_packages` — для офлайн пакетов.
- `/ezrknn-llm` — для дополнительных библиотек. - `/ezrknn-llm` — для дополнительных библиотек.
## Настройка и использование моделей ## Настройка и использование моделей

View File

@@ -1,38 +1,18 @@
#!/bin/bash #!/bin/bash
message_print() { set -e # Автоматический выход при ошибке
echo
echo "#########################################"
echo $1
echo "#########################################"
echo
}
cd / echo
echo "#########################################"
echo "Starting RKLLM Gradio Server"
echo "#########################################"
echo
message_print "Changing to repository..." # Проверка наличия моделей (монтируются через volume)
git clone https://github.com/DrHo1y/ezrknn-llm if [ ! -d "/app/models" ] || [ -z "$(ls -A /app/models 2>/dev/null)" ]; then
cd ezrknn-llm/ echo "⚠️ WARNING: Models directory is empty!"
cp ./rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib echo " Mount models volume: -v ./models:/app/models"
cp ./rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include sleep 5
fi
message_print "Compiling LLM runtime for Linux..." # Запуск сервера
cd ./rkllm-runtime/examples/rkllm_api_demo/ exec python rkllm_server_gradio.py
bash build-linux.sh
message_print "Moving rkllm to /usr/bin..."
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm
echo "* soft nofile 16384" >> /etc/security/limits.conf
echo "* hard nofile 1048576" >> /etc/security/limits.conf
message_print "Increasing file limit for all users (needed for LLMs to run)..."
echo "root soft nofile 16384" >> /etc/security/limits.conf
echo "root hard nofile 1048576" >> /etc/security/limits.conf
message_print "Done installing ezrknn-llm!"
message_print "Install python packages"
cd /offline_packages
python -m pip install --no-index --find-links . -r requirements.txt
message_print "Start Gradio server"
cd /app
python rkllm_server_gradio.py

View File

@@ -15,36 +15,9 @@ model_configs = {
"Qwen2.5-3B-Instruct-w8w8": {"filename": "Qwen2.5-3B-Instruct-w8w8.rkllm"} "Qwen2.5-3B-Instruct-w8w8": {"filename": "Qwen2.5-3B-Instruct-w8w8.rkllm"}
} }
}, },
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": { "Qwen3-4B-Instruct-2507-F16": {
"base_config": { "base_config": {
"st_model_id": "Qwen/Qwen2.5-14B-Instruct", "st_model_id": "unsloth/Qwen3-4B-Instruct-2507-GGUF",
"max_context_len": 128000,
"max_new_tokens": 8192,
"top_k": 5,
"top_p": 0.8,
"temperature": 0.2,
"repeat_penalty": 1.00,
"frequency_penalty": 0.2,
"system_prompt": """
Ты — эксперт по программированию, и твоя задача — генерировать только рабочий код без каких-либо комментариев, объяснений или дополнительного текста. Вывод должен содержать исключительно код, который можно скопировать и использовать напрямую.
<описание задачи>
Пример:
Если требуется написать функцию на Python, которая суммирует два числа, то ответ должен быть таким:
```
def sum(a, b):
return a + b
```
"""
},
"models": {
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"}
}
},
"Qwen2.5-Coder-3B-Instruct-w8w8": {
"base_config": {
"st_model_id": "Qwen/Qwen2.5-14B-Instruct",
"max_context_len": 128000, "max_context_len": 128000,
"max_new_tokens": 8192, "max_new_tokens": 8192,
"top_k": 5, "top_k": 5,
@@ -55,7 +28,7 @@ model_configs = {
"system_prompt": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant" "system_prompt": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"
}, },
"models": { "models": {
"Qwen2.5-Coder-3B-Instruct-w8w8": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"} "Qwen3-4B-Instruct-2507-F16": {"filename": "Qwen3-4B-Instruct-2507-F16.rkllm"}
} }
} }
# "LLaMA-Mesh": { # "LLaMA-Mesh": {

View File

@@ -6,8 +6,8 @@ from model_class import *
from mesh_utils import * from mesh_utils import *
# Set environment variables # Set environment variables
os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0" #os.environ["GRADIO_SERVER_NAME"] = "0.0.0.0"
os.environ["GRADIO_SERVER_PORT"] = "8080" #os.environ["GRADIO_SERVER_PORT"] = 8080
os.environ["RKLLM_LOG_LEVEL"] = "1" os.environ["RKLLM_LOG_LEVEL"] = "1"
# Set resource limit # Set resource limit
resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400)) resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))
@@ -29,7 +29,7 @@ if __name__ == "__main__":
print(init_msg) print(init_msg)
sys.stdout.flush() sys.stdout.flush()
rkllm_model = RKLLMLoaderClass(model=model) rkllm_model = RKLLMLoaderClass(model=model)
model_init = f"RKLLM Model, {rkllm_model.model_name} has been initialized successfully!" model_init = f"RKLLM Model, {rkllm_model.model_name} has been initialized successfully!"
print(model_init) print(model_init)
complete_init = "==============================" complete_init = "=============================="
print(complete_init) print(complete_init)
@@ -106,7 +106,13 @@ if __name__ == "__main__":
# Enable the event queue system. # Enable the event queue system.
chatRKLLM.queue() chatRKLLM.queue()
# Start the Gradio application. # Start the Gradio application.
chatRKLLM.launch() chatRKLLM.launch(
server_name="0.0.0.0",
server_port=8080,
share=False,
show_error=True,
quiet=False
)
print("====================") print("====================")
print("RKLLM model inference completed, releasing RKLLM model resources...") print("RKLLM model inference completed, releasing RKLLM model resources...")

20
docker-compose-prod.yml Normal file
View File

@@ -0,0 +1,20 @@
services:
rkllm:
image: git.home/drholy/rkllm-gradio-server:latest
container_name: rkllm
restart: always
privileged: true
volumes:
- ./models:/app/models:ro
ulimits:
nofile:
soft: 16384
hard: 1048576
ports:
- "8080:8080"
networks:
- apps
networks:
apps:
name: apps
driver: bridge

View File

@@ -1,19 +1,16 @@
services: services:
rkllm: rkllm:
build: build: .
context: .
dockerfile: Dockerfile
platform: linux/arm64/v8 platform: linux/arm64/v8
container_name: rkllm container_name: rkllm
restart: always restart: always
privileged: true privileged: true
volumes: volumes:
- ./app:/app - ./models:/app/models:ro
- ./cache/cache:/root/.cache ulimits:
- ./cache/site-packages:/usr/local/lib/python3.11/site-packages nofile:
- ./offline_packages:/offline_packages soft: 16384
- ./ezrknn-llm:/ezrknn-llm hard: 1048576
entrypoint: ./app/entrypoint.sh
ports: ports:
- "8080:8080" - "8080:8080"
networks: networks:
@@ -22,8 +19,4 @@ networks:
apps: apps:
name: apps name: apps
driver: bridge driver: bridge
ipam:
driver: default
config:
- subnet: "172.100.0.0/24"
gateway: "172.100.0.1"

View File

@@ -1,8 +0,0 @@
gradio==5.14.0
gradio_client==1.7.0
huggingface-hub==0.26.2
Jinja2==3.1.4
numpy==2.1.3
transformers==4.46.2
trimesh==4.5.2
sentencepiece==0.2.0

8
requirements.txt Executable file
View File

@@ -0,0 +1,8 @@
gradio==5.25.0
gradio_client>=1.7.0
huggingface-hub==0.28.1
Jinja2==3.1.4
numpy==2.1.3
transformers==4.46.2
trimesh==4.5.2
sentencepiece==0.2.0