Compare commits
11 Commits
c45c367982
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 17cf7a6a55 | |||
| a66f257fed | |||
| 7a4470ab95 | |||
| e6f2ac2195 | |||
| 35d0a9db6d | |||
| d484043db5 | |||
| 18e2dcd592 | |||
| 5a779f44a9 | |||
| 8782400d7e | |||
| d9126af9d7 | |||
| f9a81c0752 |
@@ -3,7 +3,7 @@ name: Build, Push and Redeploy Docker Image
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- master
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-push:
|
build-and-push:
|
||||||
@@ -23,8 +23,15 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
- name: Log in to Gitea Container Registry
|
- name: Log in to Gitea Container Registry
|
||||||
run: |
|
uses: docker/login-action@v3
|
||||||
echo "${{ secrets.GIT_PASSWORD }}" | docker login "${{ env.REGISTRY_HOST }}" -u "${{ github.actor }}" --password-stdin
|
with:
|
||||||
|
registry: ${{ env.REGISTRY_HOST }}
|
||||||
|
username: ${{ gitea.actor }}
|
||||||
|
password: ${{ secrets.GIT_PASSWORD }}
|
||||||
|
|
||||||
|
# - name: Log in to Gitea Container Registry
|
||||||
|
# run: |
|
||||||
|
# echo "${{ secrets.GIT_PASSWORD }}" | docker login "${{ env.REGISTRY_HOST }}" -u "${{ github.actor }}" --password-stdin
|
||||||
|
|
||||||
- name: Get short SHA
|
- name: Get short SHA
|
||||||
id: vars
|
id: vars
|
||||||
@@ -41,6 +48,6 @@ jobs:
|
|||||||
--push \
|
--push \
|
||||||
.
|
.
|
||||||
|
|
||||||
# - name: Redeploy stack in Portainer
|
- name: Redeploy stack in Portainer
|
||||||
# run: |
|
run: |
|
||||||
# curl -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
|
curl -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
|
||||||
40
Dockerfile
40
Dockerfile
@@ -16,15 +16,39 @@ RUN apt-get update && \
|
|||||||
cmake \
|
cmake \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Обновление pip и установка setuptools/wheel
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Кэшируем установку зависимостей Python
|
||||||
|
COPY requirements.txt .
|
||||||
RUN pip install --upgrade pip && \
|
RUN pip install --upgrade pip && \
|
||||||
pip install setuptools wheel
|
pip install setuptools wheel && \
|
||||||
|
pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
# Проверка версии Python и pip
|
# Сборка RKLLM runtime
|
||||||
RUN python --version && pip --version
|
RUN git clone --depth 1 https://git.byte-mate.ru/drholy/ezrknn-llm /tmp/rkllm-build && \
|
||||||
|
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib && \
|
||||||
|
cp -r /tmp/rkllm-build/rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include && \
|
||||||
|
cd /tmp/rkllm-build/rkllm-runtime/examples/rkllm_api_demo && \
|
||||||
|
bash build-linux.sh && \
|
||||||
|
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm && \
|
||||||
|
# Очистка временных файлов сборки
|
||||||
|
ldconfig && \
|
||||||
|
rm -rf /tmp/rkllm-build && \
|
||||||
|
apt-get purge -y git cmake build-essential libssl-dev && \
|
||||||
|
apt-get autoremove -y && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||||
|
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
# Копируем приложение (исключая кэш и модели)
|
||||||
|
COPY app/ .
|
||||||
|
|
||||||
# Очистка кэша
|
# Исключаем модели из образа — монтируем как volume при запуске
|
||||||
RUN apt-get clean && \
|
RUN rm -rf /app/models/* 2>/dev/null || true && \
|
||||||
rm -rf /tmp/* /var/tmp/*
|
find /app -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
|
||||||
|
chmod +x /app/entrypoint.sh
|
||||||
|
|
||||||
|
# Порт Gradio по умолчанию
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||||
@@ -1,34 +1,18 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
message_print() {
|
set -e # Автоматический выход при ошибке
|
||||||
echo
|
|
||||||
echo "#########################################"
|
|
||||||
echo $1
|
|
||||||
echo "#########################################"
|
|
||||||
echo
|
|
||||||
}
|
|
||||||
|
|
||||||
cd /
|
echo
|
||||||
|
echo "#########################################"
|
||||||
|
echo "Starting RKLLM Gradio Server"
|
||||||
|
echo "#########################################"
|
||||||
|
echo
|
||||||
|
|
||||||
message_print "Changing to repository..."
|
# Проверка наличия моделей (монтируются через volume)
|
||||||
git clone https://git.home/drholy/ezrknn-llm
|
if [ ! -d "/app/models" ] || [ -z "$(ls -A /app/models 2>/dev/null)" ]; then
|
||||||
cd ezrknn-llm/
|
echo "⚠️ WARNING: Models directory is empty!"
|
||||||
cp ./rkllm-runtime/runtime/Linux/librkllm_api/aarch64/* /usr/lib
|
echo " Mount models volume: -v ./models:/app/models"
|
||||||
cp ./rkllm-runtime/runtime/Linux/librkllm_api/include/* /usr/local/include
|
sleep 5
|
||||||
|
fi
|
||||||
|
|
||||||
message_print "Compiling LLM runtime for Linux..."
|
# Запуск сервера
|
||||||
cd ./rkllm-runtime/examples/rkllm_api_demo/
|
exec python rkllm_server_gradio.py
|
||||||
bash build-linux.sh
|
|
||||||
|
|
||||||
message_print "Moving rkllm to /usr/bin..."
|
|
||||||
cp ./build/build_linux_aarch64_Release/llm_demo /usr/bin/rkllm
|
|
||||||
echo "* soft nofile 16384" >> /etc/security/limits.conf
|
|
||||||
echo "* hard nofile 1048576" >> /etc/security/limits.conf
|
|
||||||
|
|
||||||
message_print "Increasing file limit for all users (needed for LLMs to run)..."
|
|
||||||
echo "root soft nofile 16384" >> /etc/security/limits.conf
|
|
||||||
echo "root hard nofile 1048576" >> /etc/security/limits.conf
|
|
||||||
message_print "Done installing ezrknn-llm!"
|
|
||||||
|
|
||||||
message_print "Start Gradio server"
|
|
||||||
cd /app
|
|
||||||
python rkllm_server_gradio.py
|
|
||||||
@@ -15,36 +15,9 @@ model_configs = {
|
|||||||
"Qwen2.5-3B-Instruct-w8w8": {"filename": "Qwen2.5-3B-Instruct-w8w8.rkllm"}
|
"Qwen2.5-3B-Instruct-w8w8": {"filename": "Qwen2.5-3B-Instruct-w8w8.rkllm"}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": {
|
"Qwen3-4B-Instruct-2507-F16": {
|
||||||
"base_config": {
|
"base_config": {
|
||||||
"st_model_id": "Qwen/Qwen2.5-14B-Instruct",
|
"st_model_id": "unsloth/Qwen3-4B-Instruct-2507-GGUF",
|
||||||
"max_context_len": 128000,
|
|
||||||
"max_new_tokens": 8192,
|
|
||||||
"top_k": 5,
|
|
||||||
"top_p": 0.8,
|
|
||||||
"temperature": 0.2,
|
|
||||||
"repeat_penalty": 1.00,
|
|
||||||
"frequency_penalty": 0.2,
|
|
||||||
"system_prompt": """
|
|
||||||
Ты — эксперт по программированию, и твоя задача — генерировать только рабочий код без каких-либо комментариев, объяснений или дополнительного текста. Вывод должен содержать исключительно код, который можно скопировать и использовать напрямую.
|
|
||||||
|
|
||||||
<описание задачи>
|
|
||||||
|
|
||||||
Пример:
|
|
||||||
Если требуется написать функцию на Python, которая суммирует два числа, то ответ должен быть таким:
|
|
||||||
```
|
|
||||||
def sum(a, b):
|
|
||||||
return a + b
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
},
|
|
||||||
"models": {
|
|
||||||
"Qwen2.5-Coder-3B-Instruct-w8w8-For-clear-code": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"Qwen2.5-Coder-3B-Instruct-w8w8": {
|
|
||||||
"base_config": {
|
|
||||||
"st_model_id": "Qwen/Qwen2.5-14B-Instruct",
|
|
||||||
"max_context_len": 128000,
|
"max_context_len": 128000,
|
||||||
"max_new_tokens": 8192,
|
"max_new_tokens": 8192,
|
||||||
"top_k": 5,
|
"top_k": 5,
|
||||||
@@ -55,7 +28,7 @@ model_configs = {
|
|||||||
"system_prompt": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"
|
"system_prompt": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
"Qwen2.5-Coder-3B-Instruct-w8w8": {"filename": "Qwen2.5-Coder-3B-Instruct-w8w8.rkllm"}
|
"Qwen3-4B-Instruct-2507-F16": {"filename": "Qwen3-4B-Instruct-2507-F16.rkllm"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# "LLaMA-Mesh": {
|
# "LLaMA-Mesh": {
|
||||||
|
|||||||
20
docker-compose-prod.yml
Normal file
20
docker-compose-prod.yml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
services:
|
||||||
|
rkllm:
|
||||||
|
image: git.home/drholy/rkllm-gradio-server:latest
|
||||||
|
container_name: rkllm
|
||||||
|
restart: always
|
||||||
|
privileged: true
|
||||||
|
volumes:
|
||||||
|
- ./models:/app/models:ro
|
||||||
|
ulimits:
|
||||||
|
nofile:
|
||||||
|
soft: 16384
|
||||||
|
hard: 1048576
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
networks:
|
||||||
|
- apps
|
||||||
|
networks:
|
||||||
|
apps:
|
||||||
|
name: apps
|
||||||
|
driver: bridge
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
services:
|
services:
|
||||||
rkllm:
|
rkllm:
|
||||||
build:
|
build: .
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
platform: linux/arm64/v8
|
platform: linux/arm64/v8
|
||||||
container_name: rkllm
|
container_name: rkllm
|
||||||
restart: always
|
restart: always
|
||||||
privileged: true
|
privileged: true
|
||||||
volumes:
|
volumes:
|
||||||
- ./app:/app
|
- ./models:/app/models:ro
|
||||||
- ./ezrknn-llm:/ezrknn-llm
|
ulimits:
|
||||||
entrypoint: ./app/entrypoint.sh
|
nofile:
|
||||||
|
soft: 16384
|
||||||
|
hard: 1048576
|
||||||
ports:
|
ports:
|
||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
networks:
|
networks:
|
||||||
|
|||||||
Reference in New Issue
Block a user