drholy/llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

this is work. First prompt

Browse Source
This commit is contained in:
drholy
2025-08-04 02:38:23 +00:00
parent 711ad223a5
commit 82b7be53e6
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/config.py
View File

@@ -12,7 +12,7 @@ class Config(BaseSettings):
MAX_CHUNK_SIZE: int = 2048 MAX_CHUNK_SIZE: int = 2048
CHUNK_OVERLAP: int = 50 CHUNK_OVERLAP: int = 50
LM_MODEL_NAME: str = "/models/Qwen3-Embedding-0.6B" LM_MODEL_NAME: str = "/models/Qwen3-Embedding-0.6B"
LOCAL_LLM_NAME: str = "/models/Qwen3-8B" LOCAL_LLM_NAME: str = "/models/Qwen3-4B"
QWEN_MODEL_NAME: str = "Qwen3-8B" QWEN_MODEL_NAME: str = "Qwen3-8B"

7
app/main.py
View File

@@ -3,6 +3,7 @@ import gradio as gr
import torch import torch
import asyncio import asyncio
import threading import threading
from transformers import AutoTokenizer as HFTokenizer
from loguru import logger from loguru import logger
from langchain_chroma import Chroma from langchain_chroma import Chroma
from langchain_huggingface import HuggingFaceEmbeddings from langchain_huggingface import HuggingFaceEmbeddings
@@ -31,13 +32,13 @@ class ChatWithAI:
tokenizer_mode="auto", tokenizer_mode="auto",
trust_remote_code=True, trust_remote_code=True,
dtype=torch.float16, # float16 dtype=torch.float16, # float16
gpu_memory_utilization=0.85, gpu_memory_utilization=0.5,
max_model_len=16384, max_model_len=32768,
enable_prefix_caching=True, enable_prefix_caching=True,
tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU
) )
self.engine = AsyncLLMEngine.from_engine_args(engine_args) self.engine = AsyncLLMEngine.from_engine_args(engine_args)
self.tokenizer = self.engine.engine.tokenizer.tokenizer # доступ к токенизатору self.tokenizer = HFTokenizer.from_pretrained(settings.LM_MODEL_NAME, trust_remote_code=True)
else: else:
raise ValueError(f"Неподдерживаемый провайдер: {provider}") raise ValueError(f"Неподдерживаемый провайдер: {provider}")