diff --git a/app/config.py b/app/config.py index 4183633..d9d90fd 100644 --- a/app/config.py +++ b/app/config.py @@ -12,7 +12,7 @@ class Config(BaseSettings): MAX_CHUNK_SIZE: int = 2048 CHUNK_OVERLAP: int = 50 LM_MODEL_NAME: str = "/models/Qwen3-Embedding-0.6B" - LOCAL_LLM_NAME: str = "/models/Qwen3-8B" + LOCAL_LLM_NAME: str = "/models/Qwen3-4B" QWEN_MODEL_NAME: str = "Qwen3-8B" diff --git a/app/main.py b/app/main.py index 4d23a6e..4c14656 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,7 @@ import gradio as gr import torch import asyncio import threading +from transformers import AutoTokenizer as HFTokenizer from loguru import logger from langchain_chroma import Chroma from langchain_huggingface import HuggingFaceEmbeddings @@ -31,13 +32,13 @@ class ChatWithAI: tokenizer_mode="auto", trust_remote_code=True, dtype=torch.float16, # float16 - gpu_memory_utilization=0.85, - max_model_len=16384, + gpu_memory_utilization=0.5, + max_model_len=32768, enable_prefix_caching=True, tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU ) self.engine = AsyncLLMEngine.from_engine_args(engine_args) - self.tokenizer = self.engine.engine.tokenizer.tokenizer # доступ к токенизатору + self.tokenizer = HFTokenizer.from_pretrained(settings.LM_MODEL_NAME, trust_remote_code=True) else: raise ValueError(f"Неподдерживаемый провайдер: {provider}")