From 82b7be53e697bd9d9aa3d31cd58d22e7e8acc2f7 Mon Sep 17 00:00:00 2001
From: drholy <dr.holyblack@gmail.com>
Date: Mon, 4 Aug 2025 02:38:23 +0000
Subject: [PATCH] this is work. First prompt

---
 app/config.py | 2 +-
 app/main.py   | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/config.py b/app/config.py
index 4183633..d9d90fd 100644
--- a/app/config.py
+++ b/app/config.py
@@ -12,7 +12,7 @@ class Config(BaseSettings):
     MAX_CHUNK_SIZE: int = 2048
     CHUNK_OVERLAP: int = 50
     LM_MODEL_NAME: str = "/models/Qwen3-Embedding-0.6B"
-    LOCAL_LLM_NAME: str = "/models/Qwen3-8B"
+    LOCAL_LLM_NAME: str = "/models/Qwen3-4B"
     QWEN_MODEL_NAME: str = "Qwen3-8B"
 
 
diff --git a/app/main.py b/app/main.py
index 4d23a6e..4c14656 100644
--- a/app/main.py
+++ b/app/main.py
@@ -3,6 +3,7 @@ import gradio as gr
 import torch
 import asyncio
 import threading
+from transformers import AutoTokenizer as HFTokenizer
 from loguru import logger
 from langchain_chroma import Chroma
 from langchain_huggingface import HuggingFaceEmbeddings
@@ -31,13 +32,13 @@ class ChatWithAI:
                 tokenizer_mode="auto",
                 trust_remote_code=True,
                 dtype=torch.float16,  # float16
-                gpu_memory_utilization=0.85,
-                max_model_len=16384,
+                gpu_memory_utilization=0.5,
+                max_model_len=32768,
                 enable_prefix_caching=True,
                 tensor_parallel_size=1,  # Увеличь, если у тебя несколько GPU
             )
             self.engine = AsyncLLMEngine.from_engine_args(engine_args)
-            self.tokenizer = self.engine.engine.tokenizer.tokenizer  # доступ к токенизатору
+            self.tokenizer = HFTokenizer.from_pretrained(settings.LM_MODEL_NAME, trust_remote_code=True)
         else:
             raise ValueError(f"Неподдерживаемый провайдер: {provider}")