update main.py

2025-08-04 21:42:50 +07:00
parent 4c4d2ae635
commit 4a6b8097f1
1 changed files with 2 additions and 2 deletions
--- a/app/main.py
+++ b/app/main.py
@@ -31,8 +31,8 @@ class ChatWithAI:
                tokenizer=model_name,
                tokenizer_mode="auto",
                trust_remote_code=True,
-                dtype=torch.float16,  # float16
-                gpu_memory_utilization=0.5,
+                dtype=torch.bfloat16,  # float16
+                gpu_memory_utilization=0.65,
                max_model_len=32768,
                enable_prefix_caching=True,
                tensor_parallel_size=1,  # Увеличь, если у тебя несколько GPU