diff --git a/app/main.py b/app/main.py index 3432cd8..95f615d 100644 --- a/app/main.py +++ b/app/main.py @@ -31,8 +31,8 @@ class ChatWithAI: tokenizer=model_name, tokenizer_mode="auto", trust_remote_code=True, - dtype=torch.float16, # float16 - gpu_memory_utilization=0.5, + dtype=torch.bfloat16, # float16 + gpu_memory_utilization=0.65, max_model_len=32768, enable_prefix_caching=True, tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU