drholy/llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update main.py

Browse Source
This commit is contained in:
drholy
2025-08-04 21:42:50 +07:00
parent 4c4d2ae635
commit 4a6b8097f1
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/main.py
View File

@@ -31,8 +31,8 @@ class ChatWithAI:
tokenizer=model_name, tokenizer=model_name,
tokenizer_mode="auto", tokenizer_mode="auto",
trust_remote_code=True, trust_remote_code=True,
dtype=torch.float16, # float16 dtype=torch.bfloat16, # float16
gpu_memory_utilization=0.5, gpu_memory_utilization=0.65,
max_model_len=32768, max_model_len=32768,
enable_prefix_caching=True, enable_prefix_caching=True,
tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU tensor_parallel_size=1, # Увеличь, если у тебя несколько GPU