add support for GPU (cuda)

2023-10-27 10:23:08 +02:00
parent f4484884cc
commit 5570a745f8
7 changed files with 170 additions and 22 deletions
--- a/backend/api/handlers.go
+++ b/backend/api/handlers.go
@@ -87,8 +87,11 @@ func (s *Server) handlePostTranscription(c *fiber.Ctx) error {
 	transcription.Status = models.TranscriptionStatusPending
 	transcription.Task = "transcribe"
 	transcription.SourceUrl = c.FormValue("sourceUrl")
-	// TODO: Implement other device types
-	transcription.Device = "cpu"
+	transcription.Device = c.FormValue("device")
+	if transcription.Device != "cpu" && transcription.Device != "cuda" {
+		log.Warn().Msgf("Device %v not supported, using cpu", transcription.Device)
+		transcription.Device = "cpu"
+	}

 	log.Debug().Msgf("Transcription: %+v", transcription)
 	// Save transcription to database
--- a/docker-compose.ferret.yml
+++ b/docker-compose.ferret.yml
@@ -90,7 +90,8 @@ services:
    environment:
      PUBLIC_INTERNAL_API_HOST: "http://nginx:80"
      PUBLIC_TRANSLATION_API_HOST: ""
-      PUBLIC_API_HOST: ${WHISHPER_PUBLIC_URL:-}
+      PUBLIC_API_HOST: ${WHISHPER_HOST:-}
+      PUBLIC_WHISHPER_GPU: ${WHISHPER_GPU:-false}
    depends_on:
      transcription-api:
        condition: service_healthy
--- a/docker-compose.gpu.yml
+++ b/docker-compose.gpu.yml
@@ -0,0 +1,119 @@
+version: "3.9"
+
+services:
+  mongo:
+    image: mongo
+    env_file:
+      - .env
+    restart: unless-stopped
+    volumes:
+      - ./whishper_data/db_data:/data/db
+      - ./whishper_data/db_data/logs/:/var/log/mongodb/
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${DB_USER:-whishper}
+      MONGO_INITDB_ROOT_PASSWORD: ${DB_PASS:-whishper}
+    expose:
+      - 27017
+    command: ['--logpath', '/var/log/mongodb/mongod.log']
+    healthcheck:
+      test: mongosh --eval 'db.runCommand("ping").ok' --quiet
+      interval: 2s
+      timeout: 3s
+      retries: 5
+
+  translate:
+    container_name: whisper-libretranslate
+    image: libretranslate/libretranslate:latest
+    restart: unless-stopped
+    volumes:
+      - ./whishper_data/libretranslate/data:/home/libretranslate/.local/share
+      - ./whishper_data/libretranslate/cache:/home/libretranslate/.local/cache
+    env_file:
+      - .env
+    tty: true
+    environment:
+      LT_DISABLE_WEB_UI: True
+      #LT_UPDATE_MODELS: True
+    expose:
+      - 5000
+    healthcheck:
+      test: ['CMD-SHELL', './venv/bin/python scripts/healthcheck.py']
+      interval: 15s
+      timeout: 10s
+      start_period: 5s
+      retries: 5
+
+  transcription-api:
+    container_name: whisper-api
+    image: pluja/whishper-transcription-api:latest-gpu
+    env_file:
+      - .env
+    volumes:
+      - ./whishper_data/whisper_models:/app/models
+      - ./whishper_data/uploads:/app/uploads
+    environment:
+      - WHISPER_MODELS_DIR=/app/models
+      - UPLOAD_DIR=/app/uploads
+    restart: unless-stopped
+    expose:
+      - 8000
+    deploy:
+      resources:
+        reservations:
+          devices:
+          - driver: nvidia
+            count: all
+            capabilities: [gpu]
+
+  whishper-backend:
+    container_name: whishper-backend
+    image: pluja/whishper-backend:latest
+    env_file:
+      - .env
+    volumes:
+      - ./whishper_data/uploads:/app/uploads
+    restart: unless-stopped
+    environment:
+      - FERRETDB_MONGO_URL=mongodb://${DB_USER:-whishper}:${DB_PASS:-whishper}@mongo:27017/?authMechanism=PLAIN
+    depends_on:
+      translate:
+        condition: service_healthy
+      transcription-api:
+        condition: service_healthy
+    healthcheck:
+      test: curl --fail http://localhost:8080/api/transcriptions || exit 1
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 5s
+    expose:
+      - 8080
+
+  whishper-frontend:
+    container_name: whishper-web
+    image: pluja/whishper-frontend:latest
+    env_file:
+      - .env
+    restart: unless-stopped
+    environment:
+      PUBLIC_INTERNAL_API_HOST: "http://nginx:80"
+      PUBLIC_TRANSLATION_API_HOST: ""
+      PUBLIC_API_HOST: ${WHISHPER_HOST:-}
+      PUBLIC_WHISHPER_GPU: ${WHISHPER_GPU:-true}
+    depends_on:
+      transcription-api:
+        condition: service_healthy
+    expose:
+      - 3000
+
+  nginx:
+    image: nginx:1
+    restart: unless-stopped
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./whishper_data/uploads:/app/uploads
+    depends_on:
+      whishper-backend:
+        condition: service_healthy
+    ports:
+      - 8082:80    
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,10 +17,9 @@ services:
    command: ['--logpath', '/var/log/mongodb/mongod.log']
    healthcheck:
      test: mongosh --eval 'db.runCommand("ping").ok' --quiet
-      interval: 15s
-      timeout: 10s
+      interval: 2s
+      timeout: 3s
      retries: 5
-      start_period: 5s

  translate:
    container_name: whisper-libretranslate
@@ -39,9 +38,8 @@ services:
      - 5000
    healthcheck:
      test: ['CMD-SHELL', './venv/bin/python scripts/healthcheck.py']
-      interval: 15s
-      timeout: 10s
-      start_period: 5s
+      interval: 2s
+      timeout: 3s
      retries: 5

  transcription-api:
@@ -76,10 +74,9 @@ services:
        condition: service_healthy
    healthcheck:
      test: curl --fail http://localhost:8080/api/transcriptions || exit 1
-      interval: 15s
-      timeout: 10s
+      interval: 2s
+      timeout: 2s
      retries: 5
-      start_period: 5s
    expose:
      - 8080

@@ -90,9 +87,10 @@ services:
      - .env
    restart: unless-stopped
    environment:
-      INTERNAL_API_HOST: "http://nginx:80"
-      VITE_TRANSLATION_API_HOST: ""
-      VITE_API_HOST: ${WHISHPER_HOST:-}
+      PUBLIC_INTERNAL_API_HOST: "http://nginx:80"
+      PUBLIC_TRANSLATION_API_HOST: ""
+      PUBLIC_API_HOST: ${WHISHPER_HOST:-}
+      PUBLIC_WHISHPER_GPU: ${WHISHPER_GPU:-false}
    depends_on:
      transcription-api:
        condition: service_healthy
--- a/example.env
+++ b/example.env
@@ -1,8 +1,11 @@
 # Libretranslate Configuration
-LT_LOAD_ONLY=es,en,fr
+LT_LOAD_ONLY=es,ja,fr

 # Whisper Configuration
 WHISPER_MODELS=tiny,small
-WHISHPER_HOST=localhost:8082
+WHISHPER_HOST=127.0.0.1:8082
+WHISHPER_GPU=false # true if you have cuda support, read https://whishper.net/guides/gpu
+
+# Database Configuration
 DB_USER=whishper
 DB_PASS=whishper
--- a/frontend/src/lib/components/ModalTranscriptionForm.svelte
+++ b/frontend/src/lib/components/ModalTranscriptionForm.svelte
@@ -1,5 +1,5 @@
 <script>
-	import { validateURL, CLIENT_API_HOST } from '$lib/utils.js';
+	import { validateURL, CLIENT_API_HOST, WHISHPER_GPU } from '$lib/utils.js';

 	import toast from 'svelte-french-toast';

@@ -9,6 +9,7 @@
 	let language = 'auto';
 	let sourceUrl = '';
 	let fileInput;
+	let device = (WHISHPER_GPU) ? 'cuda' : 'cpu';

 	let languages = [
 		'auto',
@@ -71,6 +72,11 @@
 		let formData = new FormData();
 		formData.append('language', language);
 		formData.append('modelSize', modelSize);
+		if (device == 'cuda' || device == 'cpu') {
+			formData.append('device', device);
+		} else {
+			formData.append('device', 'cpu');
+		}
 		formData.append('sourceUrl', sourceUrl);
 		if (sourceUrl == '') {
 			formData.append('file', fileInput.files[0]);
@@ -169,6 +175,21 @@
 					{/each}
 				</select>
 			</div>
+
+			<div class="w-full max-w-xs form-control">
+				<label for="language" class="label">
+					<span class="label-text">Device</span>
+				</label>
+				<select name="device" bind:value={device} class="select select-bordered">
+					{#if WHISHPER_GPU }
+						<option selected value="cuda">GPU</option>
+						<option value="cpu">CPU</option>
+					{:else}
+						<option selected value="cpu">CPU</option>
+						<option disabled value="cuda">GPU</option>
+					{/if}
+				</select>
+			</div>
 		</div>

 		<div class="mb-0 divider" />
--- a/frontend/src/lib/utils.js
+++ b/frontend/src/lib/utils.js
@@ -1,11 +1,14 @@
 import {transcriptions} from './stores';
 import { dev } from '$app/environment';
 import { browser } from '$app/environment';
+import { env } from '$env/dynamic/public';

-export let CLIENT_API_HOST = browser ? `${dev ? import.meta.env.VITE_API_HOST : ""}` : `${process.env.INTERNAL_API_HOST}`;
-export let CLIENT_WS_HOST = browser ? `${dev ? import.meta.env.VITE_API_HOST.replace("http://", "").replace("https://", "") : ""}` :  `${dev ? process.env.INTERNAL_API_HOST.replace("http://", "").replace("https://", "") : ""}`;
-export let TRANSLATION_API_HOST = browser ? `${dev ? import.meta.env.VITE_TRANSLATION_API_HOST : ""}` : `${dev ? process.env.VITE_TRANSLATION_API_HOST : ""}`;
-
+export let CLIENT_API_HOST = browser ? `${dev ? env.PUBLIC_API_HOST : ""}` : `${env.PUBLIC_INTERNAL_API_HOST}`;
+export let CLIENT_WS_HOST = browser ? `${dev ? env.PUBLIC_API_HOST.replace("http://", "").replace("https://", "") : ""}` :  `${dev ? env.PUBLIC_INTERNAL_API_HOST.replace("http://", "").replace("https://", "") : ""}`;
+export let TRANSLATION_API_HOST = browser ? `${dev ? env.PUBLIC_TRANSLATION_API_HOST : ""}` : `${dev ? env.PUBLIC_TRANSLATION_API_HOST : ""}`;
+export let WHISHPER_GPU = env.PUBLIC_WHISHPER_GPU === 'true' ||
+                          env.PUBLIC_WHISHPER_GPU === 'True' ||
+                          env.PUBLIC_WHISHPER_GPU === '1';
 // URL Validator
 export const validateURL = function (url) {
    try {