Skip to content

Commit e426ab7

Browse files
mudlermudler
authored
feat(faster-whisper): add backend (#4666)
Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 715071b commit e426ab7

14 files changed

+196
-3
lines changed

‎Dockerfile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ ARG TARGETARCH
1515
ARG TARGETVARIANT
1616

1717
ENV DEBIAN_FRONTEND=noninteractive
18-
ENV EXTERNAL_GRPC_BACKENDS="coqui:/build/backend/python/coqui/run.sh,transformers:/build/backend/python/transformers/run.sh,rerankers:/build/backend/python/rerankers/run.sh,autogptq:/build/backend/python/autogptq/run.sh,bark:/build/backend/python/bark/run.sh,diffusers:/build/backend/python/diffusers/run.sh,openvoice:/build/backend/python/openvoice/run.sh,kokoro:/build/backend/python/kokoro/run.sh,vllm:/build/backend/python/vllm/run.sh,mamba:/build/backend/python/mamba/run.sh,exllama2:/build/backend/python/exllama2/run.sh,parler-tts:/build/backend/python/parler-tts/run.sh"
18+
ENV EXTERNAL_GRPC_BACKENDS="coqui:/build/backend/python/coqui/run.sh,transformers:/build/backend/python/transformers/run.sh,rerankers:/build/backend/python/rerankers/run.sh,autogptq:/build/backend/python/autogptq/run.sh,bark:/build/backend/python/bark/run.sh,diffusers:/build/backend/python/diffusers/run.sh,faster-whisper:/build/backend/python/faster-whisper/run.sh,openvoice:/build/backend/python/openvoice/run.sh,kokoro:/build/backend/python/kokoro/run.sh,vllm:/build/backend/python/vllm/run.sh,mamba:/build/backend/python/mamba/run.sh,exllama2:/build/backend/python/exllama2/run.sh,parler-tts:/build/backend/python/parler-tts/run.sh"
1919

2020

2121
RUN apt-get update && \
@@ -414,6 +414,9 @@ RUN if [[ ( "${EXTRA_BACKENDS}" =~ "coqui" || -z "${EXTRA_BACKENDS}" ) && "$IMAG
414414
if [[ ( "${EXTRA_BACKENDS}" =~ "parler-tts" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
415415
make -C backend/python/parler-tts \
416416
; fi && \
417+
if [[ ( "${EXTRA_BACKENDS}" =~ "faster-whisper" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
418+
make -C backend/python/parler-tts \
419+
; fi && \
417420
if [[ ( "${EXTRA_BACKENDS}" =~ "diffusers" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
418421
make -C backend/python/diffusers \
419422
; fi

‎Makefile

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -533,10 +533,10 @@ protogen-go-clean:
533533
$(RM) bin/*
534534

535535
.PHONY: protogen-python
536-
protogen-python: autogptq-protogen bark-protogen coqui-protogen diffusers-protogen exllama2-protogen mamba-protogen rerankers-protogen transformers-protogen parler-tts-protogen kokoro-protogen vllm-protogen openvoice-protogen
536+
protogen-python: autogptq-protogen bark-protogen coqui-protogen diffusers-protogen exllama2-protogen mamba-protogen rerankers-protogen transformers-protogen parler-tts-protogen kokoro-protogen vllm-protogen openvoice-protogen faster-whisper-protogen
537537

538538
.PHONY: protogen-python-clean
539-
protogen-python-clean: autogptq-protogen-clean bark-protogen-clean coqui-protogen-clean diffusers-protogen-clean exllama2-protogen-clean mamba-protogen-clean rerankers-protogen-clean transformers-protogen-clean parler-tts-protogen-clean kokoro-protogen-clean vllm-protogen-clean openvoice-protogen-clean
539+
protogen-python-clean: autogptq-protogen-clean bark-protogen-clean coqui-protogen-clean diffusers-protogen-clean exllama2-protogen-clean mamba-protogen-clean rerankers-protogen-clean transformers-protogen-clean parler-tts-protogen-clean kokoro-protogen-clean vllm-protogen-clean openvoice-protogen-clean faster-whisper-protogen-clean
540540

541541
.PHONY: autogptq-protogen
542542
autogptq-protogen:
@@ -570,6 +570,14 @@ diffusers-protogen:
570570
diffusers-protogen-clean:
571571
$(MAKE) -C backend/python/diffusers protogen-clean
572572

573+
.PHONY: faster-whisper-protogen
574+
faster-whisper-protogen:
575+
$(MAKE) -C backend/python/faster-whisper protogen
576+
577+
.PHONY: faster-whisper-protogen-clean
578+
faster-whisper-protogen-clean:
579+
$(MAKE) -C backend/python/faster-whisper protogen-clean
580+
573581
.PHONY: exllama2-protogen
574582
exllama2-protogen:
575583
$(MAKE) -C backend/python/exllama2 protogen
@@ -641,6 +649,7 @@ prepare-extra-conda-environments: protogen-python
641649
$(MAKE) -C backend/python/bark
642650
$(MAKE) -C backend/python/coqui
643651
$(MAKE) -C backend/python/diffusers
652+
$(MAKE) -C backend/python/faster-whisper
644653
$(MAKE) -C backend/python/vllm
645654
$(MAKE) -C backend/python/mamba
646655
$(MAKE) -C backend/python/rerankers

‎backend/python/faster-whisper/Makefile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
.DEFAULT_GOAL := install
2+
3+
.PHONY: install
4+
install:
5+
bash install.sh
6+
$(MAKE) protogen
7+
8+
.PHONY: protogen
9+
protogen: backend_pb2_grpc.py backend_pb2.py
10+
11+
.PHONY: protogen-clean
12+
protogen-clean:
13+
$(RM) backend_pb2_grpc.py backend_pb2.py
14+
15+
backend_pb2_grpc.py backend_pb2.py:
16+
bash protogen.sh
17+
18+
.PHONY: clean
19+
clean: protogen-clean
20+
rm -rf venv __pycache__

‎backend/python/faster-whisper/backend.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/usr/bin/env python3
2+
"""
3+
This is an extra gRPC server of LocalAI for Bark TTS
4+
"""
5+
from concurrent import futures
6+
import time
7+
import argparse
8+
import signal
9+
import sys
10+
import os
11+
import backend_pb2
12+
import backend_pb2_grpc
13+
14+
from faster_whisper import WhisperModel
15+
16+
import grpc
17+
18+
19+
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
20+
21+
# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
22+
MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
23+
COQUI_LANGUAGE = os.environ.get('COQUI_LANGUAGE', None)
24+
25+
# Implement the BackendServicer class with the service methods
26+
class BackendServicer(backend_pb2_grpc.BackendServicer):
27+
"""
28+
BackendServicer is the class that implements the gRPC service
29+
"""
30+
def Health(self, request, context):
31+
return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
32+
def LoadModel(self, request, context):
33+
device = "cpu"
34+
# Get device
35+
# device = "cuda" if request.CUDA else "cpu"
36+
if request.CUDA:
37+
device = "cuda"
38+
39+
try:
40+
print("Preparing models, please wait", file=sys.stderr)
41+
self.model = WhisperModel(request.Model, device=device, compute_type="float16")
42+
except Exception as err:
43+
return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
44+
# Implement your logic here for the LoadModel service
45+
# Replace this with your desired response
46+
return backend_pb2.Result(message="Model loaded successfully", success=True)
47+
48+
def AudioTranscription(self, request, context):
49+
resultSegments = []
50+
text = ""
51+
try:
52+
segments, info = self.model.transcribe(request.dst, beam_size=5, condition_on_previous_text=False)
53+
id = 0
54+
for segment in segments:
55+
print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
56+
resultSegments.append(backend_pb2.TranscriptSegment(id=id, start=segment.start, end=segment.end, text=segment.text))
57+
text += segment.text
58+
id += 1
59+
except Exception as err:
60+
print(f"Unexpected {err=}, {type(err)=}", file=sys.stderr)
61+
62+
return backend_pb2.TranscriptResult(segments=resultSegments, text=text)
63+
64+
def serve(address):
65+
server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS))
66+
backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
67+
server.add_insecure_port(address)
68+
server.start()
69+
print("Server started. Listening on: " + address, file=sys.stderr)
70+
71+
# Define the signal handler function
72+
def signal_handler(sig, frame):
73+
print("Received termination signal. Shutting down...")
74+
server.stop(0)
75+
sys.exit(0)
76+
77+
# Set the signal handlers for SIGINT and SIGTERM
78+
signal.signal(signal.SIGINT, signal_handler)
79+
signal.signal(signal.SIGTERM, signal_handler)
80+
81+
try:
82+
while True:
83+
time.sleep(_ONE_DAY_IN_SECONDS)
84+
except KeyboardInterrupt:
85+
server.stop(0)
86+
87+
if __name__ == "__main__":
88+
parser = argparse.ArgumentParser(description="Run the gRPC server.")
89+
parser.add_argument(
90+
"--addr", default="localhost:50051", help="The address to bind the server to."
91+
)
92+
args = parser.parse_args()
93+
94+
serve(args.addr)

‎backend/python/faster-whisper/install.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
set -e
3+
4+
source $(dirname $0)/../common/libbackend.sh
5+
6+
# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links.
7+
# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match.
8+
# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index
9+
# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index
10+
if [ "x${BUILD_PROFILE}" == "xintel" ]; then
11+
EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
12+
fi
13+
14+
installRequirements

‎backend/python/faster-whisper/protogen.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -e
3+
4+
source $(dirname $0)/../common/libbackend.sh
5+
6+
python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto

‎backend/python/faster-whisper/requirements-cpu.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
faster-whisper
2+
opencv-python
3+
accelerate
4+
compel
5+
peft
6+
sentencepiece
7+
torch==2.4.1
8+
optimum-quanto

‎backend/python/faster-whisper/requirements-cublas11.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
--extra-index-url https://download.pytorch.org/whl/cu118
2+
torch==2.4.1+cu118
3+
faster-whisper
4+
opencv-python
5+
accelerate
6+
compel
7+
peft
8+
sentencepiece
9+
optimum-quanto

‎backend/python/faster-whisper/requirements-cublas12.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
torch==2.4.1
2+
faster-whisper
3+
opencv-python
4+
accelerate
5+
compel
6+
peft
7+
sentencepiece
8+
optimum-quanto

‎backend/python/faster-whisper/requirements-hipblas.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
--extra-index-url https://download.pytorch.org/whl/rocm6.0
2+
torch
3+
faster-whisper

‎backend/python/faster-whisper/requirements-intel.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
--extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
2+
intel-extension-for-pytorch==2.3.110+xpu
3+
torch==2.3.1+cxx11.abi
4+
oneccl_bind_pt==2.3.100+xpu
5+
optimum[openvino]
6+
faster-whisper

‎backend/python/faster-whisper/requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
grpcio==1.69.0
2+
protobuf
3+
grpcio-tools

‎backend/python/faster-whisper/run.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
source $(dirname $0)/../common/libbackend.sh
3+
4+
startBackend $@

‎backend/python/faster-whisper/test.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -e
3+
4+
source $(dirname $0)/../common/libbackend.sh
5+
6+
runUnittests

0 commit comments

Comments
 (0)