CUDA 12.3 CUDNN 9 Onnxruntime 1.20.1

--use-cuda segfaults and CUDA not working

Dockerfile

@@ -1,48 +1,13 @@
 ##########################################
 FROM nvidia/cuda:12.3.2-cudnn9-runtime-ubuntu22.04 AS build
 
-ARG PIPER_BRANCH="2023.11.14-2"
+ARG TARGETARCH=linux_x86_64
+ARG WYOMING_PIPER_VERSION="1.5.2"
+ARG ONNXRUNTIME_VERSION="1.20.1"
 
 ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN \
-    apt-get update &&\
-    apt-get install -y --no-install-recommends \
-        wget \
-        curl \
-        vim \
-        git \
-        build-essential \
-        cmake \
-        ca-certificates \
-        curl \
-        pkg-config
-
-WORKDIR /build
-
-RUN \
-    git clone -b "${PIPER_BRANCH}" https://github.com/rhasspy/piper.git /build
-
-RUN cmake -Bbuild -DCMAKE_INSTALL_PREFIX=install
-RUN cmake --build build --config Release
-RUN cmake --install build
-
-WORKDIR /dist
-RUN mkdir -p piper && \
-    cp -dR /build/install/* ./piper/
-
-RUN /dist/piper/piper --help
-
-##########################################
-FROM nvidia/cuda:12.3.2-cudnn9-runtime-ubuntu22.04 AS dist
-
-ARG TARGETARCH=linux_x86_64
-ARG WYOMING_PIPER_VERSION="1.5.0"
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PYTHONUNBUFFERED=1
-
 RUN \
     apt-get update && apt-get upgrade -y &&\
     apt-get install -y --no-install-recommends \
@@ -54,42 +19,80 @@ RUN \
         python3 \
         python3-dev \
         python3-venv \
-        python3-pip 
+        python3-pip \
+        build-essential \
+        cmake \
+        ca-certificates \
+        pkg-config
 
 RUN \
-    mkdir -p /data /app/piper &&\
-    ln -s /app/piper /usr/share/espeak-ng-data &&\
-    python3 -m venv /app
+    mkdir -p /app/lib &&\
+    cd /tmp && wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}.tgz &&\
+    tar xzf onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}.tgz &&\
+    cp -rfv /tmp/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/lib/lib* /app/lib &&\
+    cp -rfv /tmp/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/include/* /usr/local/include/
+
+WORKDIR /build
+
+COPY patches/* /tmp/
+RUN \
+    git clone https://github.com/rhasspy/piper.git /build &&\
+    cd /build && patch -p0 --forward < /tmp/piper_CMakeLists.patch
+
+RUN ONNXRUNTIME_VERSION=${ONNXRUNTIME_VERSION} cmake -Bbuild -DCMAKE_INSTALL_PREFIX=install
+RUN cmake --build build --config Release
+RUN cmake --install build
+
+WORKDIR /app
+
+RUN \
+    python3 -m venv /app &&\
+    mkdir /app/piper && cp -rfv /build/install/* /app/piper/ &&\
+    chmod 755 /app/piper/piper /app/piper/espeak-ng
+
+RUN /app/piper/piper --help
 
-COPY --from=build /dist/piper/* /app/piper/
 COPY requirements.txt /app/
+COPY run.sh /app/
+
 RUN \
     . /app/bin/activate && \
-    /app/bin/python3 -m pip install --no-cache-dir --no-deps \
+    /app/bin/python3 -m pip install --no-cache-dir \
         -r /app/requirements.txt \
         &&\
     \
     /app/bin/python3 -m pip install --no-cache-dir \
-        "wyoming-piper @ https://github.com/rhasspy/wyoming-piper/archive/refs/tags/v${WYOMING_PIPER_VERSION}.tar.gz" \
-        &&\
-    \
-    PIPER_VERSION=$(wget "https://api.github.com/repos/rhasspy/piper/releases/latest" -O -|awk '/tag_name/{print $4;exit}' FS='[""]') && \
-    \
-    wget "https://github.com/rhasspy/piper/releases/download/${PIPER_VERSION}/piper_${TARGETARCH}.tar.gz" -O -|tar -zxvf - -C /usr/share
-    
-# Patch to enable CUDA arguments for piper
-COPY patches/* /tmp/
+        "wyoming-piper @ https://github.com/rhasspy/wyoming-piper/archive/refs/tags/v${WYOMING_PIPER_VERSION}.tar.gz"
+
 RUN \
     cd /app/lib/python3.10/site-packages/wyoming_piper/; \
     for file in /tmp/wyoming_piper*.diff;do patch -p0 --forward < $file;done; \
-    cp /app/lib/python3.10/site-packages/onnxruntime/capi/libonnxruntime* /app/lib/ &&\
-    mv /app/piper/lib*.so* /app/lib/ 
+    cp /build/install/lib*.so* /app/lib/
 
-# Clean up
-RUN rm -rf /root/.cache/pip /var/lib/apt/lists/* /tmp/*
+##########################################
+FROM nvidia/cuda:12.3.2-cudnn9-runtime-ubuntu22.04 AS dist
+
+ENV PYTHONUNBUFFERED=1
+
+RUN \
+    mkdir -p /data /app &&\
+    apt-get update && apt-get upgrade -y &&\
+    apt-get install -y --no-install-recommends \
+        wget \
+        curl \
+        vim \
+        python3 \
+        python3-dev \
+        python3-venv \
+        python3-pip \
+        ca-certificates
+
+RUN rm -rf /root/.cache/pip /var/lib/apt/lists/*
 
 WORKDIR /app
-COPY run.sh /app/
+COPY --from=build /app .
+
+RUN ln -s /app/piper/espeak-ng-data /usr/share/espeak-ng-data
 
 EXPOSE 10200
 

README.md

@@ -12,4 +12,8 @@ docker pull ghcr.io/slackr31337/wyoming-piper-gpu:latest
 Use environment variable to set piper voice
 
 PIPER_VOICE="en_US-lessac-medium"
-
+PIPER_LENGTH="1.0"
+PIPER_NOISE="0.667"
+PIPER_NOISEW="0.333"
+PIPER_SPEAKER="0"
+PIPER_SILENCE="1.2"

patches/piper_CMakeLists.patch

@@ -0,0 +1,16 @@
+--- CMakeLists.txt      2025-01-12 11:49:52.100425655 -0500
++++ CMakeLists.txt.patched      2025-01-12 11:50:56.033517774 -0500
+@@ -60,11 +60,12 @@
+
+ if(NOT DEFINED PIPER_PHONEMIZE_DIR)
+   set(PIPER_PHONEMIZE_DIR "${CMAKE_CURRENT_BINARY_DIR}/pi")
++  set(ONNXRUNTIME_VERSION "$ENV{ONNXRUNTIME_VERSION}")
+   ExternalProject_Add(
+     piper_phonemize_external
+     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/p"
+     URL "https://github.com/rhasspy/piper-phonemize/archive/refs/heads/master.zip"
+-    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PIPER_PHONEMIZE_DIR}
++    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PIPER_PHONEMIZE_DIR} -DONNXRUNTIME_VERSION=${ONNXRUNTIME_VERSION}
+   )
+   add_dependencies(piper piper_phonemize_external)
+   add_dependencies(test_piper piper_phonemize_external)

patches/piper_voice.diff

@@ -1,11 +0,0 @@
---- voice.py	2025-01-04 19:49:49.000000000 +0000
-+++ voice.patched	2025-01-04 21:54:21.970873501 +0000
-@@ -41,7 +41,7 @@
-                 sess_options=onnxruntime.SessionOptions(),
-                 providers=["CPUExecutionProvider"]
-                 if not use_cuda
--                else ["CUDAExecutionProvider"],
-+                else [("CUDAExecutionProvider", {"cudnn_conv_algo_search": "HEURISTIC"})], 
-             ),
-         )
- 

patches/wyoming_piper__main__.diff

@@ -1,6 +1,17 @@
---- __main__.py	2025-01-04 21:57:16.229874190 +0000
-+++ __main__.patched	2025-01-04 21:56:57.917558487 +0000
-@@ -66,6 +66,16 @@
+--- __main__.py	2025-01-12 12:08:32.699567820 -0500
++++ __main__.py.patched	2025-01-12 12:13:42.368857466 -0500
+@@ -53,6 +53,10 @@
+     parser.add_argument(
+         "--auto-punctuation", default=".?!", help="Automatically add punctuation"
+     )
++    parser.add_argument(
++        "--sentence-silence",
++        help="Seconds of silence to add after each sentence"
++    )
+     parser.add_argument("--samples-per-chunk", type=int, default=1024)
+     parser.add_argument(
+         "--max-piper-procs",
+@@ -67,6 +71,16 @@
          help="Download latest voices.json during startup",
      )
      #
@@ -15,5 +26,5 @@
 +    )
 +    #
      parser.add_argument("--debug", action="store_true", help="Log DEBUG messages")
-     args = parser.parse_args()
- 
+     parser.add_argument(
+         "--log-format", default=logging.BASIC_FORMAT, help="Format for log messages"

patches/wyoming_piper_process.diff

@@ -1,9 +1,12 @@
---- process.py	2025-01-04 19:50:00.000000000 +0000
-+++ process.py.patched	2025-01-04 20:58:10.813174885 +0000
-@@ -148,6 +147,12 @@
+--- process.py  2025-01-12 12:08:32.699567820 -0500
++++ process.py.patched  2025-01-12 12:28:28.568004323 -0500
+@@ -148,6 +148,15 @@
              if self.args.noise_w:
                  piper_args.extend(["--noise-w", str(self.args.noise_w)])
- 
+
++            if self.args.sentence_silence:
++                piper_args.extend(["--sentence-silence", str(self.args.sentence_silence)])
++
 +            if self.args.espeak_data_dir:
 +                piper_args.extend(["--espeak_data", str(self.args.espeak_data_dir)])
 +
@@ -12,4 +15,4 @@
 +
              _LOGGER.debug(
                  "Starting piper process: %s args=%s", self.args.piper, piper_args
-             )
+             )

requirements.txt

@@ -1,3 +1,2 @@
 piper-tts>=1.2.0
-onnxruntime==1.14.0
-onnxruntime-gpu==1.14.0
+onnxruntime-gpu==1.20.1

run.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 
-export LD_LIBRARY_PATH=/app/lib/:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=/app/lib/:/usr/local/cuda-12.3/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
 
 # Run wyoming-piper server
 source /app/bin/activate
-
 /app/bin/python3 -m wyoming_piper \
     --piper '/app/piper/piper' \
     --uri 'tcp://0.0.0.0:10200' \
@@ -12,8 +11,11 @@ source /app/bin/activate
     --noise-scale "${PIPER_NOISE:-0.667}" \
     --noise-w "${PIPER_NOISEW:-0.333}" \
     --speaker "${PIPER_SPEAKER:-0}" \
+    --sentence-silence "${PIPER_SILENCE:-1.2}" \
     --voice "${PIPER_VOICE:-en_US-amy-medium}" \
     --max-piper-procs "${PIPER_PROCS:-1}" \
     --data-dir /data \
     --download-dir /data \
-    --use-cuda "$@"
+    "$@"
+
+# --use-cuda crashes with onnxruntime 1.20.1