ios : whisper.objc example

2026-10-08 23:41:15 +02:00 · 2022-10-24 18:23:07 +03:00
parent 5f7e9fa2dc
commit bb1ee266d2
17 changed files with 985 additions and 0 deletions
@@ -0,0 +1,14 @@
+//
+//  AppDelegate.h
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+
+@end
+
@@ -0,0 +1,40 @@
+//
+//  AppDelegate.m
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+
+- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+    // Override point for customization after application launch.
+    return YES;
+}
+
+
+#pragma mark - UISceneSession lifecycle
+
+
+- (UISceneConfiguration *)application:(UIApplication *)application configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession options:(UISceneConnectionOptions *)options {
+    // Called when a new scene session is being created.
+    // Use this method to select a configuration to create the new scene with.
+    return [[UISceneConfiguration alloc] initWithName:@"Default Configuration" sessionRole:connectingSceneSession.role];
+}
+
+
+- (void)application:(UIApplication *)application didDiscardSceneSessions:(NSSet<UISceneSession *> *)sceneSessions {
+    // Called when the user discards a scene session.
+    // If any sessions were discarded while the application was not running, this will be called shortly after application:didFinishLaunchingWithOptions.
+    // Use this method to release any resources that were specific to the discarded scenes, as they will not return.
+}
+
+
+@end
@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}
@@ -0,0 +1,13 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "platform" : "ios",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}
@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" xcode11CocoaTouchSystemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="21225" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina6_0" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="21207"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="System colors in document resources" minToolsVersion="11.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="390" height="844"/>
+                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" widthSizable="YES" flexibleMinY="YES" heightSizable="YES"/>
+                        <subviews>
+                            <button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" lineBreakMode="middleTruncation" id="VOi-PT-Rbu">
+                                <rect key="frame" x="35" y="121" width="156" height="34"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <color key="backgroundColor" systemColor="opaqueSeparatorColor"/>
+                                <color key="tintColor" systemColor="opaqueSeparatorColor"/>
+                                <state key="normal" title="Start Capturing">
+                                    <color key="titleColor" systemColor="labelColor"/>
+                                </state>
+                                <connections>
+                                    <action selector="toggleCapture:" destination="BYZ-38-t0r" eventType="touchUpInside" id="BuO-Wf-RgV"/>
+                                </connections>
+                            </button>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Status: Idle" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="Tgu-2q-eHQ">
+                                <rect key="frame" x="35" y="78" width="232" height="21"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" lineBreakMode="middleTruncation" id="Brs-xi-o8i">
+                                <rect key="frame" x="35" y="177" width="156" height="34"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <color key="backgroundColor" systemColor="opaqueSeparatorColor"/>
+                                <color key="tintColor" systemColor="opaqueSeparatorColor"/>
+                                <state key="normal" title="Transcribe">
+                                    <color key="titleColor" systemColor="labelColor"/>
+                                </state>
+                                <connections>
+                                    <action selector="onTranscribe:" destination="BYZ-38-t0r" eventType="touchUpInside" id="ond-bx-48O"/>
+                                    <action selector="onTranscribePrepare:" destination="BYZ-38-t0r" eventType="touchDown" id="16T-dN-dfB"/>
+                                </connections>
+                            </button>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" fixedFrame="YES" text="Record some speech and press &quot;Transcribe&quot;. The result will be displayed here." textAlignment="natural" translatesAutoresizingMaskIntoConstraints="NO" id="mv2-KD-7jn">
+                                <rect key="frame" x="35" y="248" width="320" height="300"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <color key="backgroundColor" systemColor="systemBackgroundColor"/>
+                                <color key="textColor" systemColor="labelColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                        </subviews>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                        <color key="backgroundColor" systemColor="systemBackgroundColor"/>
+                        <constraints>
+                            <constraint firstItem="Brs-xi-o8i" firstAttribute="trailing" secondItem="VOi-PT-Rbu" secondAttribute="trailing" id="8mF-AW-cbc"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="buttonTranscribe" destination="Brs-xi-o8i" id="N8h-9W-ywb"/>
+                        <outlet property="labelStatusInp" destination="Tgu-2q-eHQ" id="1hH-Ql-K6j"/>
+                        <outlet property="textviewResult" destination="mv2-KD-7jn" id="RBw-0L-iGj"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="30.769230769230766" y="-28.436018957345969"/>
+        </scene>
+    </scenes>
+    <resources>
+        <systemColor name="labelColor">
+            <color red="0.0" green="0.0" blue="0.0" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+        </systemColor>
+        <systemColor name="opaqueSeparatorColor">
+            <color red="0.77647058823529413" green="0.77647058823529413" blue="0.78431372549019607" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+        </systemColor>
+        <systemColor name="systemBackgroundColor">
+            <color white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+        </systemColor>
+    </resources>
+</document>
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>This app requires microphone access in order to transcribe speech</string>
+	<key>UIApplicationSceneManifest</key>
+	<dict>
+		<key>UIApplicationSupportsMultipleScenes</key>
+		<false/>
+		<key>UISceneConfigurations</key>
+		<dict>
+			<key>UIWindowSceneSessionRoleApplication</key>
+			<array>
+				<dict>
+					<key>UISceneConfigurationName</key>
+					<string>Default Configuration</string>
+					<key>UISceneDelegateClassName</key>
+					<string>SceneDelegate</string>
+					<key>UISceneStoryboardFile</key>
+					<string>Main</string>
+				</dict>
+			</array>
+		</dict>
+	</dict>
+</dict>
+</plist>
@@ -0,0 +1,15 @@
+//
+//  SceneDelegate.h
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import <UIKit/UIKit.h>
+
+@interface SceneDelegate : UIResponder <UIWindowSceneDelegate>
+
+@property (strong, nonatomic) UIWindow * window;
+
+@end
+
@@ -0,0 +1,57 @@
+//
+//  SceneDelegate.m
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import "SceneDelegate.h"
+
+@interface SceneDelegate ()
+
+@end
+
+@implementation SceneDelegate
+
+
+- (void)scene:(UIScene *)scene willConnectToSession:(UISceneSession *)session options:(UISceneConnectionOptions *)connectionOptions {
+    // Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.
+    // If using a storyboard, the `window` property will automatically be initialized and attached to the scene.
+    // This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).
+}
+
+
+- (void)sceneDidDisconnect:(UIScene *)scene {
+    // Called as the scene is being released by the system.
+    // This occurs shortly after the scene enters the background, or when its session is discarded.
+    // Release any resources associated with this scene that can be re-created the next time the scene connects.
+    // The scene may re-connect later, as its session was not necessarily discarded (see `application:didDiscardSceneSessions` instead).
+}
+
+
+- (void)sceneDidBecomeActive:(UIScene *)scene {
+    // Called when the scene has moved from an inactive state to an active state.
+    // Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.
+}
+
+
+- (void)sceneWillResignActive:(UIScene *)scene {
+    // Called when the scene will move from an active state to an inactive state.
+    // This may occur due to temporary interruptions (ex. an incoming phone call).
+}
+
+
+- (void)sceneWillEnterForeground:(UIScene *)scene {
+    // Called as the scene transitions from the background to the foreground.
+    // Use this method to undo the changes made on entering the background.
+}
+
+
+- (void)sceneDidEnterBackground:(UIScene *)scene {
+    // Called as the scene transitions from the foreground to the background.
+    // Use this method to save data, release shared resources, and store enough scene-specific state information
+    // to restore the scene back to its current state.
+}
+
+
+@end
@@ -0,0 +1,41 @@
+//
+//  ViewController.h
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import <UIKit/UIKit.h>
+
+#import <AVFoundation/AVFoundation.h>
+#import <AudioToolbox/AudioQueue.h>
+
+#define NUM_BUFFERS 3
+#define MAX_AUDIO_SEC 30
+#define SAMPLE_RATE 16000
+
+struct whisper_context;
+
+typedef struct
+{
+    int ggwaveId;
+    bool isCapturing;
+    UILabel * labelReceived;
+
+    AudioQueueRef queue;
+    AudioStreamBasicDescription dataFormat;
+    AudioQueueBufferRef buffers[NUM_BUFFERS];
+
+    int n_samples;
+    int16_t * audioBufferI16;
+    float   * audioBufferF32;
+
+    struct whisper_context * ctx;
+} StateInp;
+
+@interface ViewController : UIViewController
+{
+    StateInp stateInp;
+}
+
+@end
@@ -0,0 +1,229 @@
+//
+//  ViewController.m
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import "ViewController.h"
+
+#import "whisper.h"
+
+#define NUM_BYTES_PER_BUFFER 16*1024
+
+// callback used to process captured audio
+void AudioInputCallback(void * inUserData,
+                        AudioQueueRef inAQ,
+                        AudioQueueBufferRef inBuffer,
+                        const AudioTimeStamp * inStartTime,
+                        UInt32 inNumberPacketDescriptions,
+                        const AudioStreamPacketDescription * inPacketDescs);
+
+@interface ViewController ()
+
+@property (weak, nonatomic) IBOutlet UILabel *labelStatusInp;
+@property (weak, nonatomic) IBOutlet UIButton *buttonToggleCapture;
+@property (weak, nonatomic) IBOutlet UIButton *buttonTranscribe;
+@property (weak, nonatomic) IBOutlet UITextView *textviewResult;
+
+@end
+
+@implementation ViewController
+
+- (void)setupAudioFormat:(AudioStreamBasicDescription*)format
+{
+    format->mSampleRate       = 16000;
+    format->mFormatID         = kAudioFormatLinearPCM;
+    format->mFramesPerPacket  = 1;
+    format->mChannelsPerFrame = 1;
+    format->mBytesPerFrame    = 2;
+    format->mBytesPerPacket   = 2;
+    format->mBitsPerChannel   = 16;
+    format->mReserved         = 0;
+    format->mFormatFlags      = kLinearPCMFormatFlagIsSignedInteger;
+}
+
+- (void)viewDidLoad {
+    [super viewDidLoad];
+
+    // whisper.cpp initialization
+    {
+        // load the model
+        NSString *modelPath = [[NSBundle mainBundle] pathForResource:@"ggml-base.en" ofType:@"bin"];
+
+        // check if the model exists
+        if (![[NSFileManager defaultManager] fileExistsAtPath:modelPath]) {
+            NSLog(@"Model file not found");
+            return;
+        }
+
+        NSLog(@"Loading model from %@", modelPath);
+
+        // create ggml context
+        stateInp.ctx = whisper_init([modelPath UTF8String]);
+
+        // check if the model was loaded successfully
+        if (stateInp.ctx == NULL) {
+            NSLog(@"Failed to load model");
+            return;
+        }
+    }
+
+    // initialize audio format and buffers
+    {
+        [self setupAudioFormat:&stateInp.dataFormat];
+
+        stateInp.n_samples = 0;
+        stateInp.audioBufferI16 = malloc(MAX_AUDIO_SEC*SAMPLE_RATE*sizeof(int16_t));
+        stateInp.audioBufferF32 = malloc(MAX_AUDIO_SEC*SAMPLE_RATE*sizeof(float));
+    }
+}
+
+-(IBAction) stopCapturing {
+    NSLog(@"Stop capturing");
+
+    _labelStatusInp.text = @"Status: Idle";
+
+    stateInp.isCapturing = false;
+
+    AudioQueueStop(stateInp.queue, true);
+    for (int i = 0; i < NUM_BUFFERS; i++) {
+        AudioQueueFreeBuffer(stateInp.queue, stateInp.buffers[i]);
+    }
+
+    AudioQueueDispose(stateInp.queue, true);
+}
+
+- (IBAction)toggleCapture:(id)sender {
+    if (stateInp.isCapturing) {
+        // stop capturing
+        [self stopCapturing];
+        [sender setTitle:@"Start Capturing" forState:UIControlStateNormal];
+
+        return;
+    }
+
+    // initiate audio capturing
+    NSLog(@"Start capturing");
+
+    stateInp.n_samples = 0;
+
+    OSStatus status = AudioQueueNewInput(&stateInp.dataFormat,
+                                         AudioInputCallback,
+                                         &stateInp,
+                                         CFRunLoopGetCurrent(),
+                                         kCFRunLoopCommonModes,
+                                         0,
+                                         &stateInp.queue);
+
+    if (status == 0) {
+        for (int i = 0; i < NUM_BUFFERS; i++) {
+            AudioQueueAllocateBuffer(stateInp.queue, NUM_BYTES_PER_BUFFER, &stateInp.buffers[i]);
+            AudioQueueEnqueueBuffer (stateInp.queue, stateInp.buffers[i], 0, NULL);
+        }
+
+        stateInp.isCapturing = true;
+        status = AudioQueueStart(stateInp.queue, NULL);
+        if (status == 0) {
+            _labelStatusInp.text = @"Status: Capturing";
+            [sender setTitle:@"Stop Capturing" forState:UIControlStateNormal];
+        }
+    }
+
+    if (status != 0) {
+        [self stopCapturing];
+    }
+}
+
+- (IBAction)onTranscribePrepare:(id)sender {
+    _textviewResult.text = @"Processing - please wait ...";
+
+    if (stateInp.isCapturing) {
+        // stop capturing
+        [self stopCapturing];
+        [sender setTitle:@"Start Capturing" forState:UIControlStateNormal];
+
+        return;
+    }
+}
+
+- (IBAction)onTranscribe:(id)sender {
+    NSLog(@"Processing %d samples", stateInp.n_samples);
+
+    // process captured audio
+    // convert I16 to F32
+    for (int i = 0; i < stateInp.n_samples; i++) {
+        stateInp.audioBufferF32[i] = (float)stateInp.audioBufferI16[i] / 32768.0f;
+    }
+
+    // run the model
+    struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+    params.print_realtime       = true;
+    params.print_progress       = false;
+    params.print_timestamps     = true;
+    params.print_special_tokens = false;
+    params.translate            = false;
+    params.language             = "en";
+    params.n_threads            = 4;
+    params.offset_ms            = 0;
+
+    if (whisper_full(stateInp.ctx, params, stateInp.audioBufferF32, stateInp.n_samples) != 0) {
+        NSLog(@"Failed to run the model");
+        _textviewResult.text = @"Failed to run the model";
+
+        return;
+    }
+
+    // clear the text in the textview
+    _textviewResult.text = @"";
+
+    int n_segments = whisper_full_n_segments(stateInp.ctx);
+    for (int i = 0; i < n_segments; i++) {
+        const char * text_cur = whisper_full_get_segment_text(stateInp.ctx, i);
+
+        // append the text to the textview
+        _textviewResult.text = [_textviewResult.text stringByAppendingString:[NSString stringWithUTF8String:text_cur]];
+    }
+
+    whisper_print_timings(stateInp.ctx);
+}
+
+//
+// Callback implmentation
+//
+
+void AudioInputCallback(void * inUserData,
+                        AudioQueueRef inAQ,
+                        AudioQueueBufferRef inBuffer,
+                        const AudioTimeStamp * inStartTime,
+                        UInt32 inNumberPacketDescriptions,
+                        const AudioStreamPacketDescription * inPacketDescs)
+{
+    StateInp * stateInp = (StateInp*)inUserData;
+
+    if (!stateInp->isCapturing) {
+        NSLog(@"Not capturing, ignoring audio");
+        return;
+    }
+
+    const int n = inBuffer->mAudioDataByteSize / 2;
+
+    NSLog(@"Captured %d new samples", n);
+
+    if (stateInp->n_samples + n > MAX_AUDIO_SEC*SAMPLE_RATE) {
+        NSLog(@"Too much audio data, ignoring");
+        return;
+    }
+
+    for (int i = 0; i < n; i++) {
+        stateInp->audioBufferI16[stateInp->n_samples + i] = ((short*)inBuffer->mAudioData)[i];
+    }
+
+    stateInp->n_samples += n;
+
+    // put the buffer back in the queue
+    AudioQueueEnqueueBuffer(stateInp->queue, inBuffer, 0, NULL);
+}
+
+@end
@@ -0,0 +1,18 @@
+//
+//  main.m
+//  whisper.objc
+//
+//  Created by Georgi Gerganov on 23.10.22.
+//
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char * argv[]) {
+    NSString * appDelegateClassName;
+    @autoreleasepool {
+        // Setup code that might create autoreleased objects goes here.
+        appDelegateClassName = NSStringFromClass([AppDelegate class]);
+    }
+    return UIApplicationMain(argc, argv, nil, appDelegateClassName);
+}