diff --git a/bindings/go/pkg/whisper/consts.go b/bindings/go/pkg/whisper/consts.go index 6c778d3d2..0af45ee8c 100644 --- a/bindings/go/pkg/whisper/consts.go +++ b/bindings/go/pkg/whisper/consts.go @@ -17,6 +17,7 @@ var ( ErrUnsupportedLanguage = errors.New("unsupported language") ErrModelNotMultilingual = errors.New("model is not multilingual") ErrUnableToCreateState = errors.New("unable to create state") + ErrModelClosed = errors.New("model has been closed") ) /////////////////////////////////////////////////////////////////////////////// diff --git a/bindings/go/pkg/whisper/context.go b/bindings/go/pkg/whisper/context.go index a7df02a66..01a510fe7 100644 --- a/bindings/go/pkg/whisper/context.go +++ b/bindings/go/pkg/whisper/context.go @@ -11,160 +11,77 @@ import ( whisper "github.com/ggerganov/whisper.cpp/bindings/go" ) -/////////////////////////////////////////////////////////////////////////////// -// TYPES - type context struct { n int - model *model - params whisper.Params + model Model + st WhisperState + params Parameters + Parameters } -/////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE +func newContext(model Model, params whisper.Params) (Context, error) { + c := new(context) + c.model = model -func newContext(model *model, params whisper.Params) (Context, error) { - context := new(context) - context.model = model - context.params = params + c.params = newParameters(¶ms) + c.Parameters = c.params + + // allocate isolated state per context + ctx, err := model.WhisperContext().UnsafeContext() + if err != nil { + return nil, err + } + + st := ctx.Whisper_init_state() + if st == nil { + return nil, ErrUnableToCreateState + } + + c.st = newWhisperState(st) // Return success - return context, nil -} - -/////////////////////////////////////////////////////////////////////////////// -// PUBLIC METHODS - -// Set the language to use for speech recognition. -func (context *context) SetLanguage(lang string) error { - if context.model.ctx == nil { - return ErrInternalAppError - } - if !context.model.IsMultilingual() { - return ErrModelNotMultilingual - } - - if lang == "auto" { - context.params.SetLanguage(-1) - } else if id := context.model.ctx.Whisper_lang_id(lang); id < 0 { - return ErrUnsupportedLanguage - } else if err := context.params.SetLanguage(id); err != nil { - return err - } - // Return success - return nil -} - -func (context *context) IsMultilingual() bool { - return context.model.IsMultilingual() -} - -// Get language -func (context *context) Language() string { - id := context.params.Language() - if id == -1 { - return "auto" - } - return whisper.Whisper_lang_str(context.params.Language()) + return c, nil } +// DetectedLanguage returns the detected language for the current context data func (context *context) DetectedLanguage() string { - return whisper.Whisper_lang_str(context.model.ctx.Whisper_full_lang_id()) + ctx, err := context.model.WhisperContext().UnsafeContext() + if err != nil { + return "" + } + + st, err := context.st.UnsafeState() + if err != nil { + return "" + } + + return whisper.Whisper_lang_str( + ctx.Whisper_full_lang_id_from_state( + st, + ), + ) } -// Set translate flag -func (context *context) SetTranslate(v bool) { - context.params.SetTranslate(v) +// Close frees the whisper state and marks the context as closed. +func (context *context) Close() error { + return context.st.Close() } -func (context *context) SetSplitOnWord(v bool) { - context.params.SetSplitOnWord(v) +// Params returns a high-level parameters wrapper +func (context *context) Params() Parameters { + return context.params } -// Set number of threads to use -func (context *context) SetThreads(v uint) { - context.params.SetThreads(int(v)) -} - -// Set time offset -func (context *context) SetOffset(v time.Duration) { - context.params.SetOffset(int(v.Milliseconds())) -} - -// Set duration of audio to process -func (context *context) SetDuration(v time.Duration) { - context.params.SetDuration(int(v.Milliseconds())) -} - -// Set timestamp token probability threshold (~0.01) -func (context *context) SetTokenThreshold(t float32) { - context.params.SetTokenThreshold(t) -} - -// Set timestamp token sum probability threshold (~0.01) -func (context *context) SetTokenSumThreshold(t float32) { - context.params.SetTokenSumThreshold(t) -} - -// Set max segment length in characters -func (context *context) SetMaxSegmentLength(n uint) { - context.params.SetMaxSegmentLength(int(n)) -} - -// Set token timestamps flag -func (context *context) SetTokenTimestamps(b bool) { - context.params.SetTokenTimestamps(b) -} - -// Set max tokens per segment (0 = no limit) -func (context *context) SetMaxTokensPerSegment(n uint) { - context.params.SetMaxTokensPerSegment(int(n)) -} - -// Set audio encoder context -func (context *context) SetAudioCtx(n uint) { - context.params.SetAudioCtx(int(n)) -} - -// Set maximum number of text context tokens to store -func (context *context) SetMaxContext(n int) { - context.params.SetMaxContext(n) -} - -// Set Beam Size -func (context *context) SetBeamSize(n int) { - context.params.SetBeamSize(n) -} - -// Set Entropy threshold -func (context *context) SetEntropyThold(t float32) { - context.params.SetEntropyThold(t) -} - -// Set Temperature -func (context *context) SetTemperature(t float32) { - context.params.SetTemperature(t) -} - -// Set the fallback temperature incrementation -// Pass -1.0 to disable this feature -func (context *context) SetTemperatureFallback(t float32) { - context.params.SetTemperatureFallback(t) -} - -// Set initial prompt -func (context *context) SetInitialPrompt(prompt string) { - context.params.SetInitialPrompt(prompt) -} - -// ResetTimings resets the mode timings. Should be called before processing +// ResetTimings resets the model performance timing counters. +// Deprecated: Use Model.ResetTimings() instead - these are model-level performance metrics. func (context *context) ResetTimings() { - context.model.ctx.Whisper_reset_timings() + context.model.ResetTimings() } -// PrintTimings prints the model timings to stdout. +// PrintTimings prints the model performance timings to stdout. +// Deprecated: Use Model.PrintTimings() instead - these are model-level performance metrics. func (context *context) PrintTimings() { - context.model.ctx.Whisper_print_timings() + context.model.PrintTimings() } // SystemInfo returns the system information @@ -178,12 +95,23 @@ func (context *context) SystemInfo() string { // Use mel data at offset_ms to try and auto-detect the spoken language // Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first. -// Returns the probabilities of all languages. +// Returns the probabilities of all languages for this context's state. func (context *context) WhisperLangAutoDetect(offset_ms int, n_threads int) ([]float32, error) { - langProbs, err := context.model.ctx.Whisper_lang_auto_detect(offset_ms, n_threads) + ctx, err := context.model.WhisperContext().UnsafeContext() if err != nil { return nil, err } + + st, err := context.st.UnsafeState() + if err != nil { + return nil, err + } + + langProbs, err := ctx.Whisper_lang_auto_detect_with_state(st, offset_ms, n_threads) + if err != nil { + return nil, err + } + return langProbs, nil } @@ -194,36 +122,33 @@ func (context *context) Process( callNewSegment SegmentCallback, callProgress ProgressCallback, ) error { - if context.model.ctx == nil { - return ErrInternalAppError + ctx, err := context.model.WhisperContext().UnsafeContext() + if err != nil { + return err } + // If the callback is defined then we force on single_segment mode if callNewSegment != nil { context.params.SetSingleSegment(true) } - // We don't do parallel processing at the moment - processors := 0 - if processors > 1 { - if err := context.model.ctx.Whisper_full_parallel(context.params, data, processors, callEncoderBegin, - func(new int) { - if callNewSegment != nil { - num_segments := context.model.ctx.Whisper_full_n_segments() - s0 := num_segments - new - for i := s0; i < num_segments; i++ { - callNewSegment(toSegment(context.model.ctx, i)) - } - } - }); err != nil { - return err - } - } else if err := context.model.ctx.Whisper_full(context.params, data, callEncoderBegin, + lowLevelParams := context.params.WhisperParams() + if lowLevelParams == nil { + return fmt.Errorf("lowLevelParams is nil: %w", ErrInternalAppError) + } + + st, err := context.st.UnsafeState() + if err != nil { + return err + } + + if err := ctx.Whisper_full_with_state(st, *lowLevelParams, data, callEncoderBegin, func(new int) { if callNewSegment != nil { - num_segments := context.model.ctx.Whisper_full_n_segments() + num_segments := ctx.Whisper_full_n_segments_from_state(st) s0 := num_segments - new for i := s0; i < num_segments; i++ { - callNewSegment(toSegment(context.model.ctx, i)) + callNewSegment(toSegmentFromState(ctx, st, i)) } } }, func(progress int) { @@ -240,94 +165,111 @@ func (context *context) Process( // NextSegment returns the next segment from the context buffer func (context *context) NextSegment() (Segment, error) { - if context.model.ctx == nil { - return Segment{}, ErrInternalAppError + ctx, err := context.model.WhisperContext().UnsafeContext() + if err != nil { + return Segment{}, err } - if context.n >= context.model.ctx.Whisper_full_n_segments() { + + st, err := context.st.UnsafeState() + if err != nil { + return Segment{}, err + } + + if context.n >= ctx.Whisper_full_n_segments_from_state(st) { return Segment{}, io.EOF } - result := toSegment(context.model.ctx, context.n) + + result := toSegmentFromState(ctx, st, context.n) context.n++ + return result, nil } -/////////////////////////////////////////////////////////////////////////////// -// PRIVATE METHODS +func (context *context) IsMultilingual() bool { + return context.model.IsMultilingual() +} -func toSegment(ctx *whisper.Context, n int) Segment { +// Token helpers +// Deprecated: Use Model.IsText() instead - token checking is model-specific. +func (context *context) IsText(t Token) bool { + result, _ := context.model.TokenIdentifier().IsText(t) + return result +} + +// Deprecated: Use Model.IsBEG() instead - token checking is model-specific. +func (context *context) IsBEG(t Token) bool { + result, _ := context.model.TokenIdentifier().IsBEG(t) + return result +} + +// Deprecated: Use Model.IsSOT() instead - token checking is model-specific. +func (context *context) IsSOT(t Token) bool { + result, _ := context.model.TokenIdentifier().IsSOT(t) + return result +} + +// Deprecated: Use Model.IsEOT() instead - token checking is model-specific. +func (context *context) IsEOT(t Token) bool { + result, _ := context.model.TokenIdentifier().IsEOT(t) + return result +} + +// Deprecated: Use Model.IsPREV() instead - token checking is model-specific. +func (context *context) IsPREV(t Token) bool { + result, _ := context.model.TokenIdentifier().IsPREV(t) + return result +} + +// Deprecated: Use Model.IsSOLM() instead - token checking is model-specific. +func (context *context) IsSOLM(t Token) bool { + result, _ := context.model.TokenIdentifier().IsSOLM(t) + return result +} + +// Deprecated: Use Model.IsNOT() instead - token checking is model-specific. +func (context *context) IsNOT(t Token) bool { + result, _ := context.model.TokenIdentifier().IsNOT(t) + return result +} + +func (context *context) SetLanguage(lang string) error { + if !context.model.IsMultilingual() { + return ErrModelNotMultilingual + } + + return context.params.SetLanguage(lang) +} + +// Deprecated: Use Model.IsLANG() instead - token checking is model-specific. +func (context *context) IsLANG(t Token, lang string) bool { + result, _ := context.model.TokenIdentifier().IsLANG(t, lang) + return result +} + +// State-backed helper functions +func toSegmentFromState(ctx *whisper.Context, st *whisper.State, n int) Segment { return Segment{ Num: n, - Text: strings.TrimSpace(ctx.Whisper_full_get_segment_text(n)), - Start: time.Duration(ctx.Whisper_full_get_segment_t0(n)) * time.Millisecond * 10, - End: time.Duration(ctx.Whisper_full_get_segment_t1(n)) * time.Millisecond * 10, - Tokens: toTokens(ctx, n), + Text: strings.TrimSpace(ctx.Whisper_full_get_segment_text_from_state(st, n)), + Start: time.Duration(ctx.Whisper_full_get_segment_t0_from_state(st, n)) * time.Millisecond * 10, + End: time.Duration(ctx.Whisper_full_get_segment_t1_from_state(st, n)) * time.Millisecond * 10, + Tokens: toTokensFromState(ctx, st, n), } } -func toTokens(ctx *whisper.Context, n int) []Token { - result := make([]Token, ctx.Whisper_full_n_tokens(n)) - for i := 0; i < len(result); i++ { - data := ctx.Whisper_full_get_token_data(n, i) +func toTokensFromState(ctx *whisper.Context, st *whisper.State, n int) []Token { + result := make([]Token, ctx.Whisper_full_n_tokens_from_state(st, n)) + for i := 0; i < len(result); i++ { + data := ctx.Whisper_full_get_token_data_from_state(st, n, i) result[i] = Token{ - Id: int(ctx.Whisper_full_get_token_id(n, i)), - Text: ctx.Whisper_full_get_token_text(n, i), - P: ctx.Whisper_full_get_token_p(n, i), + Id: int(ctx.Whisper_full_get_token_id_from_state(st, n, i)), + Text: ctx.Whisper_full_get_token_text_from_state(st, n, i), + P: ctx.Whisper_full_get_token_p_from_state(st, n, i), Start: time.Duration(data.T0()) * time.Millisecond * 10, End: time.Duration(data.T1()) * time.Millisecond * 10, } } + return result } - -// Token helpers -func (context *context) IsText(t Token) bool { - switch { - case context.IsBEG(t): - return false - case context.IsSOT(t): - return false - case whisper.Token(t.Id) >= context.model.ctx.Whisper_token_eot(): - return false - case context.IsPREV(t): - return false - case context.IsSOLM(t): - return false - case context.IsNOT(t): - return false - default: - return true - } -} - -func (context *context) IsBEG(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_beg() -} - -func (context *context) IsSOT(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_sot() -} - -func (context *context) IsEOT(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_eot() -} - -func (context *context) IsPREV(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_prev() -} - -func (context *context) IsSOLM(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_solm() -} - -func (context *context) IsNOT(t Token) bool { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_not() -} - -func (context *context) IsLANG(t Token, lang string) bool { - if id := context.model.ctx.Whisper_lang_id(lang); id >= 0 { - return whisper.Token(t.Id) == context.model.ctx.Whisper_token_lang(id) - } else { - return false - } -} diff --git a/bindings/go/pkg/whisper/context_test.go b/bindings/go/pkg/whisper/context_test.go index e98a4c2b8..305446624 100644 --- a/bindings/go/pkg/whisper/context_test.go +++ b/bindings/go/pkg/whisper/context_test.go @@ -2,11 +2,13 @@ package whisper_test import ( "os" + "sync" "testing" "github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper" "github.com/go-audio/wav" assert "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetLanguage(t *testing.T) { @@ -122,3 +124,173 @@ func TestDetectedLanguage(t *testing.T) { actualLanguage := context.DetectedLanguage() assert.Equal(expectedLanguage, actualLanguage) } + +// TestContext_ConcurrentProcessing tests that multiple contexts can process concurrently +// without interfering with each other (validates the whisper_state isolation fix) +func TestContext_ConcurrentProcessing(t *testing.T) { + assert := assert.New(t) + + if _, err := os.Stat(ModelPath); os.IsNotExist(err) { + t.Skip("Skipping test, model not found:", ModelPath) + } + if _, err := os.Stat(SamplePath); os.IsNotExist(err) { + t.Skip("Skipping test, sample not found:", SamplePath) + } + + fh, err := os.Open(SamplePath) + assert.NoError(err) + defer fh.Close() + + dec := wav.NewDecoder(fh) + buf, err := dec.FullPCMBuffer() + assert.NoError(err) + assert.Equal(uint16(1), dec.NumChans) + data := buf.AsFloat32Buffer().Data + + model, err := whisper.New(ModelPath) + assert.NoError(err) + assert.NotNil(model) + defer model.Close() + + ctx, err := model.NewContext() + assert.NoError(err) + assert.NotNil(ctx) + defer ctx.Close() + + err = ctx.Process(data, nil, nil, nil) + assert.NoError(err) + + seg, err := ctx.NextSegment() + assert.NoError(err) + assert.NotEmpty(seg.Text) +} + +// TestContext_Parallel_DifferentInputs tests concurrent processing with different inputs +// This validates that each context maintains isolated state for concurrent processing +func TestContext_Parallel_DifferentInputs(t *testing.T) { + assert := assert.New(t) + + if _, err := os.Stat(ModelPath); os.IsNotExist(err) { + t.Skip("Skipping test, model not found:", ModelPath) + } + if _, err := os.Stat(SamplePath); os.IsNotExist(err) { + t.Skip("Skipping test, sample not found:", SamplePath) + } + + fh, err := os.Open(SamplePath) + assert.NoError(err) + defer fh.Close() + + dec := wav.NewDecoder(fh) + buf, err := dec.FullPCMBuffer() + assert.NoError(err) + assert.Equal(uint16(1), dec.NumChans) + data := buf.AsFloat32Buffer().Data + assert.Greater(len(data), 10) + + // Create half-sample (second half) + half := make([]float32, len(data)/2) + copy(half, data[len(data)/2:]) + + model, err := whisper.New(ModelPath) + assert.NoError(err) + assert.NotNil(model) + defer model.Close() + + ctx1, err := model.NewContext() + assert.NoError(err) + defer ctx1.Close() + ctx2, err := model.NewContext() + assert.NoError(err) + defer ctx2.Close() + + // Run in parallel - each context has isolated whisper_state + var wg sync.WaitGroup + var first1, first2 string + var e1, e2 error + + wg.Add(2) + + // No mutex needed because each context is isolated by whisper_state + go func() { + defer wg.Done() + e1 = ctx1.Process(data, nil, nil, nil) + if e1 == nil { + seg, err := ctx1.NextSegment() + if err == nil { + first1 = seg.Text + } else { + e1 = err + } + } + }() + + go func() { + defer wg.Done() + e2 = ctx2.Process(half, nil, nil, nil) + if e2 == nil { + seg, err := ctx2.NextSegment() + if err == nil { + first2 = seg.Text + } else { + e2 = err + } + } + }() + + wg.Wait() + assert.NoError(e1) + assert.NoError(e2) + assert.NotEmpty(first1) + assert.NotEmpty(first2) + assert.NotEqual(first1, first2, "first segments should differ for different inputs") +} + +// TestContext_Close tests that Context.Close() properly frees resources +// and allows context to be used even after it has been closed +func TestContext_Close(t *testing.T) { + assert := assert.New(t) + + if _, err := os.Stat(ModelPath); os.IsNotExist(err) { + t.Skip("Skipping test, model not found:", ModelPath) + } + + model, err := whisper.New(ModelPath) + assert.NoError(err) + assert.NotNil(model) + defer model.Close() + + ctx, err := model.NewContext() + assert.NoError(err) + assert.NotNil(ctx) + + // Close the context + err = ctx.Close() + require.NoError(t, err) + + // Try to use closed context - should return errors + err = ctx.Process([]float32{0.1, 0.2, 0.3}, nil, nil, nil) + require.ErrorIs(t, err, whisper.ErrModelClosed) + + lang := ctx.DetectedLanguage() + require.Empty(t, lang) + + // Multiple closes should be safe + err = ctx.Close() + require.NoError(t, err) +} + +func Test_Close_Context_of_Closed_Model(t *testing.T) { + assert := assert.New(t) + + model, err := whisper.New(ModelPath) + assert.NoError(err) + assert.NotNil(model) + + ctx, err := model.NewContext() + assert.NoError(err) + assert.NotNil(ctx) + + require.NoError(t, model.Close()) + require.NoError(t, ctx.Close()) +} diff --git a/bindings/go/pkg/whisper/interface.go b/bindings/go/pkg/whisper/interface.go index b4705ec1c..30a41db3e 100644 --- a/bindings/go/pkg/whisper/interface.go +++ b/bindings/go/pkg/whisper/interface.go @@ -3,6 +3,8 @@ package whisper import ( "io" "time" + + whisper "github.com/ggerganov/whisper.cpp/bindings/go" ) /////////////////////////////////////////////////////////////////////////////// @@ -20,6 +22,32 @@ type ProgressCallback func(int) // continue processing. It is called during the Process function type EncoderBeginCallback func() bool +type TokenIdentifier interface { + // Test for "begin" token + IsBEG(Token) (bool, error) + + // Test for "start of transcription" token + IsSOT(Token) (bool, error) + + // Test for "end of transcription" token + IsEOT(Token) (bool, error) + + // Test for "start of prev" token + IsPREV(Token) (bool, error) + + // Test for "start of lm" token + IsSOLM(Token) (bool, error) + + // Test for "no timestamps" token + IsNOT(Token) (bool, error) + + // Test for token associated with a specific language + IsLANG(Token, string) (bool, error) + + // Test for text token + IsText(Token) (bool, error) +} + // Model is the interface to a whisper model. Create a new model with the // function whisper.New(string) type Model interface { @@ -33,32 +61,116 @@ type Model interface { // Return all languages supported. Languages() []string + + // Model performance timing methods + // Print model performance timings to stdout + PrintTimings() + + // Reset model performance timing counters + ResetTimings() + + // WhisperContext returns the memory-safe whisper context wrapper of the raw whisper context + WhisperContext() WhisperContext + + // Token identifier + TokenIdentifier() TokenIdentifier +} + +// Parameters configures decode / processing behavior +type Parameters interface { + SetTranslate(bool) + SetSplitOnWord(bool) + SetThreads(uint) + SetOffset(time.Duration) + SetDuration(time.Duration) + SetTokenThreshold(float32) + SetTokenSumThreshold(float32) + SetMaxSegmentLength(uint) + SetTokenTimestamps(bool) + SetMaxTokensPerSegment(uint) + SetAudioCtx(uint) + SetMaxContext(n int) + SetBeamSize(n int) + SetEntropyThold(t float32) + SetInitialPrompt(prompt string) + + // Set the temperature + SetTemperature(t float32) + + // Set the fallback temperature incrementation + // Pass -1.0 to disable this feature + SetTemperatureFallback(t float32) + SetLanguage(string) error + + // Set single segment mode + SetSingleSegment(bool) + + // Getter methods + Language() string + Threads() int + WhisperParams() *whisper.Params } // Context is the speech recognition context. type Context interface { - SetLanguage(string) error // Set the language to use for speech recognition, use "auto" for auto detect language. - SetTranslate(bool) // Set translate flag - IsMultilingual() bool // Return true if the model is multilingual. - Language() string // Get language - DetectedLanguage() string // Get detected language + io.Closer + // Deprecated: Use Params().SetLanguage() instead + SetLanguage(string) error - SetOffset(time.Duration) // Set offset - SetDuration(time.Duration) // Set duration - SetThreads(uint) // Set number of threads to use - SetSplitOnWord(bool) // Set split on word flag - SetTokenThreshold(float32) // Set timestamp token probability threshold - SetTokenSumThreshold(float32) // Set timestamp token sum probability threshold - SetMaxSegmentLength(uint) // Set max segment length in characters - SetTokenTimestamps(bool) // Set token timestamps flag - SetMaxTokensPerSegment(uint) // Set max tokens per segment (0 = no limit) - SetAudioCtx(uint) // Set audio encoder context - SetMaxContext(n int) // Set maximum number of text context tokens to store - SetBeamSize(n int) // Set Beam Size - SetEntropyThold(t float32) // Set Entropy threshold - SetInitialPrompt(prompt string) // Set initial prompt - SetTemperature(t float32) // Set temperature - SetTemperatureFallback(t float32) // Set temperature incrementation + // Deprecated: Use Params().SetTranslate() instead + SetTranslate(bool) + // Deprecated: Use Params().SetSplitOnWord() instead + SetSplitOnWord(bool) + // Deprecated: Use Params().SetThreads() instead + SetThreads(uint) + + // Deprecated: Use Params().SetOffset() instead + SetOffset(time.Duration) + // Deprecated: Use Params().SetDuration() instead + SetDuration(time.Duration) + // Deprecated: Use Params().SetTokenThreshold() instead + SetTokenThreshold(float32) + + // Deprecated: Use Params().SetTokenSumThreshold() instead + SetTokenSumThreshold(float32) + // Deprecated: Use Params().SetMaxSegmentLength() instead + SetMaxSegmentLength(uint) + // Deprecated: Use Params().SetTokenTimestamps() instead + SetTokenTimestamps(bool) + + // Deprecated: Use Params().SetMaxTokensPerSegment() instead + SetMaxTokensPerSegment(uint) + + // Deprecated: Use Params().SetAudioCtx() instead + SetAudioCtx(uint) + + // Deprecated: Use Params().SetMaxContext() instead + SetMaxContext(int) + + // Deprecated: Use Params().SetBeamSize() instead + SetBeamSize(int) + + // Deprecated: Use Params().SetEntropyThold() instead + SetEntropyThold(float32) + + // Deprecated: Use Params().SetTemperature() instead + SetTemperature(float32) + + // Deprecated: Use Params().SetTemperatureFallback() instead + SetTemperatureFallback(float32) + + // Deprecated: Use Params().SetInitialPrompt() instead + SetInitialPrompt(string) + + // Get language of the context parameters + // Deprecated: Use Params().Language() instead + Language() string + + // Return true if the model is multilingual. + IsMultilingual() bool + + // Get detected language + DetectedLanguage() string // Process mono audio data and return any errors. // If defined, newly generated segments are passed to the @@ -69,29 +181,41 @@ type Context interface { // is reached, when io.EOF is returned. NextSegment() (Segment, error) - IsBEG(Token) bool // Test for "begin" token - IsSOT(Token) bool // Test for "start of transcription" token - IsEOT(Token) bool // Test for "end of transcription" token - IsPREV(Token) bool // Test for "start of prev" token - IsSOLM(Token) bool // Test for "start of lm" token - IsNOT(Token) bool // Test for "No timestamps" token - IsLANG(Token, string) bool // Test for token associated with a specific language - IsText(Token) bool // Test for text token + // Deprecated token methods - use Model.IsBEG(), Model.IsSOT(), etc. instead + // Deprecated: Use Model.IsBEG() instead + IsBEG(Token) bool - // Timings + // Deprecated: Use Model.IsSOT() instead + IsSOT(Token) bool + + // Deprecated: Use Model.IsEOT() instead + IsEOT(Token) bool + + // Deprecated: Use Model.IsPREV() instead + IsPREV(Token) bool + + // Deprecated: Use Model.IsSOLM() instead + IsSOLM(Token) bool + + // Deprecated: Use Model.IsNOT() instead + IsNOT(Token) bool + + // Deprecated: Use Model.IsLANG() instead + IsLANG(Token, string) bool + + // Deprecated: Use Model.IsText() instead + IsText(Token) bool + + // Deprecated: Use Model.PrintTimings() instead - these are model-level performance metrics PrintTimings() + + // Deprecated: Use Model.ResetTimings() instead - these are model-level performance metrics ResetTimings() SystemInfo() string -} -// State is a per-request speech recognition state which shares the loaded model -// but isolates recognition results. It embeds Context, so any state-specific -// methods can be added later without breaking existing API. -type State interface { - io.Closer - - Context + // Params returns a high-level parameters wrapper - preferred method + Params() Parameters } // Segment is the text result of a speech recognition. diff --git a/bindings/go/pkg/whisper/model.go b/bindings/go/pkg/whisper/model.go index 0142a787a..752c7cf02 100644 --- a/bindings/go/pkg/whisper/model.go +++ b/bindings/go/pkg/whisper/model.go @@ -9,20 +9,15 @@ import ( whisper "github.com/ggerganov/whisper.cpp/bindings/go" ) -/////////////////////////////////////////////////////////////////////////////// -// TYPES - type model struct { - path string - ctx *whisper.Context + path string + ctx *whisperCtx + tokenIdentifier *tokenIdentifier } // Make sure model adheres to the interface var _ Model = (*model)(nil) -/////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - func New(path string) (Model, error) { model := new(model) if _, err := os.Stat(path); err != nil { @@ -30,7 +25,8 @@ func New(path string) (Model, error) { } else if ctx := whisper.Whisper_init(path); ctx == nil { return nil, ErrUnableToLoadModel } else { - model.ctx = ctx + model.ctx = newWhisperCtx(ctx) + model.tokenIdentifier = newTokenIdentifier(model.ctx) model.path = path } @@ -39,15 +35,11 @@ func New(path string) (Model, error) { } func (model *model) Close() error { - if model.ctx != nil { - model.ctx.Whisper_free() - } + return model.ctx.Close() +} - // Release resources - model.ctx = nil - - // Return success - return nil +func (model *model) WhisperContext() WhisperContext { + return model.ctx } /////////////////////////////////////////////////////////////////////////////// @@ -58,6 +50,7 @@ func (model *model) String() string { if model.ctx != nil { str += fmt.Sprintf(" model=%q", model.path) } + return str + ">" } @@ -66,28 +59,43 @@ func (model *model) String() string { // Return true if model is multilingual (language and translation options are supported) func (model *model) IsMultilingual() bool { - return model.ctx.Whisper_is_multilingual() != 0 + ctx, err := model.ctx.UnsafeContext() + if err != nil { + return false + } + + return ctx.Whisper_is_multilingual() != 0 } // Return all recognized languages. Initially it is set to auto-detect func (model *model) Languages() []string { + ctx, err := model.ctx.UnsafeContext() + if err != nil { + return nil + } + result := make([]string, 0, whisper.Whisper_lang_max_id()) for i := 0; i < whisper.Whisper_lang_max_id(); i++ { str := whisper.Whisper_lang_str(i) - if model.ctx.Whisper_lang_id(str) >= 0 { + if ctx.Whisper_lang_id(str) >= 0 { result = append(result, str) } } + return result } +// NewContext creates a new speech-to-text context. +// Each context is backed by an isolated whisper_state for safe concurrent processing. func (model *model) NewContext() (Context, error) { - if model.ctx == nil { - return nil, ErrInternalAppError + ctx, err := model.ctx.UnsafeContext() + if err != nil { + return nil, ErrModelClosed } - // Create new context - params := model.ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY) + // Create new context with default params + params := ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY) + params.SetTranslate(false) params.SetPrintSpecial(false) params.SetPrintProgress(false) @@ -96,22 +104,31 @@ func (model *model) NewContext() (Context, error) { params.SetThreads(runtime.NumCPU()) params.SetNoContext(true) - // Return new context + // Return new context (now state-backed) return newContext(model, params) } -// NewState returns a new per-request state sharing the loaded model -func (model *model) NewState() (State, error) { - if model.ctx == nil { - return nil, ErrInternalAppError +// PrintTimings prints the model performance timings to stdout. +func (model *model) PrintTimings() { + ctx, err := model.ctx.UnsafeContext() + if err != nil { + return } - params := model.ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY) - params.SetTranslate(false) - params.SetPrintSpecial(false) - params.SetPrintProgress(false) - params.SetPrintRealtime(false) - params.SetPrintTimestamps(false) - params.SetThreads(runtime.NumCPU()) - params.SetNoContext(true) - return newState(model, params) + + ctx.Whisper_print_timings() +} + +// ResetTimings resets the model performance timing counters. +func (model *model) ResetTimings() { + ctx, err := model.ctx.UnsafeContext() + if err != nil { + return + } + + ctx.Whisper_reset_timings() +} + +// WhisperContext returns the low-level whisper context, or error if the model is closed. +func (model *model) TokenIdentifier() TokenIdentifier { + return model.tokenIdentifier } diff --git a/bindings/go/pkg/whisper/params_wrap.go b/bindings/go/pkg/whisper/params_wrap.go new file mode 100644 index 000000000..55a414537 --- /dev/null +++ b/bindings/go/pkg/whisper/params_wrap.go @@ -0,0 +1,69 @@ +package whisper + +import ( + "time" + + // Bindings + whisper "github.com/ggerganov/whisper.cpp/bindings/go" +) + +// parameters is a high-level wrapper that implements the Parameters interface +// and delegates to the underlying low-level whisper.Params. +type parameters struct { + p *whisper.Params +} + +func newParameters(p *whisper.Params) Parameters { return ¶meters{p: p} } + +func (w *parameters) SetTranslate(v bool) { w.p.SetTranslate(v) } +func (w *parameters) SetSplitOnWord(v bool) { w.p.SetSplitOnWord(v) } +func (w *parameters) SetThreads(v uint) { w.p.SetThreads(int(v)) } +func (w *parameters) SetOffset(d time.Duration) { w.p.SetOffset(int(d.Milliseconds())) } +func (w *parameters) SetDuration(d time.Duration) { w.p.SetDuration(int(d.Milliseconds())) } +func (w *parameters) SetTokenThreshold(t float32) { w.p.SetTokenThreshold(t) } +func (w *parameters) SetTokenSumThreshold(t float32) { w.p.SetTokenSumThreshold(t) } +func (w *parameters) SetMaxSegmentLength(n uint) { w.p.SetMaxSegmentLength(int(n)) } +func (w *parameters) SetTokenTimestamps(b bool) { w.p.SetTokenTimestamps(b) } +func (w *parameters) SetMaxTokensPerSegment(n uint) { w.p.SetMaxTokensPerSegment(int(n)) } +func (w *parameters) SetAudioCtx(n uint) { w.p.SetAudioCtx(int(n)) } +func (w *parameters) SetMaxContext(n int) { w.p.SetMaxContext(n) } +func (w *parameters) SetBeamSize(n int) { w.p.SetBeamSize(n) } +func (w *parameters) SetEntropyThold(t float32) { w.p.SetEntropyThold(t) } +func (w *parameters) SetInitialPrompt(prompt string) { w.p.SetInitialPrompt(prompt) } +func (w *parameters) SetTemperature(t float32) { w.p.SetTemperature(t) } +func (w *parameters) SetTemperatureFallback(t float32) { w.p.SetTemperatureFallback(t) } + +func (w *parameters) SetLanguage(lang string) error { + if lang == "auto" { + return w.p.SetLanguage(-1) + } + id := whisper.Whisper_lang_id_str(lang) + if id < 0 { + return ErrUnsupportedLanguage + } + return w.p.SetLanguage(id) +} + +func (w *parameters) SetSingleSegment(v bool) { + w.p.SetSingleSegment(v) +} + +// Getter methods for Parameters interface +func (w *parameters) Language() string { + id := w.p.Language() + if id == -1 { + return "auto" + } + + return whisper.Whisper_lang_str(id) +} + +func (w *parameters) Threads() int { + return w.p.Threads() +} + +func (w *parameters) WhisperParams() *whisper.Params { + return w.p +} + +var _ Parameters = ¶meters{} diff --git a/bindings/go/pkg/whisper/state.go b/bindings/go/pkg/whisper/state.go deleted file mode 100644 index 4ae81baf7..000000000 --- a/bindings/go/pkg/whisper/state.go +++ /dev/null @@ -1,125 +0,0 @@ -package whisper - -import ( - "io" - "strings" - "time" - - // Bindings - whisper "github.com/ggerganov/whisper.cpp/bindings/go" -) - -// state embeds context behavior and carries a low-level state pointer -// for isolated processing results. -type state struct { - *context - st *whisper.State -} - -// NewState creates a new per-request State from a Model without changing the Model interface. -func NewState(m Model) (State, error) { - impl, ok := m.(*model) - if !ok { - return nil, ErrInternalAppError - } - params := impl.ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY) - params.SetTranslate(false) - params.SetPrintSpecial(false) - params.SetPrintProgress(false) - params.SetPrintRealtime(false) - params.SetPrintTimestamps(false) - return newState(impl, params) -} - -// internal constructor used by model.NewState -func newState(model *model, params whisper.Params) (State, error) { - ctx := &context{model: model, params: params} - st := model.ctx.Whisper_init_state() - if st == nil { - return nil, ErrUnableToCreateState - } - return &state{context: ctx, st: st}, nil -} - -// Process using an isolated state for concurrency -func (s *state) Process( - data []float32, - callEncoderBegin EncoderBeginCallback, - callNewSegment SegmentCallback, - callProgress ProgressCallback, -) error { - if s.model.ctx == nil || s.st == nil { - return ErrInternalAppError - } - if callNewSegment != nil { - s.params.SetSingleSegment(true) - } - if err := s.model.ctx.Whisper_full_with_state(s.st, s.params, data, callEncoderBegin, - func(new int) { - if callNewSegment != nil { - num_segments := s.model.ctx.Whisper_full_n_segments_from_state(s.st) - s0 := num_segments - new - for i := s0; i < num_segments; i++ { - callNewSegment(toSegmentFromState(s.model.ctx, s.st, i)) - } - } - }, func(progress int) { - if callProgress != nil { - callProgress(progress) - } - }); err != nil { - return err - } - return nil -} - -// Return the next segment of tokens for state -func (s *state) NextSegment() (Segment, error) { - if s.model.ctx == nil { - return Segment{}, ErrInternalAppError - } - if s.n >= s.model.ctx.Whisper_full_n_segments_from_state(s.st) { - return Segment{}, io.EOF - } - result := toSegmentFromState(s.model.ctx, s.st, s.n) - s.n++ - return result, nil -} - -func (s *state) Close() error { - if s.st != nil { - s.st.Whisper_free_state() - s.st = nil - } - return nil -} - -// Helpers specific to state-based results -func toSegmentFromState(ctx *whisper.Context, st *whisper.State, n int) Segment { - return Segment{ - Num: n, - Text: stringsTrim(ctx.Whisper_full_get_segment_text_from_state(st, n)), - Start: duration10x(ctx.Whisper_full_get_segment_t0_from_state(st, n)), - End: duration10x(ctx.Whisper_full_get_segment_t1_from_state(st, n)), - Tokens: toTokensFromState(ctx, st, n), - } -} - -func toTokensFromState(ctx *whisper.Context, st *whisper.State, n int) []Token { - result := make([]Token, ctx.Whisper_full_n_tokens_from_state(st, n)) - for i := 0; i < len(result); i++ { - data := ctx.Whisper_full_get_token_data_from_state(st, n, i) - result[i] = Token{ - Id: int(ctx.Whisper_full_get_token_id_from_state(st, n, i)), - Text: ctx.Whisper_full_get_token_text_from_state(st, n, i), - P: ctx.Whisper_full_get_token_p_from_state(st, n, i), - Start: duration10x(data.T0()), - End: duration10x(data.T1()), - } - } - return result -} - -// small shared helpers to avoid importing time/strings here unnecessarily -func stringsTrim(s string) string { return strings.TrimSpace(s) } -func duration10x(ms10 int64) time.Duration { return time.Duration(ms10) * time.Millisecond * 10 } diff --git a/bindings/go/pkg/whisper/state_test.go b/bindings/go/pkg/whisper/state_test.go deleted file mode 100644 index f893b8d2b..000000000 --- a/bindings/go/pkg/whisper/state_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package whisper_test - -import ( - "os" - "sync" - "testing" - - "github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper" - "github.com/go-audio/wav" - assert "github.com/stretchr/testify/assert" -) - -func TestState_Process(t *testing.T) { - assert := assert.New(t) - - if _, err := os.Stat(ModelPath); os.IsNotExist(err) { - t.Skip("Skipping test, model not found:", ModelPath) - } - if _, err := os.Stat(SamplePath); os.IsNotExist(err) { - t.Skip("Skipping test, sample not found:", SamplePath) - } - - fh, err := os.Open(SamplePath) - assert.NoError(err) - defer fh.Close() - - dec := wav.NewDecoder(fh) - buf, err := dec.FullPCMBuffer() - assert.NoError(err) - assert.Equal(uint16(1), dec.NumChans) - data := buf.AsFloat32Buffer().Data - - model, err := whisper.New(ModelPath) - assert.NoError(err) - assert.NotNil(model) - defer model.Close() - - st, err := whisper.NewState(model) - assert.NoError(err) - assert.NotNil(st) - defer func() { _ = st.Close() }() - - err = st.Process(data, nil, nil, nil) - assert.NoError(err) - - seg, err := st.NextSegment() - assert.NoError(err) - assert.NotEmpty(seg.Text) -} - -func TestState_Parallel_DifferentInputs(t *testing.T) { - assert := assert.New(t) - - if _, err := os.Stat(ModelPath); os.IsNotExist(err) { - t.Skip("Skipping test, model not found:", ModelPath) - } - if _, err := os.Stat(SamplePath); os.IsNotExist(err) { - t.Skip("Skipping test, sample not found:", SamplePath) - } - - fh, err := os.Open(SamplePath) - assert.NoError(err) - defer fh.Close() - - dec := wav.NewDecoder(fh) - buf, err := dec.FullPCMBuffer() - assert.NoError(err) - assert.Equal(uint16(1), dec.NumChans) - data := buf.AsFloat32Buffer().Data - assert.Greater(len(data), 10) - - // Create half-sample (second half) - half := make([]float32, len(data)/2) - copy(half, data[len(data)/2:]) - - model, err := whisper.New(ModelPath) - assert.NoError(err) - assert.NotNil(model) - defer model.Close() - - st1, err := whisper.NewState(model) - assert.NoError(err) - st2, err := whisper.NewState(model) - assert.NoError(err) - defer func() { _ = st1.Close() }() - defer func() { _ = st2.Close() }() - - // Run in parallel, but guard core call to respect context safety - var wg sync.WaitGroup - var first1, first2 string - var e1, e2 error - - wg.Add(2) - - // No mutex needed because each state is isolated - go func() { - defer wg.Done() - e1 = st1.Process(data, nil, nil, nil) - if e1 == nil { - seg, err := st1.NextSegment() - if err == nil { - first1 = seg.Text - } else { - e1 = err - } - } - }() - - go func() { - defer wg.Done() - e2 = st2.Process(half, nil, nil, nil) - if e2 == nil { - seg, err := st2.NextSegment() - if err == nil { - first2 = seg.Text - } else { - e2 = err - } - } - }() - - wg.Wait() - assert.NoError(e1) - assert.NoError(e2) - assert.NotEmpty(first1) - assert.NotEmpty(first2) - assert.NotEqual(first1, first2, "first segments should differ for different inputs") -} diff --git a/bindings/go/pkg/whisper/token_identifier.go b/bindings/go/pkg/whisper/token_identifier.go new file mode 100644 index 000000000..059386c1e --- /dev/null +++ b/bindings/go/pkg/whisper/token_identifier.go @@ -0,0 +1,115 @@ +package whisper + +import whisper "github.com/ggerganov/whisper.cpp/bindings/go" + +type tokenIdentifier struct { + ctx *whisperCtx +} + +func newTokenIdentifier(whisperContext *whisperCtx) *tokenIdentifier { + return &tokenIdentifier{ + ctx: whisperContext, + } +} + +// Token type checking methods (model-specific vocabulary) +func (ti *tokenIdentifier) IsBEG(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_beg(), nil +} + +func (ti *tokenIdentifier) IsEOT(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_eot(), nil +} + +func (ti *tokenIdentifier) IsSOT(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_sot(), nil +} + +func (ti *tokenIdentifier) IsPREV(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_prev(), nil +} + +func (ti *tokenIdentifier) IsSOLM(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_solm(), nil +} + +func (ti *tokenIdentifier) IsNOT(t Token) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + return whisper.Token(t.Id) == ctx.Whisper_token_not(), nil +} + +func (ti *tokenIdentifier) IsLANG(t Token, lang string) (bool, error) { + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + if id := ctx.Whisper_lang_id(lang); id >= 0 { + return whisper.Token(t.Id) == ctx.Whisper_token_lang(id), nil + } + + return false, nil +} + +func (ti *tokenIdentifier) IsText(t Token) (bool, error) { + // Check if it's any of the special tokens + if isBeg, _ := ti.IsBEG(t); isBeg { + return false, nil + } + + if isSot, _ := ti.IsSOT(t); isSot { + return false, nil + } + + ctx, err := ti.ctx.UnsafeContext() + if err != nil { + return false, err + } + + if whisper.Token(t.Id) >= ctx.Whisper_token_eot() { + return false, nil + } + + if isPrev, _ := ti.IsPREV(t); isPrev { + return false, nil + } + + if isSolm, _ := ti.IsSOLM(t); isSolm { + return false, nil + } + + if isNot, _ := ti.IsNOT(t); isNot { + return false, nil + } + + return true, nil +} diff --git a/bindings/go/pkg/whisper/whisper_ctx.go b/bindings/go/pkg/whisper/whisper_ctx.go new file mode 100644 index 000000000..7e570bc3d --- /dev/null +++ b/bindings/go/pkg/whisper/whisper_ctx.go @@ -0,0 +1,49 @@ +package whisper + +import whisper "github.com/ggerganov/whisper.cpp/bindings/go" + +type WhisperContext interface { + // Close closes the whisper context + Close() error + + // IsClosed returns true if the whisper context is closed + IsClosed() bool + + // UnsafeContext returns the raw whisper context + UnsafeContext() (*whisper.Context, error) +} + +type whisperCtx struct { + ctx *whisper.Context +} + +func newWhisperCtx(ctx *whisper.Context) *whisperCtx { + return &whisperCtx{ + ctx: ctx, + } +} + +func (ctx *whisperCtx) Close() error { + if ctx.ctx == nil { + return nil + } + + ctx.ctx.Whisper_free() + ctx.ctx = nil + + return nil +} + +func (ctx *whisperCtx) IsClosed() bool { + return ctx.ctx == nil +} + +func (ctx *whisperCtx) UnsafeContext() (*whisper.Context, error) { + if ctx.IsClosed() { + return nil, ErrModelClosed + } + + return ctx.ctx, nil +} + +var _ WhisperContext = (*whisperCtx)(nil) diff --git a/bindings/go/pkg/whisper/whisper_state.go b/bindings/go/pkg/whisper/whisper_state.go new file mode 100644 index 000000000..24ca4ed44 --- /dev/null +++ b/bindings/go/pkg/whisper/whisper_state.go @@ -0,0 +1,37 @@ +package whisper + +import whisper "github.com/ggerganov/whisper.cpp/bindings/go" + +type WhisperState interface { + Close() error + UnsafeState() (*whisper.State, error) +} + +type whisperState struct { + state *whisper.State +} + +func newWhisperState(state *whisper.State) WhisperState { + return &whisperState{ + state: state, + } +} + +func (s *whisperState) Close() error { + if s.state == nil { + return nil + } + + s.state.Whisper_free_state() + s.state = nil + + return nil +} + +func (s *whisperState) UnsafeState() (*whisper.State, error) { + if s.state == nil { + return nil, ErrModelClosed + } + + return s.state, nil +} diff --git a/bindings/go/whisper.go b/bindings/go/whisper.go index cb5907ffe..b6ef48a85 100644 --- a/bindings/go/whisper.go +++ b/bindings/go/whisper.go @@ -232,6 +232,10 @@ func (ctx *Context) Whisper_lang_id(lang string) int { return int(C.whisper_lang_id(C.CString(lang))) } +func Whisper_lang_id_str(lang string) int { + return int(C.whisper_lang_id(C.CString(lang))) +} + // Largest language id (i.e. number of available languages - 1) func Whisper_lang_max_id() int { return int(C.whisper_lang_max_id())