Azure Speech recognition not working with Angular + .net core

We are trying to implement azure speech recognition in realtime using its PushStream. The audio is recorded at angular app using navigator.mediaDevices.getUserMedia(). The recorded audio when sent to backend application(.net core) using socket.io, the audio is not recognized by azure. We are using .net core 8.0, angular 17, node 18.

Below combination works, Record audio at react/angular and send it to nodejs backend using socket.io package. The received audio stream at nodejs when sent to Azure Speech to text recognition, works seamlessly.

However, when sent via .net core it is not working. There is no maintained socket.io library available for .net core. We've used signalR at .net core which receives audio data in base64Encoded and converted into byte[] to be sent to Azure. But this doesnt work. Also tried with WebSockets at .net core, that dint work as well.

Basically, we need to know how to make it work by sending recorded audio from angular, send it to .net core, which will use PushStream to send to Azure in realtime. We are not using JS based support from Azure for following reasons,

1. Tech expertise for backend available at .net not on nodeJS
2. Hardcoding azure speech config at angular or react is not an option for security reasons.

Below is the code that works with angular+nodejs+azure,

app.component.ts
import { Component, OnInit } from '@angular/core';
import { SocketIoService } from './socket-io.service';

const sampleRate = 16000;
const getMediaStream = () =>
navigator.mediaDevices.getUserMedia({
audio: {
deviceId: "default",
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1,
},
video: false,
});

@Component({
selector: 'app-root',
templateUrl: './app.component.html',
styleUrls: ['./app.component.css']
})
export class AppComponent implements OnInit {
messageText: string = "";
messages: Array<any> = new Array();;
private audioContext: any = null;
private processor: any = null;
private audioInput: any = null;
isRecording = false;
currentRecognition: string = '...';
recognitionHistory: string[] = [];
constructor(private socketService: SocketIoService) { }

ngOnInit() {
this.socketService.onReceiveAudioText((data) => this.speechRecognized(data));
this.socketService.onReceiveAzureAudioInterimText((text) => this.azureInterimSpeechRecognized(text));
this.socketService.onReceiveAzureAudioFinalText((text) => this.azureFinalSpeechRecognized(text));
}

async startRecording() {
try {
this.socketService.connect();

const stream = await navigator.mediaDevices.getUserMedia({
audio: {
deviceId: 'default',
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1
},
video: false
});

this.audioContext = new AudioContext();
await this.audioContext.audioWorklet.addModule("/assets/recorderWorkletProcessor.js");

this.audioInput = this.audioContext.createMediaStreamSource(stream);
this.processor = new AudioWorkletNode(this.audioContext, 'recorder.worklet');

this.processor.connect(this.audioContext.destination);
this.audioInput.connect(this.processor);

this.processor.port.onmessage = (event: any) => {
const audioData = event.data;
this.socketService.sendAudioData(audioData);
};

this.isRecording = true;
}
catch(err) {
console.log(err);
}
}

stopRecording() {
this.socketService.disconnect();
this.processor.disconnect();
this.audioInput.disconnect();
this.audioContext.close();
this.isRecording = false;
}

private speechRecognized(data: any) {
if (data.isFinal) {
this.currentRecognition = '...';
this.recognitionHistory.unshift(data.text);
} else {
this.currentRecognition = data.text + '...';
}
console.log('received : ', data.text);
}

private azureInterimSpeechRecognized(text: any) {
this.currentRecognition = text + '...';
console.log('received : ', text);
}
private azureFinalSpeechRecognized(text: any) {
this.currentRecognition = '...';
this.recognitionHistory.unshift(text);
console.log('received : ', text);
}

ngOnDestroy(): void {
if (this.isRecording) {
this.stopRecording();
}
}

}

// src/app/services/socket-io.service.ts
import { Injectable } from '@angular/core';
import { io, Socket } from 'socket.io-client';

@Injectable({
providedIn: 'root'
})
export class SocketIoService {
private socket: Socket;

constructor() {
this.socket = io('http://localhost:8081');
}

connect() {
this.socket.emit('send_message', 'hello world');
}

disconnect() {
this.socket.disconnect();
}

sendAudioData(audioData: Float32Array) {
this.socket.emit('send_audio_data', { audio: audioData });
}

onReceiveMessage(callback: (data: any) => void) {
this.socket.on('receive_message', callback);
}

onReceiveAudioText(callback: (data: any) => void) {
this.socket.on('receive_audio_text', callback);
}

onReceiveAzureAudioInterimText(callback: (text: any) => void) {
this.socket.on('interim_result', callback);
}

onReceiveAzureAudioFinalText(callback: (text: any) => void) {
this.socket.on('final_result', callback);
}
}

//node index.js

const express = require("express");
const fs = require("fs");
const sdk = require("microsoft-cognitiveservices-speech-sdk");

const speechConfig = sdk.SpeechConfig.fromSubscription("xxx", "xxx");
speechConfig.speechRecognitionLanguage = "en-US";

//use logger
const logger = require("morgan");

//use body parser
const bodyParser = require("body-parser");

//use corrs
const cors = require("cors");

const http = require("http");
const { Server } = require("socket.io");

const app = express();

app.use(cors());
app.use(logger("dev"));

app.use(bodyParser.json());

const server = http.createServer(app);

const io = new Server(server, {
cors: {
origin: "*",
methods: ["GET", "POST"],
},
});

io.on("connection", (socket) => {
console.log("** a user connected - " + socket.id + " **\n");

// Create the recognizer and PushAudioInputStream
const pushStream = sdk.AudioInputStream.createPushStream();
const audioConfig = sdk.AudioConfig.fromStreamInput(pushStream);
const recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);

recognizer.recognizing = (s, e) => {
console.log(`RECOGNIZING: Text=${e.result.text}`);
socket.emit("interim_result", e.result.text);
};

recognizer.recognized = (s, e) => {
if (e.result.reason === sdk.ResultReason.RecognizedSpeech) {
console.log(`RECOGNIZED: Text=${e.result.text}`);
socket.emit("final_result", e.result.text);
} else if (e.result.reason === sdk.ResultReason.NoMatch) {
console.log("No speech could be recognized.");
}
};

recognizer.startContinuousRecognitionAsync();

// Receive audio data from client via socket.io and push to the audio stream
socket.on("send_audio_data", (data) => {
pushStream.write(data.audio); // 'data.audio' is the raw audio stream bytes
});

socket.on("disconnect", () => {
console.log("Client disconnected");
recognizer.stopContinuousRecognitionAsync();
pushStream.close();
});
});

server.listen(8081, () => {
console.log("WebSocket server listening on port 8081.");
});

const encoding = "LINEAR16";
const sampleRateHertz = 16000;
const languageCode = "ko-KR"; //en-US
const alternativeLanguageCodes = ["en-US", "ko-KR"];

const request = {
config: {
encoding: encoding,
sampleRateHertz: sampleRateHertz,
languageCode: "en-US",
//alternativeLanguageCodes: alternativeLanguageCodes,
enableWordTimeOffsets: true,
enableAutomaticPunctuation: true,
enableWordConfidence: true,
enableSpeakerDiarization: true,
//diarizationSpeakerCount: 2,
//model: "video",
model: "command_and_search",
//model: "default",
useEnhanced: true,
},
interimResults: true,
};


Below code with angular + .net core + Azure that is not working,

import { Component, OnInit } from '@angular/core';
import { SignalrService } from './signalr.service';

const sampleRate = 16000;
const getMediaStream = () =>
navigator.mediaDevices.getUserMedia({
audio: {
deviceId: "default",
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1,
},
video: false,
});

@Component({
selector: 'app-root',
templateUrl: './app.component.html',
styleUrls: ['./app.component.css']
})
export class AppComponent implements OnInit {
messageText: string = "";
messages: Array<any> = new Array();;
private audioContext: any = null;
private processor: any = null;
private audioInput: any = null;
isRecording = false;
currentRecognition: string = '...';
recognitionHistory: string[] = [];
constructor(
//private socketService: SocketIoService,
private signalrService: SignalrService) { }

ngOnInit() { }

async startRecording() {
try {
this.signalrService.startConnection();
this.signalrService.sendMessage("client msg");
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
deviceId: 'default',
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1
},
video: false
});

this.audioContext = new AudioContext();
await this.audioContext.audioWorklet.addModule("/assets/recorderWorkletProcessor.js");

this.audioInput = this.audioContext.createMediaStreamSource(stream);
this.processor = new AudioWorkletNode(this.audioContext, 'recorder.worklet');

this.processor.connect(this.audioContext.destination);
this.audioInput.connect(this.processor);

this.processor.port.onmessage = (event: any) => {
const audioData = event.data;
this.signalrService.sendAudioData(audioData);
};

this.isRecording = true;
}
catch(err) {
console.log(err);
}
}

stopRecording() {
this.processor.disconnect();
this.audioInput.disconnect();
this.audioContext.close();
this.isRecording = false;
}

private speechRecognized(data: any) {
if (data.isFinal) {
this.currentRecognition = '...';
this.recognitionHistory.unshift(data.text);
} else {
this.currentRecognition = data.text + '...';
}
console.log('received : ', data.text);
}

private azureInterimSpeechRecognized(text: any) {
this.currentRecognition = text + '...';
console.log('received : ', text);
}
private azureFinalSpeechRecognized(text: any) {
this.currentRecognition = '...';
this.recognitionHistory.unshift(text);
console.log('received : ', text);
}

ngOnDestroy(): void {
if (this.isRecording) {
this.stopRecording();
}
}

}

//signalr.service.ts
import * as signalR from "@microsoft/signalr";
import { Injectable } from '@angular/core';

@Injectable({
providedIn: 'root'
})
export class SignalrService {
private hubConnection: signalR.HubConnection;

constructor() {
this.hubConnection = new signalR.HubConnectionBuilder()
.withUrl("http://localhost:8081/hub", {
withCredentials: true, // Ensure credentials are included in the request
transport: signalR.HttpTransportType.WebSockets
})
.build();
}

startConnection() {
this.hubConnection
.start()
.then(() => {
console.log("Connection started");
})
.catch(err => {
console.log("Error while starting connection: " + err)
});
}

sendMessage(message: string) {
this.hubConnection.invoke("SendMessage", message)
.catch(err => {
console.error(err)
});
}

receiveMessage() {
this.hubConnection.on("ReceiveMessage", (data: string) => {
console.log("Received message: " + data);
});
}

sendAudioData(audioData: Float32Array) {
const plainArray = Array.from(audioData);
const byteArray = new Uint8Array(audioData);
var base64String = btoa(String.fromCharCode(...byteArray));
this.hubConnection.invoke('SendAudioMessage', base64String)
.catch(err => {
console.error(err)
});
}

}


//.net core 8.0 backend
//Program.cs
using Microsoft.AspNetCore.Http;
using System.Net.Http;
using System.Net.WebSockets;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.AspNetCore.SignalR;
using System.Buffers;
using Microsoft.CognitiveServices.Speech.Audio;
using Microsoft.CognitiveServices.Speech;

internal class Program
{
public static void Main(string[] args)
{
Console.WriteLine("Hello, World!");
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.ConfigureKestrel(serverOptions =>
{
serverOptions.ListenAnyIP(8081);
//serverOptions.ListenAnyIP(8081, listenOptions => listenOptions.UseHttps()); // For HTTPS
});
builder.Services.AddSignalR();
builder.Services.AddCors(options =>
{
options.AddPolicy("AllowSpecificOrigins", builder =>
{
builder.WithOrigins("http://localhost:4200") // Only allow this specific origin
.AllowAnyMethod()
.AllowAnyHeader()
.AllowCredentials(); // Allow credentials (cookies, authentication headers, etc.)
});
});
var app = builder.Build();

app.UseCors("AllowSpecificOrigins");

app.UseRouting();
app.MapHub<ChatHub>("/hub");
app.Run();
}

}

public class ChatHub : Hub
{
private readonly IHubContext<ChatHub> _hubContext;
private SpeechRecognizer _recognizer;
private PushAudioInputStream _pushStream;

public ChatHub(IHubContext<ChatHub> hubContext)
{
_hubContext = hubContext;
InitializeSpeechRecognizer();
}

private async void InitializeSpeechRecognizer()
{
var config = SpeechConfig.FromSubscription("xxx", "xxx");
_pushStream = AudioInputStream.CreatePushStream();
var audioInput = AudioConfig.FromStreamInput(_pushStream);
_recognizer = new SpeechRecognizer(config, audioInput);

_recognizer.Recognizing += async (s, e) =>
{
if (!string.IsNullOrEmpty(e.Result.Text))
{
Console.WriteLine("Recognized : " + e.Result.Text);
await Clients.Caller.SendAsync("ReceiveText", e.Result.Text);
}
};

_recognizer.Recognized += async (s, e) =>
{
if (e.Result.Reason == ResultReason.RecognizedSpeech)
{
Console.WriteLine("Recognized : " + e.Result.Text);
await Clients.Caller.SendAsync("ReceiveText", e.Result.Text);
}
};

_recognizer.Canceled += (s, e) =>
{
Console.WriteLine($"Recognition canceled: {e.Reason}, {e.ErrorDetails}");
};

_recognizer.SessionStopped += (s, e) =>
{
Console.WriteLine("Speech session stopped.");
};

await _recognizer.StartContinuousRecognitionAsync();
}

public void StreamAudio(byte[] audioChunk)
{
// Push audio chunks into the stream for real-time recognition
_pushStream.Write(audioChunk);
}

public void StopRecognition()
{
_pushStream.Close();
}
public async Task SendMessage(string message)
{
Console.WriteLine($"Received message: {message}");
await Clients.All.SendAsync("receive_message", message);
}

public async Task SendAudioMessage(string audioBase64Data)
{
Console.WriteLine($"Received message: ");
byte[] bytes = Convert.FromBase64String(audioBase64Data);
StreamAudio(bytes);
}

public async Task ReceiveMessage(string message)
{
Console.WriteLine($"Received message: {message}");
await Clients.Caller.SendAsync("response_message", $"Server received: {message}");
}
}

Continue reading...