How to Make Your AI Voice Sound More Human and Less Robotic with C#

dotnet new web -o VoiceHumanizer
cd VoiceHumanizer

OPENAI_KEY=your_actual_openai_api_key_here
DOMAIN=ABC12345.ngrok.app

dotnet add package dotenv.net

using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
// Load .env file into environment variables
var envPath = Path.Combine(Directory.GetCurrentDirectory(), ".env");
if (File.Exists(envPath))
{
   foreach (var line in File.ReadAllLines(envPath))
   {
       var parts = line.Split('=', 2);
       if (parts.Length == 2)
           Environment.SetEnvironmentVariable(parts[0].Trim(), parts[1].Trim());
   }
}
var builder = WebApplication.CreateBuilder(args);
var app = builder.Build();
app.UseWebSockets();
app.MapPost("/voice", (IConfiguration config) =>
{
   var domain = Environment.GetEnvironmentVariable("DOMAIN") ?? config["DOMAIN"] ?? "";
   var twiml = $"""
   <Response>
       <Connect>
           <ConversationRelay
               url="wss://{domain}/ws"
               welcomeGreeting="Hey there! Oh no, I see you missed your flight. Don't worry, we'll get you sorted out. Where are you trying to head?"
               ttsProvider="ElevenLabs"
               transcriptionProvider="Deepgram"
               speechModel="nova-3-general"
               eotThreshold="0.8"
               ignoreBackchannel="true"
               interruptible="any"
               interruptSensitivity="medium"
               elevenlabsTextNormalization="auto"
               />
       </Connect>
   </Response>
   """;
   return Results.Content(twiml, "application/xml");
});
// Map our WebSocket path
app.Map("/ws", async (HttpContext context) =>
{
   if (context.WebSockets.IsWebSocketRequest)
   {
       using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
       await HandleVoiceSession(webSocket, app.Configuration);
   }
   else
   {
       context.Response.StatusCode = StatusCodes.Status400BadRequest;
   }
});
app.Run();

static async Task HandleVoiceSession(WebSocket webSocket, IConfiguration config)
{
   var buffer = new byte[1024 * 64];
   var openAiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? config["OPENAI_API_KEY"] ?? "YOUR_OPENAI_API_KEY";
   // keep track of our active streaming task so we can cancel it on demand
   CancellationTokenSource? activeLlmCts = null;
   try
   {
       while (webSocket.State == WebSocketState.Open)
       {
           var ms = new MemoryStream();
           WebSocketReceiveResult result;
           do
           {
               result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
               ms.Write(buffer, 0, result.Count);
           } while (!result.EndOfMessage);
           if (result.MessageType == WebSocketMessageType.Close) break;
           var rawJson = Encoding.UTF8.GetString(ms.ToArray());
           using var doc = JsonDocument.Parse(rawJson);
           var root = doc.RootElement;
           if (!root.TryGetProperty("type", out var typeProp)) continue;
           var msgType = typeProp.GetString();
           // Case A: The user spoke!
           if (msgType == "prompt")
           {
               var userText = root.GetProperty("voicePrompt").GetString();
               Console.WriteLine($"[User Spoke]: {userText}");
               // Cancel any outbound speech that is currently processing
               activeLlmCts?.Cancel();
               activeLlmCts = new CancellationTokenSource();
               // Fire the task in the background so our loop can keep listening for interruptions!
               _ = StreamLlmResponseToTwilio(webSocket, userText, openAiKey, activeLlmCts.Token);
           }
           // Case B: The user cut the robot off mid-sentence!
           else if (msgType == "interrupt")
           {
               Console.WriteLine("Interruption detected.");
               // Tell our local background task to stop calling OpenAI
               activeLlmCts?.Cancel();
               // Tell Twilio's audio player to clear its current buffer instantly
               var clearMsg = JsonSerializer.Serialize(new { type = "clear" });
               var bytes = Encoding.UTF8.GetBytes(clearMsg);
               await webSocket.SendAsync(new ArraySegment<byte>(bytes), WebSocketMessageType.Text, true, CancellationToken.None);
           }
       }
   }
   catch (Exception ex)
   {
       Console.WriteLine($"Session error: {ex.Message}");
   }
}

static async Task StreamLlmResponseToTwilio(WebSocket webSocket, string userInput, string apiKey, CancellationToken ct)
{
   using var client = new HttpClient();
   client.DefaultRequestHeaders.Add("Authorization", $"Bearer {apiKey}");
   var payload = new
   {
       model = "gpt-4o",
       stream = true,
       messages = new[]
       {
           new {
               role = "system",
               content = @"You are an empathetic, lightning-fast airport flight concierge.
                          Keep answers extremely short (under 2 sentences) because this is a phone call.
                          Insert a comma or ellipsis '...' where you want natural breathing pauses.
                          Use occasional casual conversational fillers like 'Oh, okay...', 'Uhm...', or 'Let me check...' to sound natural.
Use plain sentences only. No bullet points, no markdown, no emojis. Spell out all numbers (""thirty dollars"", not ""$30""). "
           },
           new { role = "user", content = userInput }
       }
   };
   var request = new HttpRequestMessage(HttpMethod.Post, "https://api.openai.com/v1/chat/completions")
   {
       Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
   };
   try
   {
       using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct);
       using var stream = await response.Content.ReadAsStreamAsync(ct);
       using var reader = new StreamReader(stream);
       while (!reader.EndOfStream && !ct.IsCancellationRequested)
       {
           var line = await reader.ReadLineAsync(ct);
           if (string.IsNullOrWhiteSpace(line) || !line.StartsWith("data: ")) continue;
           var data = line["data: ".Length..];
           if (data == "[DONE]") break;
           using var doc = JsonDocument.Parse(data);
           var delta = doc.RootElement.GetProperty("choices")[0].GetProperty("delta");
           if (delta.TryGetProperty("content", out var contentProp))
           {
               var textToken = contentProp.GetString();
               // Humanizer Trick: Dynamic SSML Formatting!
               // If the LLM generates a dramatic ellipsis '...', turn it into an explicit breathing break.
               if (textToken == "...")
               {
                   textToken = "<break time=\"200ms\"/>";
               }
               // Format token to Twilio specifications
               var twilioMsg = JsonSerializer.Serialize(new
               {
                   type = "text",
                   token = textToken,
                   last = false // Tells Twilio to expect more streaming speech chunks
               });
               var bytes = Encoding.UTF8.GetBytes(twilioMsg);
               await webSocket.SendAsync(new ArraySegment<byte>(bytes), WebSocketMessageType.Text, true, ct);
           }
       }
       // Send a final token indicating that this conversational turn is complete
       var finalMsg = JsonSerializer.Serialize(new { type = "text", token = "", last = true });
       await webSocket.SendAsync(new ArraySegment<byte>(Encoding.UTF8.GetBytes(finalMsg)), WebSocketMessageType.Text, true, CancellationToken.None);
   }
   catch (OperationCanceledException)
   {
       Console.WriteLine("Llm streaming task cancelled successfully.");
   }
}

dotnet run

ngrok http 5000