How to Orchestrate Multi-Call Conversations with an LLM and Twilio Conversation in Node.js Memory

mkdir twilio-multi-call-memory
cd twilio-multi-call-memory
npm init -y

npm install express express-ws openai dotenv axios

OPENAI_API_KEY=sk-...
TWILIO_API_KEY=SK...
TWILIO_API_SECRET=...
TWILIO_CONFIGURATION_ID=cnv_config_...
TWILIO_MEMORY_STORE_ID=mem_store_...
TWILIO_PHONE_NUMBER=+15551234567

const OpenAI = require('openai');
const MODEL = 'gpt-4o-mini';
const BASE_SYSTEM_PROMPT = `You are the phone assistant for Owlbert's Auto Repair. You are friendly, concise, and speak naturally as if on the phone.
Do not use lists, bullet points, or emojis — respond in plain sentences.
If the caller mentions their name, vehicle, or a problem with their car, remember it and refer back to it naturally as the conversation continues.
If you do not know something, say so honestly rather than guessing.
You have access to memory of previous conversations with this caller, which may include their name, vehicle, and prior issues.
Use that information to provide helpful and personalized responses.`;
const SUMMARIZER_PROMPT = `You are summarizing a phone call between a caller and Owlbert's Auto Repair.
Write a concise summary in 4-8 sentences that captures: the caller's name if given, their vehicle if mentioned, the reason for the call, any symptoms or diagnostic detail discussed, any prices or estimates mentioned, and any commitments or next steps agreed upon.
Write in plain prose (no lists or bullets). If the caller did not share meaningful information (e.g. the call was very short or unclear), respond with a single sentence noting that.
Maximum 4000 characters.`;
if (!process.env.OPENAI_API_KEY) {
  throw new Error('OPENAI_API_KEY is not set.');
}
const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
const streamResponse = async (ws, callSid, memoryContext, messages) => {
  const systemPrompt = memoryContext
    ? `${BASE_SYSTEM_PROMPT}\n\nPrior context on this customer (from previous calls):\n${memoryContext}`
    : BASE_SYSTEM_PROMPT;
  const openAiMessages = [
    { role: 'system', content: systemPrompt },
    ...messages.map((m) => ({ role: m.role, content: m.content })),
  ];
  let fullResponse = '';
  try {
    const stream = await client.chat.completions.create({
      model: MODEL,
      messages: openAiMessages,
      max_tokens: 400,
      stream: true,
    });
    for await (const chunk of stream) {
      const token = chunk.choices[0]?.delta?.content;
      if (token) {
        fullResponse += token;
        ws.send(JSON.stringify({ type: 'text', token, last: false }));
      }
    }
  } finally {
    ws.send(JSON.stringify({ type: 'text', token: '', last: true }));
  }
  console.log(`[${callSid}] Assistant: ${fullResponse}`);
  return fullResponse;
};
const summarizeConversation = async (messages) => {
  if (messages.length === 0) return '';
  const transcript = messages
    .map((m) => `${m.role.toUpperCase()}: ${m.content}`)
    .join('\n');
  const result = await client.chat.completions.create({
    model: MODEL,
    max_tokens: 500,
    messages: [
      { role: 'system', content: SUMMARIZER_PROMPT },
      { role: 'user', content: `Transcript:\n${transcript}\n\nSummary:` },
    ],
  });
  const summary = result.choices[0]?.message?.content ?? '';
  console.log(`Call summary (${summary.length} chars): ${summary}`);
  return summary;
};
module.exports = { streamResponse, summarizeConversation };

const { streamResponse, summarizeConversation } = require('./openaiService');
const { startCall, finishCall } = require('./conversationMemoryService');
const handle = (ws) => {
  let callSid = null;
  let callerPhone = '';
  let profileId = null;
  let conversationId = null;
  let memoryContext = null;
  const messages = [];
  ws.on('message', async (data) => {
    let msg;
    try {
      msg = JSON.parse(data.toString());
    } catch (err) {
      console.warn('Invalid JSON received:', err);
      return;
    }
    switch (msg.type) {
      case 'setup': {
        callSid = msg.callSid ?? null;
        callerPhone = msg.from ?? '';
        console.log(`[${callSid}] Call connected from ${callerPhone}`);
        try {
          const ctx = await startCall(callerPhone);
          if (ctx) {
            conversationId = ctx.conversationId;
            profileId = ctx.profileId || null;
            memoryContext = ctx.memoryContext;
            console.log(
              `[${callSid}] Conversation ${conversationId}, profile ${profileId ?? '(none)'}, memory ${memoryContext?.length ?? 0} chars`
            );
          }
        } catch (err) {
          console.warn(`[${callSid}] startCall failed — continuing without memory:`, err);
        }
        break;
      }
      case 'prompt': {
        if (!msg.last) break;
        const userText = msg.voicePrompt ?? '';
        console.log(`[${callSid}] Caller: ${userText}`);
        messages.push({ role: 'user', content: userText });
        const responseText = await streamResponse(ws, callSid, memoryContext, messages);
        messages.push({ role: 'assistant', content: responseText });
        break;
      }
      case 'interrupt': {
        const spoken = msg.utteranceUntilInterrupt ?? '';
        console.log(`[${callSid}] Interrupted after: '${spoken}'`);
        if (messages.length > 0 && messages[messages.length - 1].role === 'assistant') {
          messages.pop();
        }
        break;
      }
      case 'error': {
        console.warn(`[${callSid}] Conversation Relay error: ${msg.description ?? ''}`);
        break;
      }
    }
  });
  ws.on('close', async () => {
    console.log(`[${callSid}] Call ended`);
    if (profileId && conversationId && messages.length > 0) {
      try {
        const summary = await summarizeConversation(messages);
        if (summary && summary.trim()) {
          await finishCall(profileId, conversationId, summary);
        }
      } catch (err) {
        console.warn(`[${callSid}] Failed to persist call summary to memory:`, err);
      }
    }
  });
};
module.exports = { handle };

const axios = require('axios');
const MEMORY_BASE = 'https://memory.twilio.com';
const CONVERSATIONS_BASE = 'https://conversations.twilio.com';
const requireEnv = (name) => {
  const value = process.env[name];
  if (!value) throw new Error(`${name} not set.`);
  return value;
};
const apiKey = requireEnv('TWILIO_API_KEY');
const apiSecret = requireEnv('TWILIO_API_SECRET');
const storeId = requireEnv('TWILIO_MEMORY_STORE_ID');
const configurationId = requireEnv('TWILIO_CONFIGURATION_ID');
const twilioNumber = requireEnv('TWILIO_PHONE_NUMBER');
const http = axios.create({
  auth: { username: apiKey, password: apiSecret },
  validateStatus: () => true,
});
const startCall = async (callerPhone) => {
  if (!callerPhone) return null;
  const conversationId = await createConversation(callerPhone);
  if (!conversationId) return null;
  const profileId = await ensureProfile(callerPhone);
  if (!profileId) return { conversationId, profileId: '', memoryContext: '' };
  const memoryContext = await recall(profileId);
  return { conversationId, profileId, memoryContext };
};
const finishCall = async (profileId, conversationId, summary) => {
  if (!profileId || !conversationId || !summary) return;
  const body = {
    summaries: [
      {
        conversationId,
        content: summary.length > 4096 ? summary.slice(0, 4096) : summary,
        occurredAt: new Date().toISOString(),
        source: 'voice-agent',
      },
    ],
  };
  const resp = await http.post(
    `${MEMORY_BASE}/v1/Stores/${storeId}/Profiles/${profileId}/ConversationSummaries`,
    body
  );
  if (resp.status < 200 || resp.status >= 300) {
    console.warn(`ConversationSummaries write failed (${resp.status}):`, resp.data);
    return;
  }
  console.log(
    `Saved conversation summary for profile ${profileId} (conv ${conversationId}, ${summary.length} chars)`
  );
};
const createConversation = async (callerPhone) => {
  const payload = {
    configurationId,
    name: `Voice call ${new Date().toISOString()}`,
    participants: [
      {
        name: 'Caller',
        type: 'CUSTOMER',
        addresses: [{ channel: 'VOICE', address: callerPhone }],
      },
      {
        name: 'Owlbert Agent',
        type: 'AI_AGENT',
        addresses: [{ channel: 'VOICE', address: twilioNumber }],
      },
    ],
  };
  const resp = await http.post(`${CONVERSATIONS_BASE}/v2/Conversations`, payload);
  if (resp.status === 409) {
    const existing = extractConversationIdFromConflict(JSON.stringify(resp.data));
    if (existing) {
      console.log(`Reusing existing conversation ${existing} (previous call still open)`);
      return existing;
    }
    console.warn('Create Conversation 409 but could not extract existing id:', resp.data);
    return null;
  }
  if (resp.status < 200 || resp.status >= 300) {
    console.warn(`Create Conversation failed (${resp.status}):`, resp.data);
    return null;
  }
  return resp.data?.id ?? resp.data?.conversationId ?? null;
};
const extractConversationIdFromConflict = (body) => {
  const match = body.match(/conv_conversation_[0-9a-z]+/);
  return match ? match[0] : null;
};
const ensureProfile = async (callerPhone) => {
  const existing = await lookupProfileId(callerPhone);
  if (existing) return existing;
  const resp = await http.post(`${MEMORY_BASE}/v1/Stores/${storeId}/Profiles`, {
    traits: {
      Contact: { phone: callerPhone },
    },
  });
  if (resp.status < 200 || resp.status >= 300) {
    console.warn(`Create Profile failed (${resp.status}):`, resp.data);
    return null;
  }
  return resp.data?.id ?? null;
};
const lookupProfileId = async (phone) => {
  const resp = await http.post(
    `${MEMORY_BASE}/v1/Stores/${storeId}/Profiles/Lookup`,
    { idType: 'phone', value: phone }
  );
  if (resp.status === 404) return null;
  if (resp.status < 200 || resp.status >= 300) {
    console.warn(`Profile lookup failed (${resp.status}):`, resp.data);
    return null;
  }
  console.log('Profile lookup raw response:', resp.data);
  const rootId = extractProfileId(resp.data);
  if (rootId) return rootId;
  if (Array.isArray(resp.data?.profiles)) {
    for (const el of resp.data.profiles) {
      const found = extractProfileId(el);
      if (found) return found;
    }
  }
  return null;
};
const extractProfileId = (el) => {
  if (typeof el === 'string') return el.trim() || null;
  if (!el || typeof el !== 'object') return null;
  for (const name of ['profileId', 'id', 'profile_id']) {
    if (typeof el[name] === 'string' && el[name].trim()) return el[name];
  }
  return null;
};
const recall = async (profileId) => {
  const resp = await http.post(
    `${MEMORY_BASE}/v1/Stores/${storeId}/Profiles/${profileId}/Recall`,
    {
      observationsLimit: 20,
      summariesLimit: 5,
      communicationsLimit: 0,
    }
  );
  if (resp.status < 200 || resp.status >= 300) {
    console.warn(`Recall failed (${resp.status}):`, resp.data);
    return '';
  }
  console.log('Recall raw response:', resp.data);
  return formatRecall(resp.data);
};
const formatRecall = (data) => {
  const lines = [];
  if (Array.isArray(data?.summaries)) {
    for (const s of data.summaries) {
      const text = extractText(s, 'content', 'text', 'summary', 'value');
      if (text) lines.push(`Summary: ${text}`);
    }
  }
  if (Array.isArray(data?.observations)) {
    for (const o of data.observations) {
      const text = extractText(o, 'text', 'content', 'observation', 'value');
      if (text) lines.push(`- ${text}`);
    }
  }
  return lines.join('\n');
};
const extractText = (el, ...candidates) => {
  if (typeof el === 'string') return el;
  if (!el || typeof el !== 'object') return '';
  for (const name of candidates) {
    if (typeof el[name] === 'string') return el[name];
  }
  return '';
};
module.exports = { startCall, finishCall };

require('dotenv').config();
const express = require('express');
const expressWs = require('express-ws');
const { handle } = require('./conversationRelayHandler');
const app = express();
expressWs(app);
app.post('/voice', (req, res) => {
  const host = req.headers.host;
  const twiml = `<?xml version="1.0" encoding="UTF-8"?>
<Response>
    <Connect>
        <ConversationRelay url="wss://${host}/ws" welcomeGreeting="Thanks for calling Owlbert's Auto Repair. How can I help you today?" />
    </Connect>
</Response>`;
  res.type('application/xml').send(twiml);
});
app.ws('/ws', (ws) => {
  handle(ws);
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
  console.log(`Server listening on port ${PORT}`);
});

node index.js

ngrok http 3000