1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
import os
import json
import base64
import asyncio
import websockets
from fastapi import FastAPI, WebSocket, Request
from fastapi.responses import HTMLResponse
from fastapi.websockets import WebSocketDisconnect
from twilio.twiml.voice_response import VoiceResponse, Connect, Say, Stream
from dotenv import load_dotenv
load_dotenv()
# Configuration
OPENAI_API_KEY = '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' # requires OpenAI Realtime API Access
PORT = int(os.getenv('PORT', 6000))
import ssl
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
SYSTEM_MESSAGE = (
"""You are an AI assistant representing westgate. Please answer all questions based on the given information. Information is: 1 bedrrom $1000 2bedroom $1200 Apartments in Champaign IL - Westgate Apartments\nSkip to content\nWestgate Apartments - Main Page\n217-359-5330\nAvailability\nAmenities\nPhoto Gallery\nResources\nPet Policy\nVideo Tours\nMeet the Staff\nDownloads\nLocal Business Map\nNewsletter\nWestgate FAQs\nReviews\nApplicant Login\nResident Login\nWestgate Apartments\nPriority Living\nLove Living Here\nMove In Date\nApply Now\nor\nContact Us for More Information\nApply Now\nor\nContact Us for More Information\nWelcome to one of the finest, most affordable, residential communities in the Champaign-Urbana area. Westgate Apartments in Champaign, IL is ideally located for easy access to the interstates, retail shopping, restaurants, entertainment as well as Parkland College and the University of Illinois.\nNOTICE TO ALL APPLICANTS AND RESIDENTS:\nWe require EVERYONE to complete a third-party\nPet Screening Profile\neven if you do not own a pet or animal. This quick and easy process ensures that everyone understands and acknowledges our pet policies.\nWe are a Cat and Small Dog (maximum weight 75 lbs full grown) pet friendly community.\nAvailability\nWestgate Apartments offers one bedroom, and two bedroom units.\nAmenities\nYou'll love Westgate's outdoor spaces, sparkling swimming pool, gym, and on-site laundry.\nGallery\nDon’t believe us yet? Take a look at all we have to offer here at Westgate Apartments!\nOffice Hours\nMonday-Friday:\n9:00 am to 5:30 pm\nSaturday:\nClosed\nSunday:\nClosed\nPhone:\n217-359-5330\nLeave Us a Review on Google!\n1600 W Bradley Ave.\nChampaign, IL 61821\n© 2024\nWestgate Apartments\nPop Up WordPress Plugin\nTranslate ». If a question cannot be answered with the provided details, respond with: \"I'm sorry, but I don't have that information.\" For greetings or casual conversation, feel free to respond with a friendly message, but always steer back to the information about westgate. Keep it short and simple and relevant to Prop Tech."""
)
VOICE = 'alloy'
LOG_EVENT_TYPES = [
'response.content.done'
]
app = FastAPI()
if not OPENAI_API_KEY:
raise ValueError('Missing the OpenAI API key. Please set it in the .env file.')
@app.get("/", response_class=HTMLResponse)
async def index_page():
return {"message": "Twilio Media Stream Server is running!"}
@app.api_route("/incoming-call", methods=["GET", "POST"])
async def handle_incoming_call(request: Request):
"""Handle incoming call and return TwiML response to connect to Media Stream."""
response = VoiceResponse()
# <Say> punctuation to improve text-to-speech flow
response.say("Please wait while we connect your call")
response.pause(length=1)
response.say("O.K. you can start talking!")
host = request.url.hostname
connect = Connect()
connect.stream(url=f'wss://{host}/media-stream')
response.append(connect)
return HTMLResponse(content=str(response), media_type="application/xml")
@app.websocket("/media-stream")
async def handle_media_stream(websocket: WebSocket):
"""Handle WebSocket connections between Twilio and OpenAI."""
print("Client connected")
await websocket.accept()
async with websockets.connect(
'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01',
ssl=ssl_context,
extra_headers={
"Authorization": f"Bearer {OPENAI_API_KEY}",
"OpenAI-Beta": "realtime=v1"
}
) as openai_ws:
await send_session_update(openai_ws)
stream_sid = None
async def receive_from_twilio():
"""Receive audio data from Twilio and send it to the OpenAI Realtime API."""
nonlocal stream_sid
try:
async for message in websocket.iter_text():
data = json.loads(message)
if data['event'] == 'media' and openai_ws.open:
audio_append = {
"type": "input_audio_buffer.append",
"audio": data['media']['payload']
}
await openai_ws.send(json.dumps(audio_append))
elif data['event'] == 'start':
stream_sid = data['start']['streamSid']
print(f"Incoming stream has started {stream_sid}")
except WebSocketDisconnect:
print("Client disconnected.")
if openai_ws.open:
await openai_ws.close()
async def send_to_twilio():
"""Receive events from the OpenAI Realtime API, send audio back to Twilio."""
nonlocal stream_sid
try:
async for openai_message in openai_ws:
response = json.loads(openai_message)
if response['type'] in LOG_EVENT_TYPES:
print(f"Received event: {response['type']}", response)
if response['type'] == 'session.updated':
print("Session updated successfully:", response)
if response['type'] == 'response.audio.delta' and response.get('delta'):
# Audio from OpenAI
try:
audio_payload = base64.b64encode(base64.b64decode(response['delta'])).decode('utf-8')
audio_delta = {
"event": "media",
"streamSid": stream_sid,
"media": {
"payload": audio_payload
}
}
await websocket.send_json(audio_delta)
except Exception as e:
print(f"Error processing audio data: {e}")
except Exception as e:
print(f"Error in send_to_twilio: {e}")
await asyncio.gather(receive_from_twilio(), send_to_twilio())
async def send_session_update(openai_ws):
"""Send session update to OpenAI WebSocket."""
session_update = {
"type": "session.update",
"session": {
"turn_detection": {"type": "server_vad"},
"input_audio_format": "g711_ulaw",
"output_audio_format": "g711_ulaw",
"voice": VOICE,
"instructions": SYSTEM_MESSAGE,
"modalities": ["text", "audio"],
"temperature": 0.8,
}
}
print('Sending session update:', json.dumps(session_update))
await openai_ws.send(json.dumps(session_update))
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=PORT)