WIP push-to-talk Letta chat frontend
1use std::{io::Cursor, sync::Arc};
2
3use base64::prelude::{Engine as _, BASE64_STANDARD};
4use tauri::async_runtime::spawn;
5use tokio::sync::{
6 mpsc::{unbounded_channel, UnboundedReceiver},
7 Mutex, RwLock,
8};
9
10use crate::{
11 cartesia::tts::{TtsContext, TtsInputMessage, TtsManager, TtsMessage},
12 conversation::types::{Turn, TurnMessage},
13 letta::{
14 types::{LettaCompletionMessage, LettaMessageContent},
15 LettaManager,
16 },
17};
18
19mod types;
20
21pub struct ConversationManager {
22 turn: Option<Arc<RwLock<Turn>>>,
23 current_msg_index: RwLock<usize>,
24 letta_manager: Arc<LettaManager>,
25 tts_manager: Arc<TtsManager>,
26}
27impl ConversationManager {
28 pub fn new(letta_manager: Arc<LettaManager>, tts_manager: Arc<TtsManager>) -> Self {
29 Self {
30 turn: None,
31 current_msg_index: RwLock::new(0),
32 letta_manager,
33 tts_manager,
34 }
35 }
36
37 pub fn is_idle(&self) -> bool {
38 self.turn.is_none()
39 }
40
41 /// Start a new conversation turn
42 pub async fn start_turn(&mut self, prompt: String) {
43 if !self.is_idle() {
44 return;
45 };
46
47 let turn = Arc::new(RwLock::new(Turn::new()));
48
49 {
50 let mut idx = self.current_msg_index.write().await;
51
52 *idx = 0;
53 self.turn = Some(turn.clone());
54 }
55
56 spawn(handle_letta_messages(
57 prompt,
58 self.letta_manager.clone(),
59 self.tts_manager.clone(),
60 turn.clone(),
61 ));
62 }
63}
64
65async fn handle_letta_messages(
66 prompt: String,
67 letta: Arc<LettaManager>,
68 tts: Arc<TtsManager>,
69 turn: Arc<RwLock<Turn>>,
70) {
71 match letta.start_completion(prompt).await {
72 Ok(mut iter) => {
73 while let Some(msg) = iter.recv().await {
74 let turn = turn.read().await;
75
76 match turn.latest() {
77 Some(latest) => match latest {
78 TurnMessage::TextMessage {
79 id,
80 reader: _,
81 writer,
82 } if id == msg.id => {
83 // Add current message to chunks
84 }
85 TurnMessage::AudioMessage {
86 id,
87 reader: _,
88 writer,
89 context,
90 cursor: _,
91 timestamps: _,
92 } => {
93 // Add current message to chunks
94 }
95 },
96 None => {
97 // Create a new message
98 // Append message to chunks
99 }
100 }
101 }
102 }
103 Err(err) => eprintln!("failed to start completion: {}", err),
104 }
105}
106
107async fn create_turn_message_from_letta(
108 src: LettaCompletionMessage,
109 tts: Arc<TtsManager>,
110) -> Option<TurnMessage> {
111 let (writer, reader) = unbounded_channel::<LettaCompletionMessage>();
112
113 writer.send(src.clone());
114
115 match src {
116 LettaCompletionMessage::ApprovalRequestMessage { id, .. }
117 | LettaCompletionMessage::ApprovalResponseMessage { id, .. }
118 | LettaCompletionMessage::HiddenReasoningMessage { id, .. }
119 | LettaCompletionMessage::SystemMessage { id, .. }
120 | LettaCompletionMessage::ToolCallMessage { id, .. }
121 | LettaCompletionMessage::ToolReturnMessage { id, .. } => {
122 Some(TurnMessage::TextMessage { id, reader, writer })
123 }
124 LettaCompletionMessage::AssistantMessage {
125 id,
126 content: blocks,
127 ..
128 } => {
129 let cursor = Cursor::new(Vec::new());
130 let mut content = "".to_owned();
131
132 for b in blocks {
133 match b {
134 LettaMessageContent::Text { text } => content.push_str(&text),
135 _ => (),
136 }
137 }
138
139 let context = tts
140 .new_context(id.clone())
141 .await
142 .expect("failed to create new TTS context");
143
144 context.send(content, false).await;
145
146 // Spawn task for handling audio generation
147 spawn((async |reader: Arc<
148 Mutex<UnboundedReceiver<TtsMessage>>,
149 >| {
150 // Listen to context and append to cursor
151 while let Some(msg) = reader.lock().await.recv().await {
152 match msg {
153 TtsMessage::Chunk { data, .. } => {
154 // Decode chunk and write to cursor
155 let out = BASE64_STANDARD.decode_vec(data, cursor.get_mut());
156 }
157 TtsMessage::Timestamps {
158 word_timestamps, ..
159 } => {
160 // Append timestamps
161 }
162 _ => (),
163 }
164 }
165 })(context.reader.clone()));
166
167 Some(TurnMessage::AudioMessage {
168 id: id.clone(),
169 reader,
170 writer,
171 context,
172 cursor,
173 timestamps: Vec::new(),
174 })
175 }
176 _ => None,
177 }
178}