//! Reconnecting WebSocket client for the SDK wire protocol. use std::collections::{HashMap, VecDeque}; use std::fmt; use std::future::pending; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use futures_util::{SinkExt, StreamExt}; use serde_json::Value; use tokio::net::TcpStream; use tokio::sync::{mpsc, oneshot, watch, Mutex as AsyncMutex, RwLock}; use tokio::task::JoinHandle; use tokio::time::{sleep, timeout, timeout_at, Instant}; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; use url::Url; use uuid::Uuid; use crate::error::{SdkError, SdkErrorCode, SdkResult, SdkSentRecord}; use crate::frame::{SdkFrame, MAX_FRAME_BYTES, PROTOCOL_VERSION}; const MAX_SENT_RECORDS: usize = 256; const CLOSE_GRACE: Duration = Duration::from_secs(5); const CANCEL_ACK_GRACE: Duration = Duration::from_millis(100); type ClientSocket = WebSocketStream>; /// Runtime options for an SDK WebSocket client. #[derive(Debug, Clone)] pub struct SdkClientConfig { /// WebSocket endpoint (`ws://` or `wss://`). pub url: String, /// Broker authentication token, sent as the `token` query parameter. pub token: String, /// Maximum UTF-8 frame size accepted in either direction. pub max_frame_bytes: usize, /// Default timeout for an individual request. pub request_timeout: Duration, /// Timeout for each socket-open and hello-handshake attempt. pub connect_timeout: Duration, /// Maximum time spent across a connection or reconnect cycle. pub reconnect_deadline: Duration, /// Number of retries after the initial connection attempt. pub reconnect_attempts: u32, /// Initial reconnect delay; subsequent delays double up to `reconnect_max_backoff`. pub reconnect_initial_backoff: Duration, /// Maximum reconnect delay. pub reconnect_max_backoff: Duration, /// Capabilities sent in the protocol-v3 client hello. pub capabilities: Vec, } impl Default for SdkClientConfig { fn default() -> Self { Self { url: "ws://127.0.0.1:8080".to_string(), token: String::new(), max_frame_bytes: MAX_FRAME_BYTES, request_timeout: Duration::from_secs(10), connect_timeout: Duration::from_secs(10), reconnect_deadline: Duration::from_secs(30), reconnect_attempts: 3, reconnect_initial_backoff: Duration::from_millis(25), reconnect_max_backoff: Duration::from_secs(2), capabilities: Vec::new(), } } } impl SdkClientConfig { fn validate(&self) -> SdkResult<()> { let endpoint = Url::parse(&self.url) .map_err(|error| SdkError::invalid_input(format!("invalid WebSocket URL: {error}")))?; if !matches!(endpoint.scheme(), "ws" | "wss") { return Err(SdkError::invalid_input( "SDK endpoint must use the ws:// or wss:// scheme", )); } if self.max_frame_bytes == 0 { return Err(SdkError::invalid_input( "maximum frame size must be greater than zero", )); } if self.connect_timeout.is_zero() || self.reconnect_deadline.is_zero() { return Err(SdkError::invalid_input( "connection timeout and reconnect deadline must be greater than zero", )); } if self.reconnect_max_backoff < self.reconnect_initial_backoff { return Err(SdkError::invalid_input( "maximum reconnect backoff must be at least the initial backoff", )); } Ok(()) } } struct SentRecordStore { records: HashMap, order: VecDeque, } impl SentRecordStore { fn new() -> Self { Self { records: HashMap::new(), order: VecDeque::new(), } } fn insert(&mut self, record: SdkSentRecord) { if self.records.contains_key(&record.id) { self.order.retain(|id| *id != record.id); } self.order.push_back(record.id); self.records.insert(record.id, record); while self.records.len() > MAX_SENT_RECORDS { if let Some(oldest) = self.order.pop_front() { self.records.remove(&oldest); } } } fn remove(&mut self, id: Uuid) { self.records.remove(&id); self.order.retain(|record_id| *record_id != id); } fn get(&self, id: Uuid) -> Option { self.records.get(&id).cloned() } fn all(&self) -> Vec { self.order .iter() .filter_map(|id| self.records.get(id).cloned()) .collect() } } struct RequestMessage { id: Uuid, operation: Option, serialized: String, deadline: Instant, response_tx: oneshot::Sender>, } enum ClientCommand { Request(RequestMessage), Cancel { id: Uuid, ack: oneshot::Sender>, }, } struct PendingRequest { response_tx: oneshot::Sender>, sent_record: SdkSentRecord, } struct ConnectedSocket { socket: ClientSocket, connection_id: Option, } /// SDK client with UUID correlation, handshake, timeout handling, and reconnects. pub struct SdkClient { config: SdkClientConfig, command_tx: mpsc::UnboundedSender, shutdown_tx: watch::Sender, task_handle: AsyncMutex>>, sent_records: Arc>, connection_id: Arc>>, closed: AtomicBool, } impl fmt::Debug for SdkClient { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("SdkClient") .field("config", &self.config) .field("closed", &self.closed.load(Ordering::Acquire)) .finish_non_exhaustive() } } impl SdkClient { /// Connect, complete the protocol-v3 hello handshake, and start reconnect supervision. pub async fn connect(config: SdkClientConfig) -> SdkResult { config.validate()?; let connected = connect_with_retry(&config).await?; let connection_id = Arc::new(RwLock::new(connected.connection_id.clone())); let sent_records = Arc::new(Mutex::new(SentRecordStore::new())); let (command_tx, command_rx) = mpsc::unbounded_channel(); let (shutdown_tx, shutdown_rx) = watch::channel(false); let task_handle = tokio::spawn(run_client( connected, config.clone(), command_rx, shutdown_rx, sent_records.clone(), connection_id.clone(), )); Ok(Self { config, command_tx, shutdown_tx, task_handle: AsyncMutex::new(Some(task_handle)), sent_records, connection_id, closed: AtomicBool::new(false), }) } /// Send a broker operation with the configured default request timeout. pub async fn request(&self, operation: impl Into, input: Value) -> SdkResult { self.request_with_timeout(operation, input, self.config.request_timeout) .await } /// Send a broker operation with a per-request timeout override. pub async fn request_with_timeout( &self, operation: impl Into, input: Value, request_timeout: Duration, ) -> SdkResult { let operation = operation.into(); let frame = SdkFrame::broker_request(Uuid::new_v4(), operation, input); self.request_frame_with_timeout(frame, request_timeout) .await } /// Send a typed frame, replacing its `id` with a fresh UUID for correlation. pub async fn request_frame(&self, frame: SdkFrame) -> SdkResult { self.request_frame_with_timeout(frame, self.config.request_timeout) .await } /// Send a typed frame with a per-request timeout override. pub async fn request_frame_with_timeout( &self, mut frame: SdkFrame, request_timeout: Duration, ) -> SdkResult { if self.closed.load(Ordering::Acquire) { return Err(SdkError::new( SdkErrorCode::ConnectionClosed, "SDK client is closed", )); } let id = Uuid::new_v4(); frame.set_id(id); let operation = frame.operation(); let serialized = frame.to_json()?; if serialized.len() > self.config.max_frame_bytes { return Err(SdkError::payload_too_large(format!( "frame size {} exceeds maximum {}", serialized.len(), self.config.max_frame_bytes ))); } let deadline = Instant::now() + request_timeout; let (response_tx, response_rx) = oneshot::channel(); self.command_tx .send(ClientCommand::Request(RequestMessage { id, operation, serialized, deadline, response_tx, })) .map_err(|_| { SdkError::new( SdkErrorCode::ConnectionClosed, "SDK connection task is no longer running", ) })?; match timeout_at(deadline, response_rx).await { Ok(Ok(response)) => response, Ok(Err(_)) => Err(SdkError::connection_error( "SDK connection task dropped the response channel", )), Err(_) => { let (ack_tx, ack_rx) = oneshot::channel(); let _ = self .command_tx .send(ClientCommand::Cancel { id, ack: ack_tx }); let acknowledged = timeout(CANCEL_ACK_GRACE, ack_rx) .await .ok() .and_then(Result::ok) .flatten(); let sent_record = acknowledged.or_else(|| self.get_sent_record(id)); match sent_record { Some(record) => Err(SdkError::uncertain_after_send(record)), None => Err(SdkError::timeout(format!( "SDK request timed out after {}ms before a response was received", request_timeout.as_millis() ))), } } } } /// Current broker connection identifier from the most recent hello frame. pub async fn connection_id(&self) -> Option { self.connection_id.read().await.clone() } /// Look up a retained request identity after an uncertain send. pub fn get_sent_record(&self, id: Uuid) -> Option { self.sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(id) } /// List outstanding sent records that require caller-side reconciliation. pub fn uncertain_sends(&self) -> Vec { self.sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .all() } /// Forget a request record after the caller has reconciled its outcome. pub fn forget_sent_record(&self, id: Uuid) -> bool { let mut records = self .sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let found = records.records.contains_key(&id); records.remove(id); found } /// Close the socket and stop reconnect attempts. Safe to call more than once. pub async fn close(&self) -> SdkResult<()> { if !self.closed.swap(true, Ordering::AcqRel) { let _ = self.shutdown_tx.send(true); } if let Some(mut task) = self.task_handle.lock().await.take() { match timeout(CLOSE_GRACE, &mut task).await { Ok(Ok(())) => {} Ok(Err(error)) => { return Err(SdkError::connection_error(format!( "SDK connection task failed while closing: {error}" ))) } Err(_) => { task.abort(); return Err(SdkError::timeout( "SDK connection task did not stop in time", )); } } } Ok(()) } } impl Drop for SdkClient { fn drop(&mut self) { if !self.closed.swap(true, Ordering::AcqRel) { let _ = self.shutdown_tx.send(true); } } } fn authenticated_url(config: &SdkClientConfig) -> SdkResult { let mut url = Url::parse(&config.url) .map_err(|error| SdkError::invalid_input(format!("invalid WebSocket URL: {error}")))?; if !matches!(url.scheme(), "ws" | "wss") { return Err(SdkError::invalid_input( "SDK endpoint must use the ws:// or wss:// scheme", )); } let mut query: Vec<(String, String)> = url .query_pairs() .filter(|(key, _)| key != "token") .map(|(key, value)| (key.into_owned(), value.into_owned())) .collect(); query.push(("token".to_string(), config.token.clone())); url.set_query(None); url.query_pairs_mut().extend_pairs(query); Ok(url) } async fn connect_with_retry(config: &SdkClientConfig) -> SdkResult { let deadline = Instant::now() + config.reconnect_deadline; let mut last_error = None; for attempt in 0..=config.reconnect_attempts { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { return Err(SdkError::timeout("SDK reconnect deadline elapsed")); } let attempt_timeout = remaining.min(config.connect_timeout); match timeout(attempt_timeout, connect_once(config)).await { Ok(Ok(connected)) => return Ok(connected), Ok(Err(error)) => last_error = Some(error), Err(_) => { last_error = Some(SdkError::timeout(format!( "SDK connection attempt timed out after {}ms", attempt_timeout.as_millis() ))) } } if attempt == config.reconnect_attempts { break; } let delay = reconnect_delay(config, attempt); let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { return Err(SdkError::timeout("SDK reconnect deadline elapsed")); } sleep(delay.min(remaining)).await; } let cause = last_error .map(|error| error.to_string()) .unwrap_or_else(|| "no connection attempt was made".to_string()); Err(SdkError::new( SdkErrorCode::ReconnectExhausted, format!("SDK WebSocket reconnect attempts exhausted: {cause}"), )) } fn reconnect_delay(config: &SdkClientConfig, attempt: u32) -> Duration { let multiplier = 1_u32.checked_shl(attempt).unwrap_or(u32::MAX); config .reconnect_initial_backoff .saturating_mul(multiplier) .min(config.reconnect_max_backoff) } async fn connect_once(config: &SdkClientConfig) -> SdkResult { let url = authenticated_url(config)?; let (mut socket, _) = connect_async(url.as_str()).await.map_err(|error| { SdkError::connection_error(format!("WebSocket connection failed: {error}")) })?; let hello = SdkFrame::client_hello(&config.capabilities).to_json()?; socket.send(Message::Text(hello)).await.map_err(|error| { SdkError::connection_error(format!("client hello send failed: {error}")) })?; let handshake_deadline = Instant::now() + config.connect_timeout; loop { let message = timeout_at(handshake_deadline, socket.next()) .await .map_err(|_| SdkError::timeout("SDK server hello handshake timed out"))? .ok_or_else(|| SdkError::connection_error("WebSocket closed before server hello"))? .map_err(|error| { SdkError::connection_error(format!("WebSocket failed before server hello: {error}")) })?; match message { Message::Text(text) => { if text.len() > config.max_frame_bytes { return Err(SdkError::payload_too_large(format!( "server hello frame size {} exceeds maximum {}", text.len(), config.max_frame_bytes ))); } let frame = SdkFrame::from_json(&text)?; if frame.is_server_hello() { if let Some(version) = frame.data.get("protocolVersion").and_then(Value::as_u64) { if version != u64::from(PROTOCOL_VERSION) { return Err(SdkError::new( SdkErrorCode::ProtocolError, format!( "server protocol version {version} does not match {PROTOCOL_VERSION}" ), )); } } let connection_id = frame .connection_id() .filter(|value| !value.is_empty()) .map(str::to_owned); return Ok(ConnectedSocket { socket, connection_id, }); } } Message::Ping(payload) => { socket.send(Message::Pong(payload)).await.map_err(|error| { SdkError::connection_error(format!("WebSocket pong failed: {error}")) })? } Message::Close(_) => { return Err(SdkError::connection_error( "WebSocket closed before server hello", )); } Message::Binary(_) | Message::Pong(_) | Message::Frame(_) => {} } } } async fn await_reconnect( reconnect_task: &mut Option>>, ) -> SdkResult { match reconnect_task { Some(task) => task.await.map_err(|error| { SdkError::connection_error(format!("SDK reconnect task failed: {error}")) })?, None => pending().await, } } async fn run_client( connected: ConnectedSocket, config: SdkClientConfig, mut command_rx: mpsc::UnboundedReceiver, mut shutdown_rx: watch::Receiver, sent_records: Arc>, connection_id: Arc>>, ) { let mut socket = Some(connected.socket); let mut queued = VecDeque::new(); let mut pending_requests = HashMap::new(); let mut reconnect_task = None; loop { let incoming = async { match socket.as_mut() { Some(connected) => connected.next().await, None => pending().await, } }; tokio::select! { changed = shutdown_rx.changed() => { if changed.is_err() || *shutdown_rx.borrow() { break; } } message = incoming => { match message { Some(Ok(Message::Text(text))) => { if text.len() > config.max_frame_bytes { tracing::warn!(size = text.len(), "SDK server frame exceeds configured limit"); fail_pending_uncertain(&mut pending_requests); start_reconnect(&config, &mut socket, &mut reconnect_task); } else { handle_server_text(&text, &mut pending_requests, &sent_records, &connection_id).await; } } Some(Ok(Message::Ping(payload))) => { let send_result = match socket.as_mut() { Some(connected) => connected.send(Message::Pong(payload)).await, None => continue, }; if send_result.is_err() { fail_pending_uncertain(&mut pending_requests); start_reconnect(&config, &mut socket, &mut reconnect_task); } } Some(Ok(Message::Close(_))) | Some(Err(_)) | None => { fail_pending_uncertain(&mut pending_requests); start_reconnect(&config, &mut socket, &mut reconnect_task); } Some(Ok(Message::Binary(_))) | Some(Ok(Message::Pong(_))) | Some(Ok(Message::Frame(_))) => {} } } command = command_rx.recv() => { let Some(command) = command else { break }; match command { ClientCommand::Request(request) => { if socket.is_none() { queued.push_back(request); } else if !send_request( request, socket.as_mut().expect("socket checked above"), &mut pending_requests, &sent_records, ).await { fail_pending_uncertain(&mut pending_requests); start_reconnect(&config, &mut socket, &mut reconnect_task); } } ClientCommand::Cancel { id, ack } => { let record = cancel_request( id, &mut queued, &mut pending_requests, &sent_records, ); let _ = ack.send(record); } } } result = await_reconnect(&mut reconnect_task) => { reconnect_task = None; match result { Ok(connected) => { *connection_id.write().await = connected.connection_id; socket = Some(connected.socket); while let Some(request) = queued.pop_front() { if !send_request( request, socket.as_mut().expect("reconnected socket is present"), &mut pending_requests, &sent_records, ).await { fail_pending_uncertain(&mut pending_requests); start_reconnect(&config, &mut socket, &mut reconnect_task); break; } } } Err(error) => { for request in queued.drain(..) { let _ = request.response_tx.send(Err(error.clone())); } break; } } } } } if let Some(task) = reconnect_task { task.abort(); } fail_pending_uncertain(&mut pending_requests); for request in queued.drain(..) { let _ = request.response_tx.send(Err(SdkError::new( SdkErrorCode::ConnectionClosed, "SDK client closed before request was sent", ))); } if let Some(mut connected) = socket { let _ = timeout(CLOSE_GRACE, connected.close(None)).await; } } fn start_reconnect( config: &SdkClientConfig, socket: &mut Option, reconnect_task: &mut Option>>, ) { socket.take(); if reconnect_task.is_none() { let config = config.clone(); *reconnect_task = Some(tokio::spawn( async move { connect_with_retry(&config).await }, )); } } async fn send_request( request: RequestMessage, socket: &mut ClientSocket, pending_requests: &mut HashMap, sent_records: &Arc>, ) -> bool { if Instant::now() >= request.deadline { let _ = request.response_tx.send(Err(SdkError::timeout( "SDK request deadline elapsed before send", ))); return true; } let sent_record = SdkSentRecord { id: request.id, operation: request.operation, sent_at: SystemTime::now(), }; sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(sent_record.clone()); pending_requests.insert( request.id, PendingRequest { response_tx: request.response_tx, sent_record: sent_record.clone(), }, ); let send_result = timeout_at( request.deadline, socket.send(Message::Text(request.serialized)), ) .await; match send_result { Ok(Ok(())) => true, Ok(Err(error)) => { if let Some(pending) = pending_requests.remove(&request.id) { let _ = pending .response_tx .send(Err(SdkError::uncertain_after_send(pending.sent_record))); } tracing::warn!(%error, "SDK request send failed after dispatch began"); false } Err(_) => { if let Some(pending) = pending_requests.remove(&request.id) { let _ = pending .response_tx .send(Err(SdkError::uncertain_after_send(pending.sent_record))); } false } } } fn cancel_request( id: Uuid, queued: &mut VecDeque, pending_requests: &mut HashMap, sent_records: &Arc>, ) -> Option { if let Some(position) = queued.iter().position(|request| request.id == id) { queued.remove(position); return None; } if let Some(pending) = pending_requests.remove(&id) { return Some(pending.sent_record); } sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(id) } async fn handle_server_text( text: &str, pending_requests: &mut HashMap, sent_records: &Arc>, connection_id: &Arc>>, ) { let frame = match SdkFrame::from_json(text) { Ok(frame) => frame, Err(error) => { tracing::warn!(%error, "SDK server sent malformed JSON frame"); return; } }; if frame.is_server_hello() { if let Some(id) = frame.connection_id().filter(|id| !id.is_empty()) { *connection_id.write().await = Some(id.to_string()); } return; } let id = match frame.response_id() { Ok(Some(id)) => id, Ok(None) => return, Err(error) => { tracing::warn!(%error, "SDK server sent an invalid correlation ID"); return; } }; if let Some(pending) = pending_requests.remove(&id) { sent_records .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .remove(id); let result = if frame.data.get("ok") == Some(&Value::Bool(false)) || frame.data.get("status").and_then(Value::as_str) == Some("error") { Err(error_from_response(&frame)) } else { Ok(frame) }; let _ = pending.response_tx.send(result); } } fn error_from_response(frame: &SdkFrame) -> SdkError { match frame.error() { Some(Ok((code, message))) => SdkError::from_remote(&code, message), Some(Err(error)) => error, None => SdkError::new( SdkErrorCode::Unavailable, "SDK request failed without error details", ), } } fn fail_pending_uncertain(pending_requests: &mut HashMap) { for (_, pending) in pending_requests.drain() { let _ = pending .response_tx .send(Err(SdkError::uncertain_after_send(pending.sent_record))); } } #[cfg(test)] mod tests { use super::*; use serde_json::json; use tokio::net::TcpListener; use tokio_tungstenite::accept_async; use tokio_tungstenite::WebSocketStream as TestWebSocketStream; type ServerSocket = TestWebSocketStream; async fn test_server() -> (TcpListener, String) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); (listener, format!("ws://{address}/sdk?keep=1")) } fn test_config(url: String) -> SdkClientConfig { SdkClientConfig { url, token: "token with & symbols".to_string(), connect_timeout: Duration::from_millis(250), reconnect_deadline: Duration::from_secs(2), reconnect_attempts: 1, reconnect_initial_backoff: Duration::from_millis(5), reconnect_max_backoff: Duration::from_millis(20), ..SdkClientConfig::default() } } async fn accept(listener: &TcpListener) -> ServerSocket { let (stream, _) = listener.accept().await.unwrap(); accept_async(stream).await.unwrap() } async fn read_frame(socket: &mut ServerSocket) -> SdkFrame { loop { match socket.next().await.unwrap().unwrap() { Message::Text(text) => return SdkFrame::from_json(&text).unwrap(), Message::Ping(payload) => socket.send(Message::Pong(payload)).await.unwrap(), _ => {} } } } async fn assert_no_text_frame(socket: &mut ServerSocket) { if let Ok(Some(Ok(Message::Text(_)))) = timeout(Duration::from_millis(50), socket.next()).await { panic!("client unexpectedly dispatched a request frame"); } } async fn complete_hello(socket: &mut ServerSocket, connection_id: &str) -> SdkFrame { let hello = read_frame(socket).await; assert_eq!(hello.frame_type().unwrap(), crate::frame::FrameType::Hello); socket .send(Message::Text( json!({"type":"hello", "connectionId":connection_id}).to_string(), )) .await .unwrap(); hello } async fn send_response(socket: &mut ServerSocket, id: Uuid, result: Value) { socket .send(Message::Text( json!({"type":"broker_response", "id":id, "ok":true, "result":result}).to_string(), )) .await .unwrap(); } #[test] fn config_defaults_use_ten_second_timeout_and_bounded_backoff() { let config = SdkClientConfig::default(); assert_eq!(config.request_timeout, Duration::from_secs(10)); assert_eq!(config.max_frame_bytes, MAX_FRAME_BYTES); let delays: Vec<_> = (0..8) .map(|attempt| reconnect_delay(&config, attempt)) .collect(); assert_eq!( delays, [ Duration::from_millis(25), Duration::from_millis(50), Duration::from_millis(100), Duration::from_millis(200), Duration::from_millis(400), Duration::from_millis(800), Duration::from_millis(1_600), Duration::from_secs(2), ] ); } #[test] fn auth_url_preserves_query_and_encodes_token_once() { let config = SdkClientConfig { url: "wss://sdk.example/path?keep=a%20b&token=stale".to_string(), token: "a token&value".to_string(), ..SdkClientConfig::default() }; let url = authenticated_url(&config).unwrap(); let pairs: Vec<_> = url .query_pairs() .map(|(key, value)| (key.into_owned(), value.into_owned())) .collect(); assert_eq!( pairs, [ ("keep".to_string(), "a b".to_string()), ("token".to_string(), "a token&value".to_string()) ] ); } #[tokio::test] async fn rejects_non_websocket_endpoint_schemes() { let config = SdkClientConfig { url: "https://sdk.example".to_string(), ..SdkClientConfig::default() }; assert_eq!( SdkClient::connect(config).await.unwrap_err().code, SdkErrorCode::InvalidInput ); } #[tokio::test] async fn handshake_advertises_capabilities_and_correlates_response_uuid() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; let hello = complete_hello(&mut socket, "conn-1").await; assert_eq!(hello.data["protocolVersion"], PROTOCOL_VERSION); assert_eq!(hello.data["capabilities"], json!(["session.observe.v1"])); let request = read_frame(&mut socket).await; let id = request.id().unwrap().unwrap(); assert_eq!(request.operation().as_deref(), Some("session.get")); send_response(&mut socket, id, json!({"session": 7})).await; }); let mut config = test_config(url); config.capabilities = vec!["session.observe.v1".to_string()]; let client = SdkClient::connect(config).await.unwrap(); assert_eq!(client.connection_id().await.as_deref(), Some("conn-1")); let response = client .request("session.get", json!({"id": "s-1"})) .await .unwrap(); assert_eq!(response.result(), Some(json!({"session": 7}))); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn requests_are_correlated_when_responses_arrive_out_of_order() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "concurrent").await; let first = read_frame(&mut socket).await; let second = read_frame(&mut socket).await; let first_id = first.id().unwrap().unwrap(); let second_id = second.id().unwrap().unwrap(); socket .send(Message::Text( json!({"type":"broker_response", "id":second_id, "ok":true, "result":"second"}) .to_string(), )) .await .unwrap(); socket .send(Message::Text( json!({"type":"broker_response", "id":first_id, "ok":true, "result":"first"}) .to_string(), )) .await .unwrap(); }); let client = SdkClient::connect(test_config(url)).await.unwrap(); let first = client.request("op.first", json!({})); let second = client.request("op.second", json!({})); let (first, second) = tokio::join!(first, second); assert_eq!(first.unwrap().result(), Some(json!("first"))); assert_eq!(second.unwrap().result(), Some(json!("second"))); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn broker_errors_are_returned_with_typed_codes() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "errors").await; let request = read_frame(&mut socket).await; let id = request.id().unwrap().unwrap(); socket .send(Message::Text( json!({"type":"broker_response", "id":id, "ok":false, "error":{"code":"not_found", "message":"missing"}}) .to_string(), )) .await .unwrap(); }); let client = SdkClient::connect(test_config(url)).await.unwrap(); let error = client .request("session.missing", json!({})) .await .unwrap_err(); assert_eq!(error.code, SdkErrorCode::NotFound); assert_eq!(error.message, "missing"); assert!(error.sent_record.is_none()); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn timeout_override_marks_sent_request_uncertain_and_retains_record() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "timeout").await; let request = read_frame(&mut socket).await; assert!(request.id().unwrap().is_some()); tokio::time::sleep(Duration::from_millis(80)).await; }); let mut config = test_config(url); config.request_timeout = Duration::from_secs(1); let client = SdkClient::connect(config).await.unwrap(); let error = client .request_with_timeout( "session.create", json!({"secret": "not retained"}), Duration::from_millis(20), ) .await .unwrap_err(); assert_eq!(error.code, SdkErrorCode::UncertainAfterSend); let record = error.sent_record.unwrap(); assert_eq!(record.operation.as_deref(), Some("session.create")); assert_eq!(client.get_sent_record(record.id), Some(record.clone())); assert_eq!(client.uncertain_sends(), vec![record.clone()]); assert!(client.forget_sent_record(record.id)); assert!(client.get_sent_record(record.id).is_none()); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn oversized_outgoing_frame_is_rejected_before_dispatch() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "size").await; assert_no_text_frame(&mut socket).await; }); let mut config = test_config(url); config.max_frame_bytes = 128; let client = SdkClient::connect(config).await.unwrap(); let error = client .request("large", json!({"payload": "x".repeat(256)})) .await .unwrap_err(); assert_eq!(error.code, SdkErrorCode::PayloadTooLarge); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn reconnects_after_close_and_never_replays_uncertain_request() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut first = accept(&listener).await; complete_hello(&mut first, "first").await; let uncertain = read_frame(&mut first).await; let uncertain_id = uncertain.id().unwrap().unwrap(); first.send(Message::Close(None)).await.unwrap(); drop(first); let mut second = accept(&listener).await; complete_hello(&mut second, "second").await; let next = read_frame(&mut second).await; let next_id = next.id().unwrap().unwrap(); assert_ne!(next_id, uncertain_id); assert_eq!(next.operation().as_deref(), Some("after.reconnect")); send_response(&mut second, next_id, json!({"connected": true})).await; }); let client = SdkClient::connect(test_config(url)).await.unwrap(); let uncertain = client.request("uncertain.mutation", json!({})); let error = uncertain.await.unwrap_err(); assert_eq!(error.code, SdkErrorCode::UncertainAfterSend); let next = client.request("after.reconnect", json!({})).await.unwrap(); assert_eq!(next.result(), Some(json!({"connected": true}))); assert_eq!(client.connection_id().await.as_deref(), Some("second")); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn retries_failed_initial_connections_with_exponential_backoff() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let (failed_stream, _) = listener.accept().await.unwrap(); drop(failed_stream); let (stream, _) = listener.accept().await.unwrap(); let mut socket = accept_async(stream).await.unwrap(); let hello = complete_hello(&mut socket, "retry-success").await; assert_eq!(hello.frame_type().unwrap(), crate::frame::FrameType::Hello); }); let mut config = test_config(url); config.reconnect_attempts = 2; let client = SdkClient::connect(config).await.unwrap(); assert_eq!( client.connection_id().await.as_deref(), Some("retry-success") ); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn connection_attempts_stop_at_the_reconnect_deadline() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let _socket = accept(&listener).await; tokio::time::sleep(Duration::from_millis(100)).await; }); let mut config = test_config(url); config.connect_timeout = Duration::from_millis(500); config.reconnect_deadline = Duration::from_millis(25); config.reconnect_attempts = 8; let error = SdkClient::connect(config).await.unwrap_err(); assert_eq!(error.code, SdkErrorCode::Timeout); assert!(error.message.contains("deadline")); server.await.unwrap(); } #[tokio::test] async fn exhausted_reconnect_budget_returns_typed_error() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("ws://{}", listener.local_addr().unwrap()); drop(listener); let mut config = test_config(url); config.reconnect_attempts = 0; let error = SdkClient::connect(config).await.unwrap_err(); assert_eq!(error.code, SdkErrorCode::ReconnectExhausted); } #[tokio::test] async fn request_timeout_before_send_does_not_create_uncertain_record() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "immediate-timeout").await; assert_no_text_frame(&mut socket).await; }); let client = SdkClient::connect(test_config(url)).await.unwrap(); let error = client .request_with_timeout("too.short", json!({}), Duration::ZERO) .await .unwrap_err(); assert_eq!(error.code, SdkErrorCode::Timeout); assert!(client.uncertain_sends().is_empty()); client.close().await.unwrap(); server.await.unwrap(); } #[tokio::test] async fn malformed_server_frames_do_not_settle_unrelated_requests() { let (listener, url) = test_server().await; let server = tokio::spawn(async move { let mut socket = accept(&listener).await; complete_hello(&mut socket, "malformed").await; let request = read_frame(&mut socket).await; let id = request.id().unwrap().unwrap(); socket .send(Message::Text("not-json".to_string())) .await .unwrap(); send_response(&mut socket, id, json!("ok-after-malformed")).await; }); let client = SdkClient::connect(test_config(url)).await.unwrap(); let response = client.request("malformed.noise", json!({})).await.unwrap(); assert_eq!(response.result(), Some(json!("ok-after-malformed"))); client.close().await.unwrap(); server.await.unwrap(); } async fn round_trip_case(case: u8) { let (listener, url) = test_server().await; let operation = format!("round.trip.case_{case:03}"); let input = json!({"case": case, "value": format!("request-{case}")}); let server_operation = operation.clone(); let server_input = input.clone(); let server = tokio::spawn(async move { let mut socket = accept(&listener).await; let hello = complete_hello(&mut socket, "matrix").await; assert_eq!( hello.data["capabilities"], json!([format!("matrix.{case}")]) ); let request = read_frame(&mut socket).await; let id = request.id().unwrap().expect("request has a UUID ID"); assert_eq!(request.operation(), Some(server_operation)); assert_eq!(request.input(), Some(server_input)); send_response(&mut socket, id, json!({"echo": format!("request-{case}")})).await; }); let mut config = test_config(url); config.capabilities = vec![format!("matrix.{case}")]; let client = SdkClient::connect(config).await.unwrap(); let response = client.request(operation, input).await.unwrap(); assert_eq!( response.result(), Some(json!({"echo": format!("request-{case}")})) ); client.close().await.unwrap(); server.await.unwrap(); } macro_rules! sdk_round_trip_matrix { ($($name:ident: $case:literal),+ $(,)?) => { $( #[tokio::test] async fn $name() { round_trip_case($case).await; } )+ }; } sdk_round_trip_matrix! { protocol_case_000: 0, protocol_case_001: 1, protocol_case_002: 2, protocol_case_003: 3, protocol_case_004: 4, protocol_case_005: 5, protocol_case_006: 6, protocol_case_007: 7, protocol_case_008: 8, protocol_case_009: 9, protocol_case_010: 10, protocol_case_011: 11, protocol_case_012: 12, protocol_case_013: 13, protocol_case_014: 14, protocol_case_015: 15, protocol_case_016: 16, protocol_case_017: 17, protocol_case_018: 18, protocol_case_019: 19, protocol_case_020: 20, protocol_case_021: 21, protocol_case_022: 22, protocol_case_023: 23, protocol_case_024: 24, protocol_case_025: 25, protocol_case_026: 26, protocol_case_027: 27, protocol_case_028: 28, protocol_case_029: 29, protocol_case_030: 30, protocol_case_031: 31, protocol_case_032: 32, protocol_case_033: 33, protocol_case_034: 34, protocol_case_035: 35, protocol_case_036: 36, protocol_case_037: 37, protocol_case_038: 38, protocol_case_039: 39, protocol_case_040: 40, protocol_case_041: 41, protocol_case_042: 42, protocol_case_043: 43, protocol_case_044: 44, protocol_case_045: 45, protocol_case_046: 46, protocol_case_047: 47, protocol_case_048: 48, protocol_case_049: 49, protocol_case_050: 50, protocol_case_051: 51, protocol_case_052: 52, protocol_case_053: 53, protocol_case_054: 54, protocol_case_055: 55, protocol_case_056: 56, protocol_case_057: 57, protocol_case_058: 58, protocol_case_059: 59, protocol_case_060: 60, protocol_case_061: 61, protocol_case_062: 62, protocol_case_063: 63, } }