From d56e76c23ec92ef86a54b843308cf759de4e70d0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 25 Dec 2023 20:05:05 -0500 Subject: [PATCH] raise various timeouts, make exp backoff consistent Signed-off-by: strawberry --- src/api/appservice_server.rs | 2 +- src/api/client_server/keys.rs | 4 ++-- src/api/client_server/membership.rs | 2 +- src/main.rs | 2 +- src/service/globals/mod.rs | 4 ++-- src/service/rooms/event_handler/mod.rs | 4 ++-- src/service/sending/mod.rs | 9 +++++---- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/api/appservice_server.rs b/src/api/appservice_server.rs index 9d47f8b6..e296ef26 100644 --- a/src/api/appservice_server.rs +++ b/src/api/appservice_server.rs @@ -44,7 +44,7 @@ where let mut reqwest_request = reqwest::Request::try_from(http_request)?; - *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + *reqwest_request.timeout_mut() = Some(Duration::from_secs(120)); let url = reqwest_request.url().clone(); let mut response = match services() diff --git a/src/api/client_server/keys.rs b/src/api/client_server/keys.rs index d71ac888..bc82bd98 100644 --- a/src/api/client_server/keys.rs +++ b/src/api/client_server/keys.rs @@ -370,7 +370,7 @@ pub(crate) async fn get_keys_helper bool>( .get(server) { // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); + let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { min_elapsed_duration = Duration::from_secs(60 * 60 * 24); } @@ -391,7 +391,7 @@ pub(crate) async fn get_keys_helper bool>( ( server, tokio::time::timeout( - Duration::from_secs(25), + Duration::from_secs(50), services().sending.send_federation_request( server, federation::keys::get_keys::v1::Request { diff --git a/src/api/client_server/membership.rs b/src/api/client_server/membership.rs index f1513ca8..640d8048 100644 --- a/src/api/client_server/membership.rs +++ b/src/api/client_server/membership.rs @@ -1163,7 +1163,7 @@ fn validate_and_add_event_id( .get(&event_id) { // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); + let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { min_elapsed_duration = Duration::from_secs(60 * 60 * 24); } diff --git a/src/main.rs b/src/main.rs index 4ec3e48f..84ab470e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -545,7 +545,7 @@ async fn shutdown_signal(handle: ServerHandle, tx: Sender<()>) -> Result<()> { } warn!("Received {}, shutting down...", sig); - handle.graceful_shutdown(Some(Duration::from_secs(30))); + handle.graceful_shutdown(Some(Duration::from_secs(60))); services().globals.shutdown(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 228424f5..090b35a4 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -537,8 +537,8 @@ impl Service<'_> { fn reqwest_client_builder(config: &Config) -> Result { let mut reqwest_client_builder = reqwest::Client::builder() .pool_max_idle_per_host(0) - .connect_timeout(Duration::from_secs(30)) - .timeout(Duration::from_secs(60 * 3)); + .connect_timeout(Duration::from_secs(60)) + .timeout(Duration::from_secs(60 * 4)); if let Some(proxy) = config.proxy.to_proxy()? { reqwest_client_builder = reqwest_client_builder.proxy(proxy); diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 729a7a2b..f11b59cb 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1461,7 +1461,7 @@ impl Service { .get(event_id) { // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); + let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { min_elapsed_duration = Duration::from_secs(60 * 60 * 24); } @@ -1748,7 +1748,7 @@ impl Service { .get(&signature_ids) { // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); + let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { min_elapsed_duration = Duration::from_secs(60 * 60 * 24); } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 46bcc022..9a12f7b4 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -219,7 +219,8 @@ impl Service { } TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); + let mut min_elapsed_duration = + Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { min_elapsed_duration = Duration::from_secs(60 * 60 * 24); } @@ -719,13 +720,13 @@ impl Service { let permit = self.maximum_requests.acquire().await; debug!("Got permit"); let response = tokio::time::timeout( - Duration::from_secs(2 * 60), + Duration::from_secs(5 * 60), server_server::send_request(destination, request), ) .await .map_err(|_| { - warn!("Timeout waiting for server response of {destination}"); - Error::BadServerResponse("Timeout waiting for server response") + warn!("Timeout after 300 seconds waiting for server response of {destination}"); + Error::BadServerResponse("Timeout after 300 seconds waiting for server response") })?; drop(permit);