retry instance fetching

scanner/src/lib.rs

@@ -12,8 +12,7 @@ use miette::{Context, IntoDiagnostic};
 use profile_parser::ProfileParser;
 use regex::{Regex, RegexBuilder};
 use reqwest::{
-    header::{HeaderMap, HeaderValue},
-    Client, ClientBuilder,
+    Client, ClientBuilder, header::{HeaderMap, HeaderValue}, retry
 };
 use sea_orm::{ConnectionTrait, DatabaseConnection, DbBackend, FromQueryResult, Statement};
 use thiserror::Error;
@@ -183,9 +182,8 @@ impl Scanner {
             .tls_backend_native()
             .gzip(true)
             .user_agent(user_agent)
-            .connect_timeout(std::time::Duration::from_secs(3))
+            .connect_timeout(std::time::Duration::from_secs(5))
             .timeout(std::time::Duration::from_secs(10))
-            .pool_max_idle_per_host(1)
             .default_headers(headers);
         http_client
     }
@@ -307,11 +305,14 @@ impl Scanner {
                 if let Err(e) = self.check_uptime().await {
                     tracing::error!(error=?e,"Failed checking instances for health");
                 }
-            }
-            if self.is_instance_stats_outdated() {
-                tracing::info!("Checking instance stats");
-                if let Err(e) = self.check_health().await {
-                    tracing::error!(error=?e,"Failed checking instances for stats");
+                // schedule instance stats always post health checks
+                // and wait enough time
+                if self.is_instance_stats_outdated() {
+                    sleep(std::time::Duration::from_secs(1)).await;
+                    tracing::info!("Checking instance stats");
+                    if let Err(e) = self.check_health().await {
+                        tracing::error!(error=?e,"Failed checking instances for stats");
+                    }
                 }
             }
             if let Err(e) = self.update_cache().await {

scanner/src/list_update.rs

@@ -14,6 +14,7 @@ use sea_orm::{
 };
 use sea_query::OnConflict;
 use tokio::task::JoinSet;
+use tokio::time::sleep;
 use tracing::instrument;
 
 use crate::Result;
@@ -25,7 +26,23 @@ impl Scanner {
     #[instrument]
     pub(crate) async fn update_instacelist(&mut self) -> Result<()> {
         let start = Instant::now();
-        let html: String = self.fetch_instance_list().await?;
+        // retry loop
+        let mut fetch_result;
+        fetch_result = self.fetch_instance_list().await;
+        for attempt in 1..6 {
+            match fetch_result.as_ref() {
+                Err(error) => tracing::warn!(attempt,%error,"Failed fetching instance list"),
+                Ok(_) => {
+                    break;
+                }
+            }
+            sleep(Duration::from_secs(1)).await;
+            fetch_result = self.fetch_instance_list().await;
+        }
+        let html = match fetch_result {
+            Ok(v) => v,
+            Err(e) => return Err(e),
+        };
         let parsed_instances = self.inner.instance_parser.parse_instancelist(
             &html,
             &self.inner.config.additional_hosts,

server/src/admin/errors.rs

@@ -23,8 +23,6 @@ pub async fn errors_view(
     Path(instance): Path<i32>,
     session: Session,
 ) -> Result<axum::response::Response> {
-    tracing::info!(?session);
-
     let (host, _login) = get_specific_login_host(instance, &session, db).await?;
 
     let errors = check_errors::Entity::find()