cleanup old errors

scanner/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2021"
 [dependencies]
 reqwest = { workspace = true, features = ["deflate","gzip","brotli","cookies", "rustls-tls"] }
 sea-orm = { workspace = true, features = [ "sqlx-sqlite", "runtime-tokio-native-tls", "macros" ] }
+sea-query = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
 tokio = { workspace = true, features = ["full"] }

scanner/src/cleanup.rs

@@ -0,0 +1,59 @@
+use entities::check_errors;
+use entities::health_check;
+use entities::host;
+use sea_orm::ColumnTrait;
+use sea_orm::EntityTrait;
+use sea_orm::Order;
+use sea_orm::QueryFilter;
+use sea_orm::QueryOrder;
+use sea_orm::QuerySelect;
+use sea_query::Query;
+use tokio::time::interval;
+
+use crate::Scanner;
+use crate::Result;
+
+impl Scanner {
+    /// Setup scheduled job for cleaning up old data
+    pub(crate) fn schedule_cleanup(&self) -> Result<()> {
+        let c = self.clone();
+        tokio::spawn(async move {
+            let mut interval = interval(c.inner.config.cleanup_interval);
+            loop {
+                interval.tick().await;
+                if let Err(e) = c.cleanup().await {
+                    tracing::error!(error=%e);
+                }
+            }
+        });
+        Ok(())
+    }
+    /// Perform cleanup of outdated data
+    async fn cleanup(&self) -> Result<()> {
+        self.cleanup_errors().await?;
+        Ok(())
+    }
+
+    /// Remove all but recent host error entries
+    async fn cleanup_errors(&self) -> Result<()> {
+        // I wish this was easier without row ids or giant SQL queries
+        let hosts = host::Entity::find().all(&self.inner.db).await?;
+        for host in hosts {
+            let res = check_errors::Entity::delete_many()
+            .filter(check_errors::Column::Host.eq(host.id))
+            .filter(check_errors::Column::Time.not_in_subquery(
+                Query::select()
+                .column(check_errors::Column::Time)
+                .from(check_errors::Entity)
+                .and_where(check_errors::Column::Host.eq(host.id))
+                .order_by(check_errors::Column::Time, Order::Desc)
+                .limit(self.inner.config.error_retention_per_host as _)
+                .to_owned()
+            ))
+            .exec(&self.inner.db).await?;
+            tracing::debug!(host=host.id,deleted_errors=res.rows_affected);
+        }
+
+        Ok(())
+    }
+}

scanner/src/lib.rs

@@ -36,6 +36,7 @@ mod instance_check;
 mod instance_parser;
 mod profile_parser;
 mod version_check;
+mod cleanup;
 
 const CAPTCHA_TEXT: &'static str = "Enable JavaScript and cookies to continue";
 const CAPTCHA_CODE: u16 = 403;
@@ -116,6 +117,7 @@ pub async fn run_scanner(
     disable_health_checks: bool,
 ) -> Result<()> {
     let scanner = Scanner::new(db, config, app_state).await?;
+    scanner.schedule_cleanup().unwrap();
 
     if disable_health_checks {
         tracing::error!("Health checks disabled!");
@@ -284,7 +286,7 @@ impl Scanner {
     #[instrument]
     async fn update_instacelist(&mut self) -> Result<()> {
         let start = Instant::now();
-        let html: String = self.fetch_instancelist().await?;
+        let html: String = self.fetch_instance_list().await?;
         let parsed_instances = self.inner.instance_parser.parse_instancelist(
             &html,
             &self.inner.config.additional_hosts,
@@ -403,7 +405,7 @@ impl Scanner {
         Ok(())
     }
 
-    async fn fetch_instancelist(&self) -> Result<String> {
+    async fn fetch_instance_list(&self) -> Result<String> {
         let (_, body) = self.fetch_url(&self.inner.config.instance_list_url).await?;
         Ok(body)
     }
@@ -554,7 +556,7 @@ mod test {
         let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
             .await
             .unwrap();
-        let res = scanner.fetch_instancelist().await.unwrap();
+        let res = scanner.fetch_instance_list().await.unwrap();
         let mut file = File::create("test_data/instancelist.html").await.unwrap();
         file.write_all(&res.as_bytes()).await.unwrap();
     }