cleanup

entities/src/host_overrides.rs

@@ -3,16 +3,11 @@
 use sea_orm::entity::prelude::*;
 use serde::Serialize;
 
-use crate::host;
-
 /// Module of override keys
 pub mod keys {
     use std::{collections::HashMap, sync::LazyLock};
 
-    use sea_orm::{
+    use sea_orm::{ActiveValue, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, QueryFilter};
-        ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, DbErr, EntityTrait,
-        QueryFilter,
-    };
     use sea_query::OnConflict;
     use serde::Serialize;
     use tracing::warn;
@@ -37,8 +32,12 @@ pub mod keys {
     pub static NOT_LOCKED_DEFAULTS: &'static [&'static str] =
         &[KEY_HOST_HEALTH_PATH, KEY_HOST_BEARER, KEY_HOST_HEALTH_QUERY];
 
-    pub static ALL_OVERRIDES: &'static [&'static str] =
+    pub static ALL_OVERRIDES: &'static [&'static str] = &[
-        &[KEY_BAD_HOST, KEY_HOST_BEARER, KEY_HOST_HEALTH_PATH, KEY_HOST_HEALTH_QUERY];
+        KEY_BAD_HOST,
+        KEY_HOST_BEARER,
+        KEY_HOST_HEALTH_PATH,
+        KEY_HOST_HEALTH_QUERY,
+    ];
 
     #[derive(Serialize, Clone, Copy)]
     pub enum ValueType {
@@ -139,9 +138,7 @@ pub mod keys {
 
         /// Retrieve override value for key
         pub fn get(&self, key: &str) -> Option<&str> {
-            self.entries
+            self.entries.get(key).and_then(|v| v.value.as_deref())
-            .get(key)
-            .and_then(|v| v.value.as_deref())
         }
 
         pub fn entries(&self) -> &HashMap<String, Override> {

entities/src/instance_stats.rs

@@ -135,7 +135,7 @@ pub struct StatsCSVEntry {
 
 impl StatsCSVEntry {
     /// Fetch health check graph data for all or selected hosts in the selected time range.
-    pub async fn fetch(db: &DatabaseConnection,hosts: Option<&[i32]>,) -> Result<Vec<Self>, DbErr> {
+    pub async fn fetch(db: &DatabaseConnection, hosts: Option<&[i32]>) -> Result<Vec<Self>, DbErr> {
         let builder = db.get_database_backend();
         let columns = ["limited_accs", "total_accs", "total_requests"];
         let mut stmt: sea_query::SelectStatement = Query::select();

migration/src/m20250216_185501_overrides.rs

@@ -14,7 +14,8 @@ impl MigrationTrait for Migration {
             CONSTRAINT "pk_override_hostkey" PRIMARY KEY ("host", "key"),
             FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE
         ) STRICT;"#;
-        let cmd_overrides_index = r#"CREATE INDEX "index_overrides_key" ON host_overrides ("key");"#;
+        let cmd_overrides_index =
+            r#"CREATE INDEX "index_overrides_key" ON host_overrides ("key");"#;
         let cmd_log = r#"CREATE TABLE "log" (
             "user_host" integer NOT NULL,
             "host_affected" integer,

scanner/src/cache_update.rs

@@ -1,19 +1,19 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 use std::cmp;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
 use chrono::{Days, Utc};
 use chrono::{Duration, TimeZone};
 use entities::host_overrides::keys::{KEY_BAD_HOST, VAL_BOOL_TRUE};
-use entities::{host, host_overrides};
 use entities::prelude::*;
 use entities::state::CacheData;
 use entities::state::CacheHost;
-use sea_orm::{ColumnTrait, QuerySelect, QueryTrait};
+use entities::{host, host_overrides};
 use sea_orm::EntityTrait;
 use sea_orm::QueryFilter;
 use sea_orm::QueryOrder;
 use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement};
+use sea_orm::{ColumnTrait, QuerySelect};
 
 use crate::version_check::fetch_git_state;
 use crate::LatestCheck;
@@ -193,8 +193,11 @@ impl Scanner {
 
     async fn query_bad_hosts(&self) -> Result<Vec<i32>> {
         let res: Vec<i32> = HostOverrides::find()
-            .filter(host_overrides::Column::Key.eq(KEY_BAD_HOST)
+            .filter(
-            .and(host_overrides::Column::Value.eq(VAL_BOOL_TRUE)))
+                host_overrides::Column::Key
+                    .eq(KEY_BAD_HOST)
+                    .and(host_overrides::Column::Value.eq(VAL_BOOL_TRUE)),
+            )
             .select_only()
             .column(host_overrides::Column::Host)
             .into_tuple()

scanner/src/cleanup.rs

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
 use entities::check_errors;
 use entities::host;
 use sea_orm::ColumnTrait;

scanner/src/instance_check.rs

@@ -3,10 +3,10 @@
 use std::time::Instant;
 
 use chrono::Utc;
-use entities::state::error_cache::HostError;
-use entities::{check_errors, health_check, host_overrides};
-use entities::{host, prelude::*};
 use entities::host_overrides::keys::HostOverrides;
+use entities::state::error_cache::HostError;
+use entities::{check_errors, health_check};
+use entities::{host, prelude::*};
 use reqwest::Url;
 use sea_orm::prelude::DateTimeUtc;
 use sea_orm::ColumnTrait;
@@ -40,8 +40,9 @@ impl Scanner {
                 .find(|v| v.host == model.id)
                 .map_or(false, |check| !check.healthy);
             join_set.spawn(async move {
-                if let Err(e) = scanner.health_check_host(model, muted_host).await {
+                if let Err(error) = scanner.health_check_host(model, muted_host).await {
-
+                    // should only happen if for example the DB fails
+                    tracing::error!(%error,"Failed to perform host health check");
                 }
             });
         }
@@ -191,7 +192,12 @@ impl Scanner {
     }
 
     /// Check nitter version
-    pub(crate) async fn nitter_version(&self, url: &mut Url, api_token: Option<&str>, mute: bool) -> Option<AboutParsed> {
+    pub(crate) async fn nitter_version(
+        &self,
+        url: &mut Url,
+        api_token: Option<&str>,
+        mute: bool,
+    ) -> Option<AboutParsed> {
         url.set_path(&self.inner.config.about_path);
         match self.fetch_url(url.as_str(), api_token).await {
             Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) {

scanner/src/lib.rs

@@ -6,21 +6,16 @@ use std::{
 
 use about_parser::AboutParser;
 use chrono::{DateTime, Duration, TimeZone, Utc};
-use entities::{
+use entities::state::{error_cache::HostError, scanner::ScannerConfig, AppState};
-    health_check, host, state::{error_cache::HostError, scanner::ScannerConfig, AppState}
-};
 use instance_parser::InstanceParser;
 use miette::{Context, IntoDiagnostic};
 use profile_parser::ProfileParser;
 use regex::{Regex, RegexBuilder};
 use reqwest::{
     header::{HeaderMap, HeaderValue},
-    Client, ClientBuilder, Url,
+    Client, ClientBuilder,
-};
-use sea_orm::{
-    ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, EntityTrait, FromQueryResult,
-    QuerySelect, Statement,
 };
+use sea_orm::{ConnectionTrait, DatabaseConnection, DbBackend, FromQueryResult, Statement};
 use thiserror::Error;
 use tokio::time::sleep;
 
@@ -33,8 +28,8 @@ mod instance_check;
 mod instance_parser;
 mod list_update;
 mod profile_parser;
-mod version_check;
 mod update_stats;
+mod version_check;
 
 const CAPTCHA_TEXT: &'static str = "Enable JavaScript and cookies to continue";
 const CAPTCHA_CODE: u16 = 403;
@@ -73,7 +68,7 @@ pub enum ScannerError {
     InstanceUrlParse,
     /// Failed to join a tokio thread in a JoinSet
     #[error("Failed to join worker task {0}")]
-    JoinError(#[from] tokio::task::JoinError)
+    JoinError(#[from] tokio::task::JoinError),
 }
 
 #[derive(Error, Debug)]
@@ -208,16 +203,16 @@ impl Scanner {
             .build()
             .into_diagnostic()?;
 
-        let last_uptime_check = Self::query_last_entry_for_table(&db,"health_check")
+        let last_uptime_check = Self::query_last_entry_for_table(&db, "health_check")
             .await
             .into_diagnostic()
             .wrap_err("Fetching last uptime check failed!")?;
 
-        let last_stats_check = Self::query_last_entry_for_table(&db,"instance_stats")
+        let last_stats_check = Self::query_last_entry_for_table(&db, "instance_stats")
             .await
             .into_diagnostic()
             .wrap_err("Fetching last stats check failed!")?;
-        tracing::info!(?last_uptime_check, ?last_stats_check);// TODO: wrong timestamp
+        tracing::info!(?last_uptime_check, ?last_stats_check); // TODO: wrong timestamp
         let scanner = Self {
             inner: Arc::new(InnerScanner {
                 db,
@@ -247,7 +242,10 @@ impl Scanner {
     }
 
     /// Retrieves the last stats fetching that happened
-    pub async fn query_last_entry_for_table(db: &DatabaseConnection, table: &str) -> Result<DateTime<Utc>> {
+    pub async fn query_last_entry_for_table(
+        db: &DatabaseConnection,
+        table: &str,
+    ) -> Result<DateTime<Utc>> {
         #[derive(Debug, FromQueryResult, Default)]
         pub(crate) struct TimeMax {
             pub max: Option<i64>,
@@ -315,12 +313,20 @@ impl Scanner {
     }
 
     async fn sleep_till_deadline(&self) {
-        let delay_instance_check = self.last_uptime_check() + self.inner.config.instance_check_interval;
+        let delay_instance_check =
+            self.last_uptime_check() + self.inner.config.instance_check_interval;
         let delay_list_update = self.last_list_fetch() + self.inner.config.list_fetch_interval;
-        let delay_stats_update = self.last_stats_fetch() + self.inner.config.instance_stats_interval;
+        let delay_stats_update =
-        tracing::debug!(?delay_list_update, ?delay_instance_check, ?delay_stats_update);
+            self.last_stats_fetch() + self.inner.config.instance_stats_interval;
+        tracing::debug!(
+            ?delay_list_update,
+            ?delay_instance_check,
+            ?delay_stats_update
+        );
 
-        let next_deadline = delay_instance_check.min(delay_list_update).min(delay_stats_update);
+        let next_deadline = delay_instance_check
+            .min(delay_list_update)
+            .min(delay_stats_update);
         let now = Utc::now();
         let sleep_time = next_deadline.signed_duration_since(now);
         if sleep_time <= Duration::zero() {
@@ -363,11 +369,17 @@ impl Scanner {
     }
 
     async fn fetch_instance_list(&self) -> Result<String> {
-        let (_, body) = self.fetch_url(&self.inner.config.instance_list_url, None).await?;
+        let (_, body) = self
+            .fetch_url(&self.inner.config.instance_list_url, None)
+            .await?;
         Ok(body)
     }
 
-    async fn fetch_url(&self, url: &str, api_token: Option<&str>) -> std::result::Result<(u16, String), FetchError> {
+    async fn fetch_url(
+        &self,
+        url: &str,
+        api_token: Option<&str>,
+    ) -> std::result::Result<(u16, String), FetchError> {
         let mut request = self.inner.client.get(url);
         if let Some(token) = api_token {
             request = request.bearer_auth(token);
@@ -389,24 +401,34 @@ impl Scanner {
             }
             if code == 403 && body_text.contains("You have been blocked") {
                 // cloudflare block
-                return Err(FetchError::KnownHttpResponseStatus(code, message, body_text));
+                return Err(FetchError::KnownHttpResponseStatus(
+                    code, message, body_text,
+                ));
             }
             if code == 429 && body_text.contains("Instance has been rate limited") {
                 // out of non-limited accounts
-                return Err(FetchError::KnownHttpResponseStatus(code, message, body_text));
+                return Err(FetchError::KnownHttpResponseStatus(
+                    code, message, body_text,
+                ));
             }
             if code == 404 {
                 // don't spam the body on 404s
-                return Err(FetchError::KnownHttpResponseStatus(code, message, body_text));
+                return Err(FetchError::KnownHttpResponseStatus(
+                    code, message, body_text,
+                ));
             }
             if code >= 502 && code <= 504 {
                 // don't spam the body on Bad Gateway/Service Unavailable/Gateway Timeout
-                return Err(FetchError::KnownHttpResponseStatus(code, message, body_text));
+                return Err(FetchError::KnownHttpResponseStatus(
+                    code, message, body_text,
+                ));
             }
             if code >= 520 && code <= 527 {
                 // don't spam the body on Cloudflare errors
                 // https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
-                return Err(FetchError::KnownHttpResponseStatus(code, message, body_text));
+                return Err(FetchError::KnownHttpResponseStatus(
+                    code, message, body_text,
+                ));
             }
             return Err(FetchError::HttpResponseStatus(code, message, body_text));
         }
@@ -481,7 +503,7 @@ mod test {
         let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
             .await
             .unwrap();
-        let (_, res) = scanner.fetch_url("example.com/jack",None).await.unwrap();
+        let (_, res) = scanner.fetch_url("example.com/jack", None).await.unwrap();
         let mut file = File::create("test_data/blocked.html").await.unwrap();
         file.write_all(&res.as_bytes()).await.unwrap();
     }

scanner/src/list_update.rs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 //! Updates the list of available instances, fetching all required fields
 
-use std::collections::HashMap;
 use std::time::Duration;
 use std::time::Instant;
 

scanner/src/update_stats.rs

@@ -27,8 +27,11 @@ struct InstanceStats {
 struct InstanceStatsAccs {
     total: i32,
     limited: i32,
+    #[allow(unused)]
     oldest: DateTimeUtc,
+    #[allow(unused)]
     newest: DateTimeUtc,
+    #[allow(unused)]
     average: DateTimeUtc,
 }
 
@@ -102,7 +105,7 @@ impl Scanner {
         host: &host::Model,
     ) -> Result<instance_stats::ActiveModel> {
         let overrides = HostOverrides::load(&host, &self.inner.db).await?;
-        let mut url = Url::parse(&host.url).map_err(|e| ScannerError::InstanceUrlParse)?;
+        let mut url = Url::parse(&host.url).map_err(|_e| ScannerError::InstanceUrlParse)?;
         url.set_path(".health");
         if let Some(url_override) = overrides.health_path() {
             url.set_path(url_override);

server/src/admin/logs.rs

@@ -1,7 +1,4 @@
-use std::{
+use std::{collections::HashMap, sync::Arc};
-    collections::HashMap,
-    sync::Arc,
-};
 
 use crate::{Result, ServerError};
 use axum::{
@@ -9,10 +6,7 @@ use axum::{
     response::{Html, IntoResponse},
 };
 use chrono::{TimeZone, Utc};
-use entities::{
+use entities::{host, log};
-    host,
-    log,
-};
 use sea_orm::EntityTrait;
 use sea_orm::{DatabaseConnection, QueryOrder, QuerySelect};
 use serde::Serialize;

server/src/admin/mod.rs

@@ -1,7 +1,6 @@
 use std::collections::HashSet;
 use std::sync::Arc;
 
-use axum::extract::Path;
 use axum::extract::State;
 use axum::response::Html;
 use axum::response::IntoResponse;
@@ -24,7 +23,6 @@ use sea_orm::ColumnTrait;
 use sea_orm::DatabaseConnection;
 use sea_orm::EntityTrait;
 use sea_orm::QueryFilter;
-use sea_orm::QueryOrder;
 use serde::Deserialize;
 use serde::Serialize;
 use sha2::{Digest, Sha256};
@@ -42,21 +40,21 @@ use crate::ADMIN_OVERVIEW_URL;
 use crate::LOGIN_URL;
 
 mod errors;
-mod settings;
 mod locks;
 mod logs;
+mod settings;
 mod stats;
 
 pub use errors::errors_view;
-pub use settings::post_settings;
-pub use settings::settings_view;
 pub use locks::locks_view;
 pub use locks::post_locks;
 pub use logs::log_view;
-pub use stats::stats_view;
+pub use settings::post_settings;
+pub use settings::settings_view;
 pub use stats::health_csv_api;
-pub use stats::stats_csv_api;
 pub use stats::overview_csv_api;
+pub use stats::stats_csv_api;
+pub use stats::stats_view;
 
 /// Stored session login information
 #[derive(Serialize, Deserialize, Default)]
@@ -368,34 +366,6 @@ pub async fn history_json(
     .into_response())
 }
 
-pub async fn history_view(
-    State(ref app_state): State<AppState>,
-    State(ref template): State<Arc<tera::Tera>>,
-    State(ref db): State<DatabaseConnection>,
-    Path(host): Path<i32>,
-    session: Session,
-) -> Result<axum::response::Response> {
-    tracing::info!(?session);
-
-    let (host, _login) = get_specific_login_host(host, &session, db).await?;
-
-    let mut context = tera::Context::new();
-    let res = {
-        let guard = app_state
-            .cache
-            .read()
-            .map_err(|_| ServerError::MutexFailure)?;
-        let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
-        context.insert("last_updated", &time);
-        context.insert("HOST_DOMAIN", &host.domain);
-
-        let res = Html(template.render("errors_admin.html.j2", &context)?).into_response();
-        drop(guard);
-        res
-    };
-    Ok(res)
-}
-
 /// Get all [host::Model] for current [Session], optionally return all hosts for admins
 async fn get_all_login_hosts(
     session: &Session,

server/src/admin/settings.rs

@@ -1,7 +1,4 @@
-use std::{
+use std::sync::Arc;
-    sync::Arc,
-    time::{SystemTime, UNIX_EPOCH},
-};
 
 use crate::{Result, ServerError};
 use axum::{

server/src/admin/stats.rs

@@ -1,27 +1,16 @@
-use std::{collections::HashMap, fmt::Write, sync::Arc};
+use std::{fmt::Write as _, sync::Arc};
 
 use axum::{
     extract::{Path, State},
     http::HeaderValue,
-    response::{Html, IntoResponse, Redirect},
+    response::{Html, IntoResponse},
-    Form,
 };
 use chrono::{TimeZone, Utc};
-use entities::{
+use entities::{health_check::ResponseTimeStat, instance_stats};
-    health_check::{self, ResponseTimeStat},
+use sea_orm::DatabaseConnection;
-    host_overrides::{
-        self,
-        keys::{HostOverrides, LOCKED_FALSE, LOCKED_TRUE},
-    },
-    instance_stats,
-};
-use sea_orm::{sea_query::OnConflict, ActiveValue, DatabaseConnection, EntityTrait};
 use tower_sessions::Session;
 
-use crate::{
+use crate::{admin::get_all_login_hosts, Result, ServerError};
-    admin::{get_all_login_hosts, get_session_login},
-    Result, ServerError,
-};
 
 use super::get_specific_login_host;
 

server/src/api.rs

@@ -2,13 +2,8 @@
 use crate::{Result, ServerError};
 use axum::response::IntoResponse;
 use axum::{extract::State, Json};
-use chrono::{TimeZone, Utc};
 use entities::state::AppState;
-use entities::{health_check, instance_stats};
 use hyper::http::HeaderValue;
-use sea_orm::DatabaseConnection;
-use serde::Serialize;
-use std::fmt::Write;
 use std::sync::Arc;
 
 pub async fn instances(

server/src/website.rs

@@ -78,26 +78,3 @@ pub async fn about(
     );
     Ok(res)
 }
-
-pub async fn rip(
-    State(ref app_state): State<AppState>,
-    State(ref template): State<Arc<tera::Tera>>,
-    State(ref scanner_config): State<ScannerConfig>,
-) -> Result<axum::response::Response> {
-    let mut context = tera::Context::new();
-    {
-        let guard = app_state
-            .cache
-            .read()
-            .map_err(|_| ServerError::MutexFailure)?;
-        let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
-        context.insert("last_updated", &time);
-    }
-
-    let mut res = Html(template.render("rip.html.j2", &context)?).into_response();
-    res.headers_mut().insert(
-        "cache-control",
-        HeaderValue::from_static("public, max-age=900"),
-    );
-    Ok(res)
-}

server/static/admin.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 document.addEventListener("DOMContentLoaded", function() {
     const startDateInput = document.getElementById('startDate');
     const endDateInput = document.getElementById('endDate');

server/static/admin_index.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 document.addEventListener("DOMContentLoaded", function (event) {
   const startDateInput = document.getElementById('startDate');
   const endDateInput = document.getElementById('endDate');

server/static/history.2.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 document.addEventListener("DOMContentLoaded", function (event) {
   var loadGraphBtn = document.getElementById("loadGraphBtn");
   if (loadGraphBtn) {

server/static/instance.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 let chart = undefined;
 document.addEventListener("DOMContentLoaded", function() {
     const startDateInput = document.getElementById('startDate');

server/static/login_key.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 async function sha256(message) {
     const msgBuffer = new TextEncoder().encode(message);                    
     const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);

server/static/stats.js

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: AGPL-3.0-only
 document.addEventListener("DOMContentLoaded", function (event) {
   var host_id = document.getElementById("body").getAttribute("data-hid");
   var loadGraphBtn = document.getElementById("loadGraphBtn");
@@ -19,7 +20,7 @@ async function loadHealthGraph(host_id) {
   try {
     let g = new Dygraph(
       graphDiv,
-      "/admin/api/graphs/health/"+host_id,
+      "/admin/api/graphs/health/" + host_id,
       {
         title: 'Historic Instance Healthiness',
         showRangeSelector: true,
@@ -34,6 +35,7 @@ async function loadHealthGraph(host_id) {
         ylabel: 'Nitter Instances',
       }
     );
+    // fixme: find valid annotation dates, regardless of the host
     g.ready(function () {
       g.setAnnotations([
         {
@@ -66,7 +68,7 @@ async function loadStatsGraph(host_id) {
   try {
     let g = new Dygraph(
       graphDiv,
-      "/admin/api/graphs/stats/"+host_id,
+      "/admin/api/graphs/stats/" + host_id,
       {
         title: 'Historic Average Instance Statistics',
         showRangeSelector: true,

server/static/table_toggle.js

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
 // Get the table and checkboxes
 const dataTable = document.getElementById('status-tbl');
 const toggleColumnCheckboxes = document.querySelectorAll('.toggle-column');

server/templates/dygraph_fix.html.j2

@@ -1,3 +1,4 @@
+{#- SPDX-License-Identifier: AGPL-3.0-only #}
     <style>
       .dygraph-annotation {
         height: auto !important;