From f2dcfc12112560fc4a2b637257aae8832c8b5f78 Mon Sep 17 00:00:00 2001 From: 4cuck <40539855+4cuck@users.noreply.github.com> Date: Sun, 16 Feb 2025 21:06:57 +0100 Subject: [PATCH] host overrides and admin keys --- .env.example | 4 +- .tokeignore | 4 + .vscode/settings.json | 3 +- Cargo.lock | 3 +- entities/Cargo.toml | 1 + entities/src/host.rs | 26 +- entities/src/host_overrides.rs | 226 ++++++++++++++++++ entities/src/log.rs | 38 +++ entities/src/mod.rs | 2 + entities/src/prelude.rs | 2 + entities/src/state/scanner.rs | 3 - migration/src/lib.rs | 2 + migration/src/m20250216_185501_overrides.rs | 47 ++++ scanner/src/cache_update.rs | 28 ++- scanner/src/cleanup.rs | 12 +- scanner/src/instance_check.rs | 24 +- scanner/src/lib.rs | 17 +- scanner/src/list_update.rs | 47 ++-- scanner/src/update_stats.rs | 39 ++- server/src/admin/errors.rs | 54 +++++ server/src/admin/locks.rs | 84 +++++++ server/src/admin/logs.rs | 105 ++++++++ server/src/{admin.rs => admin/mod.rs} | 106 ++++---- server/src/admin/settings.rs | 121 ++++++++++ server/src/lib.rs | 25 +- server/templates/admin.html.j2 | 3 +- server/templates/admin_logs.html.j2 | 44 ++++ ..._admin.html.j2 => instance_errors.html.j2} | 0 server/templates/instance_locks.html.j2 | 38 +++ server/templates/instance_settings.html.j2 | 84 +++++++ src/main.rs | 36 ++- 31 files changed, 1088 insertions(+), 140 deletions(-) create mode 100644 .tokeignore create mode 100644 entities/src/host_overrides.rs create mode 100644 entities/src/log.rs create mode 100644 migration/src/m20250216_185501_overrides.rs create mode 100644 server/src/admin/errors.rs create mode 100644 server/src/admin/locks.rs create mode 100644 server/src/admin/logs.rs rename server/src/{admin.rs => admin/mod.rs} (86%) create mode 100644 server/src/admin/settings.rs create mode 100644 server/templates/admin_logs.html.j2 rename server/templates/{instance_admin.html.j2 => instance_errors.html.j2} (100%) create mode 100644 server/templates/instance_locks.html.j2 create mode 100644 server/templates/instance_settings.html.j2 diff --git a/.env.example b/.env.example index 50296bb..dfcf400 100644 --- a/.env.example +++ b/.env.example @@ -26,8 +26,6 @@ RSS_CONTENT=' RelationDef { match self { Self::UpdateCheck => Entity::has_many(super::health_check::Entity).into(), + Self::CheckErrors => Entity::has_many(super::check_errors::Entity).into(), + Self::HostOverrides => Entity::has_many(super::host_overrides::Entity).into(), + Self::InstanceStats => Entity::has_many(super::instance_stats::Entity).into(), } } } @@ -115,4 +121,22 @@ impl Related for Entity { } } -impl ActiveModelBehavior for ActiveModel {} \ No newline at end of file +impl Related for Entity { + fn to() -> RelationDef { + Relation::CheckErrors.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::HostOverrides.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::InstanceStats.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/entities/src/host_overrides.rs b/entities/src/host_overrides.rs new file mode 100644 index 0000000..60b36ee --- /dev/null +++ b/entities/src/host_overrides.rs @@ -0,0 +1,226 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 + +use sea_orm::entity::prelude::*; +use serde::Serialize; + +use crate::host; + +/// Module of override keys +pub mod keys { + use std::{collections::HashMap, sync::LazyLock}; + + use sea_orm::{ + ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, + QueryFilter, + }; + use sea_query::OnConflict; + use serde::Serialize; + use tracing::warn; + + use super::{Column, Entity}; + use crate::host; + pub type Result = std::result::Result; + + /// Override key to mark host as blocking health checks + pub const KEY_BAD_HOST: &str = "BAD_HOST"; + /// Expected value for a true value + pub const VAL_BOOL_TRUE: &str = "true"; + /// Override key for host health path to check + pub const KEY_HOST_HEALTH_PATH: &str = "HEALTH_PATH"; + /// Override key for host health query to check + pub const KEY_HOST_HEALTH_QUERY: &str = "HEALTH_QUERY"; + /// Override key for host bearer token + pub const KEY_HOST_BEARER: &str = "BEARER_TOKEN"; + pub const LOCKED_TRUE: i32 = 1; + pub const LOCKED_FALSE: i32 = 0; + /// Overrides that aren't locked by default + pub static NOT_LOCKED_DEFAULTS: &'static [&'static str] = + &[KEY_HOST_HEALTH_PATH, KEY_HOST_BEARER, KEY_HOST_HEALTH_QUERY]; + + pub static ALL_OVERRIDES: &'static [&'static str] = + &[KEY_BAD_HOST, KEY_HOST_BEARER, KEY_HOST_HEALTH_PATH, KEY_HOST_HEALTH_QUERY]; + + #[derive(Serialize, Clone, Copy)] + pub enum ValueType { + String, + Bool, + } + + static KEY_TYPES: LazyLock> = LazyLock::new(|| { + HashMap::from([ + (KEY_BAD_HOST, ValueType::Bool), + (KEY_HOST_HEALTH_PATH, ValueType::String), + (KEY_HOST_HEALTH_QUERY, ValueType::String), + (KEY_HOST_BEARER, ValueType::String), + ]) + }); + + #[derive(Serialize)] + pub struct Override { + pub value: Option, + pub locked: bool, + pub value_type: ValueType, + } + + impl Override { + fn new(value_type: ValueType, key: &&str) -> Self { + Self { + value: Default::default(), + locked: !NOT_LOCKED_DEFAULTS.contains(key), + value_type, + } + } + } + /// Host overrides for a single host + pub struct HostOverrides { + host: i32, + /// Key -> Override + entries: HashMap, + } + impl HostOverrides { + /// Load HostOverrides for a given domain - will be empty if not found + pub async fn load_by_domain(domain: &str, db: &DatabaseConnection) -> Result { + let host = host::Entity::find() + .filter(host::Column::Domain.eq(domain)) + .one(db) + .await?; + match host { + None => Ok(Self { + host: -1, + entries: HashMap::new(), + }), + Some(host) => Self::load(&host, db).await, + } + } + + pub async fn load(host: &host::Model, db: &DatabaseConnection) -> Result { + let mut overrides: HashMap = KEY_TYPES + .iter() + .map(|(key, value_type)| (key.to_string(), Override::new(*value_type, key))) + .collect(); + + let entries = Entity::find() + .filter(Column::Host.eq(host.id)) + .all(db) + .await?; + + entries.into_iter().for_each(|fetched_entry| { + if let Some(entry) = overrides.get_mut(&fetched_entry.key) { + entry.value = fetched_entry.value; + entry.locked = fetched_entry.locked == LOCKED_TRUE; + } else { + warn!( + host = host.id, + key = fetched_entry.key, + "Found unknown override key!" + ); + } + }); + Ok(Self { + host: host.id, + entries: overrides, + }) + } + + /// Retrieve [KEY_HOST_BEARER] + pub fn bearer(&self) -> Option<&str> { + self.get(KEY_HOST_BEARER) + } + + /// Retrieve [KEY_HOST_HEALTH_PATH] + pub fn health_path(&self) -> Option<&str> { + self.get(KEY_HOST_HEALTH_PATH) + } + + /// Retrieve [KEY_HOST_HEALTH_QUERY] + pub fn health_query(&self) -> Option<&str> { + self.get(KEY_HOST_HEALTH_QUERY) + } + + /// Retrieve override value for key + pub fn get(&self, key: &str) -> Option<&str> { + self.entries + .get(key) + .and_then(|v| v.value.as_deref()) + } + + pub fn entries(&self) -> &HashMap { + &self.entries + } + + pub fn host(&self) -> i32 { + self.host + } + + /// Returns whether an entry is locked, will be true if not found + pub async fn is_locked(&self, key: &str) -> bool { + self.entries.get(key).map(|v| v.locked).unwrap_or(true) + } + + pub async fn update_value( + &mut self, + key: String, + value: Option, + db: &DatabaseConnection, + ) -> Result<()> { + match self.entries.get_mut(&key) { + Some(entry) => { + if entry.value == value { + return Ok(()); + } + entry.value = value; + let locked_val = match entry.locked { + true => LOCKED_TRUE, + false => LOCKED_FALSE, + }; + super::Entity::insert(super::ActiveModel { + host: ActiveValue::Set(self.host), + key: ActiveValue::Set(key), + locked: ActiveValue::Set(locked_val), + value: ActiveValue::Set(entry.value.clone()), + }) + .on_conflict( + dbg!(OnConflict::columns([Column::Key, Column::Host])) + .update_columns([Column::Locked, Column::Value]) + .to_owned(), + ) + .exec(db) + .await?; + Ok(()) + } + None => todo!(), + } + } + } +} + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] +#[sea_orm(table_name = "host_overrides")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub host: i32, + #[sea_orm(primary_key)] + pub key: String, + pub locked: i32, + pub value: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::host::Entity", + from = "Column::Host", + to = "super::host::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Host, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Host.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/entities/src/log.rs b/entities/src/log.rs new file mode 100644 index 0000000..7c6892f --- /dev/null +++ b/entities/src/log.rs @@ -0,0 +1,38 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 + +use sea_orm::entity::prelude::*; +use serde::Serialize; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] +#[sea_orm(table_name = "log")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub user_host: i32, + pub host_affected: Option, + pub key: String, + #[sea_orm(primary_key, auto_increment = false)] + pub time: i64, + pub new_value: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::host::Entity", + from = "Column::UserHost", + to = "super::host::Column::Id", + on_update = "Cascade", + on_delete = "Restrict" + )] + Host2, + #[sea_orm( + belongs_to = "super::host::Entity", + from = "Column::HostAffected", + to = "super::host::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Host1, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/entities/src/mod.rs b/entities/src/mod.rs index 7ceaffe..7bdd915 100644 --- a/entities/src/mod.rs +++ b/entities/src/mod.rs @@ -5,7 +5,9 @@ pub mod prelude; pub mod check_errors; pub mod health_check; pub mod host; +pub mod host_overrides; pub mod instance_stats; +pub mod log; // has to be re-added on entity regeneration pub mod state; \ No newline at end of file diff --git a/entities/src/prelude.rs b/entities/src/prelude.rs index 4004557..7a1a484 100644 --- a/entities/src/prelude.rs +++ b/entities/src/prelude.rs @@ -3,4 +3,6 @@ pub use super::check_errors::Entity as CheckErrors; pub use super::health_check::Entity as HealthCheck; pub use super::host::Entity as Host; +pub use super::host_overrides::Entity as HostOverrides; pub use super::instance_stats::Entity as InstanceStats; +pub use super::log::Entity as Log; diff --git a/entities/src/state/scanner.rs b/entities/src/state/scanner.rs index 9dce4e5..85c9e25 100644 --- a/entities/src/state/scanner.rs +++ b/entities/src/state/scanner.rs @@ -37,8 +37,6 @@ pub struct Config { pub source_git_url: String, /// Git branch to fetch the current commit from pub source_git_branch: String, - /// Hosts known to be bad (ip block), domain per entry - pub bad_hosts: Vec, /// Interval to run cleanup operations in, to remove old data pub cleanup_interval: Duration, /// Amount of latest errors to keep per instance/host @@ -67,7 +65,6 @@ impl Config { auto_mute: true, source_git_branch: String::from("master"), source_git_url: String::from("https://github.com/zedeus/nitter.git"), - bad_hosts: vec![String::from("tweet.whateveritworks.org")], cleanup_interval: Duration::from_secs(24 * 60 * 60), error_retention_per_host: 100, connectivity_path: String::from("/"), diff --git a/migration/src/lib.rs b/migration/src/lib.rs index f14722c..f68fdb6 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -8,6 +8,7 @@ mod m20230829_201916_country; mod m20230914_231514_connectivity; mod m20231011_231223_errors; mod m20231112_142206_stats; +mod m20250216_185501_overrides; pub struct Migrator; @@ -23,6 +24,7 @@ impl MigratorTrait for Migrator { Box::new(m20230914_231514_connectivity::Migration), Box::new(m20231011_231223_errors::Migration), Box::new(m20231112_142206_stats::Migration), + Box::new(m20250216_185501_overrides::Migration), ] } } diff --git a/migration/src/m20250216_185501_overrides.rs b/migration/src/m20250216_185501_overrides.rs new file mode 100644 index 0000000..5eeb2b9 --- /dev/null +++ b/migration/src/m20250216_185501_overrides.rs @@ -0,0 +1,47 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cmd_host_overrides = r#"CREATE TABLE "host_overrides" ( + "host" integer NOT NULL, + "key" text NOT NULL, + "locked" integer NOT NULL, + "value" text, + CONSTRAINT "pk_override_hostkey" PRIMARY KEY ("host", "key"), + FOREIGN KEY ("host") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE + ) STRICT;"#; + let cmd_overrides_index = r#"CREATE INDEX "index_overrides_key" ON host_overrides ("key");"#; + let cmd_log = r#"CREATE TABLE "log" ( + "user_host" integer NOT NULL, + "host_affected" integer, + "key" text NOT NULL, + "time" integer NOT NULL, + "new_value" text, + CONSTRAINT "pk_log_usertime" PRIMARY KEY ("user_host", "time"), + FOREIGN KEY ("host_affected") REFERENCES "host" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY ("user_host") REFERENCES "host" ("id") ON DELETE RESTRICT ON UPDATE CASCADE + ) STRICT;"#; + let cmd_log_index = r#"CREATE INDEX "index_log_time" ON log ("time");"#; + let db = manager.get_connection(); + db.execute_unprepared("BEGIN EXCLUSIVE").await?; + tracing::info!("adding host overrides table.."); + db.execute_unprepared(cmd_host_overrides).await?; + tracing::info!("adding host overrides index.."); + db.execute_unprepared(cmd_overrides_index).await?; + tracing::info!("adding log table.."); + db.execute_unprepared(cmd_log).await?; + tracing::info!("adding log index.."); + db.execute_unprepared(cmd_log_index).await?; + db.execute_unprepared("COMMIT TRANSACTION").await?; + db.execute_unprepared("VACUUM").await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + panic!("Can't migrate down"); + } +} diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index 0af1620..b9d24f5 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -1,14 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-only use std::cmp; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use chrono::{Days, Utc}; use chrono::{Duration, TimeZone}; -use entities::host; +use entities::host_overrides::keys::{KEY_BAD_HOST, VAL_BOOL_TRUE}; +use entities::{host, host_overrides}; use entities::prelude::*; use entities::state::CacheData; use entities::state::CacheHost; -use sea_orm::ColumnTrait; +use sea_orm::{ColumnTrait, QuerySelect, QueryTrait}; use sea_orm::EntityTrait; use sea_orm::QueryFilter; use sea_orm::QueryOrder; @@ -94,6 +95,8 @@ impl Scanner { let mut healthy_percentage_total = self.query_healthy_percentage().await?; + let bad_hosts = self.query_bad_hosts().await?; + let mut host_statistics = Vec::with_capacity(hosts.len()); let default_health_check = LatestCheck::default(); for host in hosts { @@ -118,11 +121,6 @@ impl Scanner { let points = stats_3h_host * points; let last_check = latest_check.get(&host.id).unwrap_or(&default_health_check); - // // don't rank currently down instances highly - // let points = match last_check.healthy { - // true => (points * 100.0) as i32, - // false => 0, - // }; let points = (points * 100.0) as i32; let latest_version = host @@ -134,7 +132,7 @@ impl Scanner { .as_ref() .map_or(false, |url| current_version.is_same_repo(&url)); - let is_bad_host = self.inner.config.bad_hosts.contains(&host.domain); + let is_bad_host = bad_hosts.contains(&host.id); let host_ping_data = ping_data.remove(&host.id); let last_healthy = last_healthy_check.remove(&host.id); @@ -193,6 +191,18 @@ impl Scanner { }) } + async fn query_bad_hosts(&self) -> Result> { + let res: Vec = HostOverrides::find() + .filter(host_overrides::Column::Key.eq(KEY_BAD_HOST) + .and(host_overrides::Column::Value.eq(VAL_BOOL_TRUE))) + .select_only() + .column(host_overrides::Column::Host) + .into_tuple() + .all(&self.inner.db) + .await?; + Ok(res) + } + async fn query_hosts_enabled(&self) -> Result> { Ok(Host::find() .filter(host::Column::Enabled.eq(true)) diff --git a/scanner/src/cleanup.rs b/scanner/src/cleanup.rs index bedf664..6799581 100644 --- a/scanner/src/cleanup.rs +++ b/scanner/src/cleanup.rs @@ -58,11 +58,13 @@ impl Scanner { .exec(&self.inner.db) .await? }; - tracing::debug!( - host = host.id, - host.enabled, - deleted_errors = res.rows_affected - ); + if res.rows_affected > 0 { + tracing::debug!( + host = host.id, + host.enabled, + deleted_errors = res.rows_affected + ); + } } Ok(()) diff --git a/scanner/src/instance_check.rs b/scanner/src/instance_check.rs index 866ab0a..d9b3549 100644 --- a/scanner/src/instance_check.rs +++ b/scanner/src/instance_check.rs @@ -4,8 +4,9 @@ use std::time::Instant; use chrono::Utc; use entities::state::error_cache::HostError; -use entities::{check_errors, health_check}; +use entities::{check_errors, health_check, host_overrides}; use entities::{host, prelude::*}; +use entities::host_overrides::keys::HostOverrides; use reqwest::Url; use sea_orm::prelude::DateTimeUtc; use sea_orm::ColumnTrait; @@ -39,8 +40,9 @@ impl Scanner { .find(|v| v.host == model.id) .map_or(false, |check| !check.healthy); join_set.spawn(async move { - scanner.health_check_host(model, muted_host).await; - + if let Err(e) = scanner.health_check_host(model, muted_host).await { + + } }); } // wait till all of them are finished, preventing DoS @@ -54,7 +56,7 @@ impl Scanner { } #[instrument] - async fn health_check_host(&self, host: host::Model, muted: bool) { + async fn health_check_host(&self, host: host::Model, muted: bool) -> Result<()> { let now = Utc::now(); let mut url = match Url::parse(&host.url) { Err(e) => { @@ -68,13 +70,14 @@ impl Scanner { None, ) .await; - return; + return Ok(()); } Ok(v) => v, }; url.set_path(&self.inner.config.profile_path); + let overrides = HostOverrides::load(&host, &self.inner.db).await?; let start = Instant::now(); - let fetch_res = self.fetch_url(url.as_str()).await; + let fetch_res = self.fetch_url(url.as_str(), overrides.bearer()).await; let end = Instant::now(); let took_ms = end.saturating_duration_since(start).as_millis(); match fetch_res { @@ -156,12 +159,13 @@ impl Scanner { } } } + Ok(()) } /// Check if rss is available - pub(crate) async fn has_rss(&self, url: &mut Url, mute: bool) -> bool { + pub(crate) async fn has_rss(&self, url: &mut Url, api_token: Option<&str>, mute: bool) -> bool { url.set_path(&self.inner.config.rss_path); - match self.fetch_url(url.as_str()).await { + match self.fetch_url(url.as_str(), api_token).await { Ok((code, content)) => match self.inner.rss_check_regex.is_match(&content) { true => return true, false => { @@ -187,9 +191,9 @@ impl Scanner { } /// Check nitter version - pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option { + pub(crate) async fn nitter_version(&self, url: &mut Url, api_token: Option<&str>, mute: bool) -> Option { url.set_path(&self.inner.config.about_path); - match self.fetch_url(url.as_str()).await { + match self.fetch_url(url.as_str(), api_token).await { Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) { Ok(v) => Some(v), Err(e) => { diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index ed6cb17..83d926e 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -7,8 +7,7 @@ use std::{ use about_parser::AboutParser; use chrono::{DateTime, Duration, TimeZone, Utc}; use entities::{ - health_check, - state::{error_cache::HostError, scanner::ScannerConfig, AppState}, + health_check, host, state::{error_cache::HostError, scanner::ScannerConfig, AppState} }; use instance_parser::InstanceParser; use miette::{Context, IntoDiagnostic}; @@ -16,7 +15,7 @@ use profile_parser::ProfileParser; use regex::{Regex, RegexBuilder}; use reqwest::{ header::{HeaderMap, HeaderValue}, - Client, ClientBuilder, + Client, ClientBuilder, Url, }; use sea_orm::{ ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, @@ -364,12 +363,16 @@ impl Scanner { } async fn fetch_instance_list(&self) -> Result { - let (_, body) = self.fetch_url(&self.inner.config.instance_list_url).await?; + let (_, body) = self.fetch_url(&self.inner.config.instance_list_url, None).await?; Ok(body) } - async fn fetch_url(&self, url: &str) -> std::result::Result<(u16, String), FetchError> { - let fetch_res = self.inner.client.get(url).send().await?; + async fn fetch_url(&self, url: &str, api_token: Option<&str>) -> std::result::Result<(u16, String), FetchError> { + let mut request = self.inner.client.get(url); + if let Some(token) = api_token { + request = request.bearer_auth(token); + } + let fetch_res = request.send().await?; let code = fetch_res.status().as_u16(); if !fetch_res.status().is_success() { let message = fetch_res @@ -478,7 +481,7 @@ mod test { let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) .await .unwrap(); - let (_, res) = scanner.fetch_url("example.com/jack").await.unwrap(); + let (_, res) = scanner.fetch_url("example.com/jack",None).await.unwrap(); let mut file = File::create("test_data/blocked.html").await.unwrap(); file.write_all(&res.as_bytes()).await.unwrap(); } diff --git a/scanner/src/list_update.rs b/scanner/src/list_update.rs index c387aa3..f0cf708 100644 --- a/scanner/src/list_update.rs +++ b/scanner/src/list_update.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only //! Updates the list of available instances, fetching all required fields +use std::collections::HashMap; use std::time::Duration; use std::time::Instant; use chrono::Utc; use entities::host; +use entities::host_overrides::keys::HostOverrides; use entities::prelude::Host; use reqwest::Url; use sea_orm::{ @@ -55,7 +57,9 @@ impl Scanner { removed += 1; } } + // now update/insert the existing ones + let found_instances: usize = parsed_instances.len(); // find last update checks to detect spam let last_status = self.query_latest_check(&transaction).await?; @@ -65,6 +69,8 @@ impl Scanner { let mut join_set = JoinSet::new(); for (_, instance) in parsed_instances { + let overrides = HostOverrides::load_by_domain(&instance.domain, &self.inner.db).await?; + // TODO: parallelize this! let scanner_c = self.clone(); // detect already offline host and prevent log spam @@ -88,9 +94,14 @@ impl Scanner { let connectivity = scanner_c.check_connectivity(&mut url).await; // prevent DoS tokio::time::sleep(Duration::from_secs(1)).await; - let rss = scanner_c.has_rss(&mut url, muted_host).await; + let rss = scanner_c + .has_rss(&mut url, overrides.bearer(), muted_host) + .await; tokio::time::sleep(Duration::from_secs(1)).await; - match scanner_c.nitter_version(&mut url, muted_host).await { + match scanner_c + .nitter_version(&mut url, overrides.bearer(), muted_host) + .await + { Some(version) => ( connectivity, rss, @@ -124,22 +135,22 @@ impl Scanner { // insert all at once let transaction = self.inner.db.begin().await?; Host::insert_many(update_models) - .on_conflict( - OnConflict::column(host::Column::Domain) - .update_columns([ - host::Column::Enabled, - host::Column::Updated, - host::Column::Url, - host::Column::Rss, - host::Column::Version, - host::Column::VersionUrl, - host::Column::Country, - host::Column::Connectivity, - ]) - .to_owned(), - ) - .exec(&transaction) - .await?; + .on_conflict( + OnConflict::column(host::Column::Domain) + .update_columns([ + host::Column::Enabled, + host::Column::Updated, + host::Column::Url, + host::Column::Rss, + host::Column::Version, + host::Column::VersionUrl, + host::Column::Country, + host::Column::Connectivity, + ]) + .to_owned(), + ) + .exec(&transaction) + .await?; transaction.commit().await?; let end = Instant::now(); diff --git a/scanner/src/update_stats.rs b/scanner/src/update_stats.rs index 4901dae..aed03f0 100644 --- a/scanner/src/update_stats.rs +++ b/scanner/src/update_stats.rs @@ -5,11 +5,12 @@ use std::time::Instant; use chrono::Utc; -use entities::{host, instance_stats}; +use entities::host_overrides::keys::HostOverrides; use entities::prelude::Host; +use entities::{host, instance_stats}; use reqwest::Url; use sea_orm::prelude::DateTimeUtc; -use sea_orm::{QueryFilter, ColumnTrait, EntityTrait, ActiveValue}; +use sea_orm::{ActiveValue, ColumnTrait, EntityTrait, QueryFilter}; use serde::Deserialize; use tokio::task::JoinSet; @@ -79,10 +80,11 @@ impl Scanner { stat_data.push(data); } } - tracing::debug!(db_stats_entries=stat_data.len()); + tracing::debug!(db_stats_entries = stat_data.len()); if !stat_data.is_empty() { instance_stats::Entity::insert_many(stat_data) - .exec(&self.inner.db).await?; + .exec(&self.inner.db) + .await?; } let end = Instant::now(); @@ -94,15 +96,24 @@ impl Scanner { Ok(()) } - async fn fetch_instance_stats(&self, time: DateTimeUtc, host: &host::Model) -> Result { - let mut url = Url::parse(&host.url) - .map_err(|e|ScannerError::InstanceUrlParse)?; + async fn fetch_instance_stats( + &self, + time: DateTimeUtc, + host: &host::Model, + ) -> Result { + let overrides = HostOverrides::load(&host, &self.inner.db).await?; + let mut url = Url::parse(&host.url).map_err(|e| ScannerError::InstanceUrlParse)?; url.set_path(".health"); - let (_code, body) = self.fetch_url(url.as_str()).await?; + if let Some(url_override) = overrides.health_path() { + url.set_path(url_override); + } + if let Some(path_override) = overrides.health_query() { + url.set_query(Some(path_override)); + } + let (_code, body) = self.fetch_url(url.as_str(), overrides.bearer()).await?; - let stats_data: InstanceStats = - serde_json::from_str(&body) - .map_err(|e|ScannerError::StatsParsing(e,body))?; + let stats_data: InstanceStats = + serde_json::from_str(&body).map_err(|e| ScannerError::StatsParsing(e, body))?; let stats_model = instance_stats::ActiveModel { time: ActiveValue::Set(time.timestamp()), @@ -118,9 +129,11 @@ impl Scanner { req_tweet_detail: ActiveValue::Set(stats_data.requests.apis.tweetDetail), req_list: ActiveValue::Set(stats_data.requests.apis.list), req_user_tweets: ActiveValue::Set(stats_data.requests.apis.userTweets), - req_user_tweets_and_replies: ActiveValue::Set(stats_data.requests.apis.userTweetsAndReplies), + req_user_tweets_and_replies: ActiveValue::Set( + stats_data.requests.apis.userTweetsAndReplies, + ), }; Ok(stats_model) } -} \ No newline at end of file +} diff --git a/server/src/admin/errors.rs b/server/src/admin/errors.rs new file mode 100644 index 0000000..6649c65 --- /dev/null +++ b/server/src/admin/errors.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; + +use crate::Result; +use axum::{ + extract::{Path, State}, + response::{Html, IntoResponse}, +}; +use entities::{check_errors, state::AppState}; +use sea_orm::ColumnTrait; +use sea_orm::DatabaseConnection; +use sea_orm::EntityTrait; +use sea_orm::QueryFilter; +use sea_orm::QueryOrder; +use sea_orm::QuerySelect; +use tower_sessions::Session; + +use crate::{admin::get_specific_login_host, ServerError}; + +pub async fn errors_view( + State(ref app_state): State, + State(ref template): State>, + State(ref db): State, + Path(instance): Path, + session: Session, +) -> Result { + tracing::info!(?session); + + let (host, _login) = get_specific_login_host(instance, &session, db).await?; + + let errors = check_errors::Entity::find() + .filter(check_errors::Column::Host.eq(host.id)) + .order_by_desc(check_errors::Column::Time) + .limit(20) + .all(db) + .await?; + + let mut context = tera::Context::new(); + let res = { + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; + let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); + context.insert("last_updated", &time); + context.insert("ERRORS", &errors); + context.insert("HOST_DOMAIN", &host.domain); + context.insert("HOST_ID", &instance); + + let res = Html(template.render("instance_errors.html.j2", &context)?).into_response(); + drop(guard); + res + }; + Ok(res) +} diff --git a/server/src/admin/locks.rs b/server/src/admin/locks.rs new file mode 100644 index 0000000..a104060 --- /dev/null +++ b/server/src/admin/locks.rs @@ -0,0 +1,84 @@ +use std::{collections::HashMap, sync::Arc}; + +use axum::{ + extract::{Path, State}, + response::{Html, IntoResponse, Redirect}, + Form, +}; +use entities::host_overrides::{ + self, + keys::{HostOverrides, LOCKED_FALSE, LOCKED_TRUE}, +}; +use sea_orm::{sea_query::OnConflict, ActiveValue, DatabaseConnection, EntityTrait}; +use tower_sessions::Session; + +use crate::{Result, ServerError}; + +use super::get_specific_login_host; + +pub async fn locks_view( + State(ref template): State>, + State(ref db): State, + Path(instance): Path, + session: Session, +) -> Result { + let (host, login) = get_specific_login_host(instance, &session, db).await?; + + if !login.admin { + return Err(ServerError::MissingPermission); + } + + let overrides = HostOverrides::load(&host, db).await?; + + let mut context = tera::Context::new(); + context.insert("HOST_DOMAIN", &host.domain); + context.insert("HOST_ID", &instance); + context.insert("OVERRIDES", overrides.entries()); + + let res = Html(template.render("instance_locks.html.j2", &context)?).into_response(); + Ok(res) +} + +/// Override Form +// #[derive(Deserialize, Debug)] +pub type LocksFormInput = HashMap; + +pub async fn post_locks( + State(db): State, + Path(instance): Path, + session: Session, + Form(input): Form, +) -> Result { + let (host, login) = get_specific_login_host(instance, &session, &db).await?; + + if !login.admin { + return Err(ServerError::MissingPermission); + } + + let overrides = HostOverrides::load(&host, &db).await?; + + let mut updated = Vec::with_capacity(input.len()); + + for (key, _value) in overrides.entries() { + let locked = match input.contains_key(key) { + true => LOCKED_TRUE, + false => LOCKED_FALSE, + }; + updated.push(host_overrides::ActiveModel { + host: ActiveValue::Set(host.id), + key: ActiveValue::Set(key.to_string()), + locked: ActiveValue::Set(locked), + value: ActiveValue::Set(None), + }); + } + host_overrides::Entity::insert_many(updated) + .on_conflict( + OnConflict::columns([host_overrides::Column::Host, host_overrides::Column::Key]) + .update_columns([host_overrides::Column::Locked]) + .to_owned(), + ) + .exec(&db) + .await?; + + Ok(Redirect::to(&format!("/admin/instance/locks/{}", instance)).into_response()) +} diff --git a/server/src/admin/logs.rs b/server/src/admin/logs.rs new file mode 100644 index 0000000..0ddd8d5 --- /dev/null +++ b/server/src/admin/logs.rs @@ -0,0 +1,105 @@ +use std::{ + collections::HashMap, + sync::Arc, +}; + +use crate::{Result, ServerError}; +use axum::{ + extract::State, + response::{Html, IntoResponse}, +}; +use chrono::{TimeZone, Utc}; +use entities::{ + host, + log, +}; +use sea_orm::EntityTrait; +use sea_orm::{DatabaseConnection, QueryOrder, QuerySelect}; +use serde::Serialize; +use tower_sessions::Session; + +use super::get_session_login; + +#[derive(Serialize)] +struct LogEntry { + time: String, + by_user_host: String, + key: String, + for_host: Option, + value: Option, +} +#[derive(Serialize)] +struct ForHost { + domain: String, + id: i32, +} + +pub async fn log_view( + State(ref template): State>, + State(ref db): State, + session: Session, +) -> Result { + let login = get_session_login(&session)?; + if !login.admin { + return Err(ServerError::MissingPermission); + } + + let mut host_cache = HostCache::default(); + + let logs_raw = log::Entity::find() + .order_by_desc(log::Column::Time) + .all(db) + .await?; + let mut logs = Vec::with_capacity(logs_raw.len()); + + for entry in logs_raw { + let by_host = host_cache.get(entry.user_host, db).await?; + let for_host = match entry.host_affected { + None => None, + Some(v) => Some(ForHost { + domain: host_cache.get(v, db).await?, + id: v, + }), + }; + let time = Utc + .timestamp_opt(entry.time, 0) + .unwrap() + .format("%Y-%m-%d %H:%M:%S") + .to_string(); + logs.push(LogEntry { + by_user_host: by_host, + for_host, + key: entry.key, + time, + value: entry.new_value, + }); + } + + let mut context = tera::Context::new(); + context.insert("LOGS", &logs); + + let res = Html(template.render("admin_logs.html.j2", &context)?).into_response(); + Ok(res) +} + +#[derive(Default)] +struct HostCache(HashMap); + +impl HostCache { + async fn get(&mut self, host_id: i32, db: &DatabaseConnection) -> Result { + if let Some(v) = self.0.get(&host_id) { + return Ok(v.clone()); + } + let domain: Option = host::Entity::find_by_id(host_id) + .select_only() + .column(host::Column::Domain) + .into_tuple() + .one(db) + .await?; + let Some(domain) = domain else { + return Err(ServerError::HostNotFound(host_id)); + }; + self.0.insert(host_id, domain.clone()); + Ok(domain) + } +} diff --git a/server/src/admin.rs b/server/src/admin/mod.rs similarity index 86% rename from server/src/admin.rs rename to server/src/admin/mod.rs index 44e8c75..a9739d0 100644 --- a/server/src/admin.rs +++ b/server/src/admin/mod.rs @@ -11,7 +11,6 @@ use axum::Json; use chrono::DateTime; use chrono::Utc; use constant_time_eq::constant_time_eq; -use entities::check_errors; use entities::health_check; use entities::host; use entities::instance_stats; @@ -26,12 +25,12 @@ use sea_orm::DatabaseConnection; use sea_orm::EntityTrait; use sea_orm::QueryFilter; use sea_orm::QueryOrder; -use sea_orm::QuerySelect; use serde::Deserialize; use serde::Serialize; use sha2::{Digest, Sha256}; use thiserror::Error; use tower_sessions::Session; +use tracing::trace; use trust_dns_resolver::config::ResolverConfig; use trust_dns_resolver::config::ResolverOpts; use trust_dns_resolver::AsyncResolver; @@ -42,6 +41,18 @@ use crate::ServerError; use crate::ADMIN_OVERVIEW_URL; use crate::LOGIN_URL; +mod errors; +mod settings; +mod locks; +mod logs; + +pub use errors::errors_view; +pub use settings::post_settings; +pub use settings::settings_view; +pub use locks::locks_view; +pub use locks::post_locks; +pub use logs::log_view; + /// Stored session login information #[derive(Serialize, Deserialize, Default)] pub struct ActiveLogin { @@ -107,7 +118,11 @@ pub async fn login( .unwrap_or_default() { // already logged in - return Ok(Redirect::temporary(&input.referrer).into_response()); + let url = match input.referrer.trim().is_empty() || input.referrer.trim() == LOGIN_URL { + true => ADMIN_OVERVIEW_URL, + false => &input.referrer, + }; + return Ok(Redirect::to(url).into_response()); } match login_inner(config, login_client, &input, host).await { @@ -128,11 +143,11 @@ pub async fn login( }; session.insert(LOGIN_KEY, session_value)?; let referrer = input.referrer.trim(); - let location = match referrer.trim().is_empty() && referrer != LOGIN_URL { + let location = match referrer.is_empty() || referrer == LOGIN_URL { true => ADMIN_OVERVIEW_URL, false => input.referrer.trim(), }; - let mut res = Redirect::temporary(location).into_response(); + let mut res = Redirect::to(location).into_response(); *res.status_mut() = StatusCode::FOUND; Ok(res) } @@ -166,6 +181,15 @@ async fn login_inner( ) -> LoginResult { let host = host.ok_or_else(|| LoginError::HostNotFound(input.domain.clone()))?; + // check for configured host keys - skip enabled hosts on purpose + if let Some(hash) = config.admin_keys.get(&host.domain) { + trace!(host = host.domain, "found admin key override"); + if verify_key(hash.as_str(), &input.key).is_ok() { + return Ok(host); + } + } + // fallback to normal auth + if !host.enabled { return Err(LoginError::DisabledHost(input.domain.clone())); } @@ -291,7 +315,7 @@ pub async fn overview( ) -> Result { tracing::info!(?session); - let (login, hosts) = get_all_login_hosts(&session, db,true).await?; + let (login, hosts) = get_all_login_hosts(&session, db, true).await?; let mut context = tera::Context::new(); let res = { @@ -323,16 +347,20 @@ pub async fn history_json( pub stats: Vec, } let (_login, hosts) = get_all_login_hosts(&session, db, false).await?; - let host_ids: Vec<_> = hosts.iter().map(|host|host.id).collect(); - let data_owned = health_check::HealthyAmount::fetch(db, Some(input.start), Some(input.end),Some(&host_ids)).await?; - let data_global = health_check::HealthyAmount::fetch(db, Some(input.start), Some(input.end),None).await?; - let data_stats = instance_stats::StatsAmount::fetch(db, input.start, input.end,None).await?; + let host_ids: Vec<_> = hosts.iter().map(|host| host.id).collect(); + let data_owned = + health_check::HealthyAmount::fetch(db, Some(input.start), Some(input.end), Some(&host_ids)) + .await?; + let data_global = + health_check::HealthyAmount::fetch(db, Some(input.start), Some(input.end), None).await?; + let data_stats = instance_stats::StatsAmount::fetch(db, input.start, input.end, None).await?; Ok(Json(ReturnData { global: data_global, user: data_owned, stats: data_stats, - }).into_response()) + }) + .into_response()) } pub async fn history_json_specific( @@ -341,7 +369,7 @@ pub async fn history_json_specific( Path(host): Path, Json(input): Json, ) -> Result { - let host = get_specific_login_host(host, &session, db).await?; + let (host, _login) = get_specific_login_host(host, &session, db).await?; #[derive(Debug, Serialize)] struct ReturnData { pub stats: Vec, @@ -357,14 +385,17 @@ pub async fn history_json_specific( let history_stats: Vec = instance_stats::Entity::find() .filter(instance_stats::Column::Host.eq(host.id)) .order_by_asc(instance_stats::Column::Time) - .filter(instance_stats::Column::Time.between(input.start.timestamp(), input.end.timestamp())) + .filter( + instance_stats::Column::Time.between(input.start.timestamp(), input.end.timestamp()), + ) .all(db) .await?; - Ok(Json(ReturnData{ + Ok(Json(ReturnData { health: history_health, - stats: history_stats - }).into_response()) + stats: history_stats, + }) + .into_response()) } pub async fn history_view( @@ -376,7 +407,7 @@ pub async fn history_view( ) -> Result { tracing::info!(?session); - let host = get_specific_login_host(host, &session, db).await?; + let (host, _login) = get_specific_login_host(host, &session, db).await?; let mut context = tera::Context::new(); let res = { @@ -395,43 +426,6 @@ pub async fn history_view( Ok(res) } -pub async fn instance_view( - State(ref app_state): State, - State(ref template): State>, - State(ref db): State, - Path(instance): Path, - session: Session, -) -> Result { - tracing::info!(?session); - - let host = get_specific_login_host(instance, &session, db).await?; - - let errors = check_errors::Entity::find() - .filter(check_errors::Column::Host.eq(host.id)) - .order_by_desc(check_errors::Column::Time) - .limit(20) - .all(db) - .await?; - - let mut context = tera::Context::new(); - let res = { - let guard = app_state - .cache - .read() - .map_err(|_| ServerError::MutexFailure)?; - let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); - context.insert("last_updated", &time); - context.insert("ERRORS", &errors); - context.insert("HOST_DOMAIN", &host.domain); - context.insert("HOST_ID", &instance); - - let res = Html(template.render("instance_admin.html.j2", &context)?).into_response(); - drop(guard); - res - }; - Ok(res) -} - /// Get all [host::Model] for current [Session], optionally return all hosts for admins async fn get_all_login_hosts( session: &Session, @@ -462,7 +456,7 @@ async fn get_specific_login_host( wanted_host_id: i32, session: &Session, db: &DatabaseConnection, -) -> Result { +) -> Result<(host::Model, ActiveLogin)> { let login = get_session_login(&session)?; if !login.hosts.contains(&wanted_host_id) && !login.admin { @@ -478,7 +472,7 @@ async fn get_specific_login_host( session.delete(); return Err(ServerError::HostNotFound(wanted_host_id)); } - Some(host) => Ok(host), + Some(host) => Ok((host, login)), } } diff --git a/server/src/admin/settings.rs b/server/src/admin/settings.rs new file mode 100644 index 0000000..6a01ce5 --- /dev/null +++ b/server/src/admin/settings.rs @@ -0,0 +1,121 @@ +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + +use crate::{Result, ServerError}; +use axum::{ + extract::{Path, State}, + response::{Html, IntoResponse}, + Form, +}; +use chrono::Utc; +use entities::{ + host_overrides::{self, keys::*}, + log, +}; +use sea_orm::sea_query::OnConflict; +use sea_orm::ActiveModelTrait; +use sea_orm::EntityTrait; +use sea_orm::{ActiveValue, DatabaseConnection}; +use serde::Deserialize; +use tower_sessions::Session; +use tracing::trace; + +use super::get_specific_login_host; + +pub async fn settings_view( + State(ref template): State>, + State(ref db): State, + Path(instance): Path, + session: Session, +) -> Result { + let (host, login) = get_specific_login_host(instance, &session, db).await?; + + let overrides = HostOverrides::load(&host, db).await?; + + let mut context = tera::Context::new(); + context.insert("HOST_DOMAIN", &host.domain); + context.insert("HOST_ID", &instance); + context.insert("IS_ADMIN", &login.admin); + context.insert("OVERRIDES", overrides.entries()); + + let res = Html(template.render("instance_settings.html.j2", &context)?).into_response(); + Ok(res) +} + +/// Override Form +#[derive(Deserialize, Debug)] +pub struct OverrideFormInput { + /// Some if checked a checkbox, none otherwise + value: Option, + key: String, +} + +pub async fn post_settings( + State(template): State>, + State(db): State, + Path(instance): Path, + session: Session, + Form(input): Form, +) -> Result { + trace!(form=?input,host=instance,"post_override"); + let (host, login) = get_specific_login_host(instance, &session, &db).await?; + + let overrides = HostOverrides::load(&host, &db).await?; + let Some(entry) = overrides.entries().get(&input.key) else { + trace!("unknown override key"); + return Err(ServerError::InvalidOverrideKey); + }; + + if entry.locked && !login.admin { + trace!(locked = entry.locked, "missing permissions"); + return Err(ServerError::MissingPermission); + } + + let value = match (input.value.as_deref().map(|v| v.trim()), entry.value_type) { + (None, _) => None, + // don't insert empty strings + (Some(""), ValueType::String) => None, + (value, ValueType::String) => value, + (Some(VAL_BOOL_TRUE), ValueType::Bool) => Some("true"), + // don't allow arbitrary data + (Some(_), ValueType::Bool) => Some("false"), + } + .map(|v| v.to_owned()); + + // only for first-time inserts, not updated! + let locked_value = ActiveValue::Set(match entry.locked { + true => LOCKED_TRUE, + false => LOCKED_FALSE, + }); + + let time = Utc::now().timestamp(); + let user_host = login.hosts.iter().next().unwrap(); + log::ActiveModel { + user_host: ActiveValue::Set(*user_host), + host_affected: ActiveValue::Set(Some(host.id)), + key: ActiveValue::Set(input.key.clone()), + time: ActiveValue::Set(time), + new_value: ActiveValue::Set(value.clone()), + } + .insert(&db) + .await?; + + let model = host_overrides::ActiveModel { + host: ActiveValue::Set(host.id), + key: ActiveValue::Set(input.key), + locked: locked_value, + value: ActiveValue::Set(value), + }; + host_overrides::Entity::insert(model) + .on_conflict( + OnConflict::columns([host_overrides::Column::Host, host_overrides::Column::Key]) + .update_columns([host_overrides::Column::Value]) + .to_owned(), + ) + .exec(&db) + .await?; + + settings_view(State(template), State(db), Path(instance), session).await +} diff --git a/server/src/lib.rs b/server/src/lib.rs index 5674006..060b246 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-only -use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; +use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc, time::SystemTimeError}; use axum::{ error_handling::HandleErrorLayer, @@ -40,7 +40,10 @@ pub struct Config { pub max_age: usize, pub session_ttl_seconds: u64, pub login_token_name: String, + /// Domains for admin hosts pub admin_domains: Vec, + /// Additional host keys to auth against + pub admin_keys: HashMap, pub session_db_uri: String, } @@ -149,12 +152,15 @@ pub async fn start( .route("/api/csv/stats", get(api::graph_csv_stats)) .nest(ADMIN_OVERVIEW_URL, Router::new() .route("/", get(admin::overview)) - .route("/instance/:instance", get(admin::instance_view)) + .route("/instance/errors/:instance", get(admin::errors_view)) + .route("/instance/settings/:instance", get(admin::settings_view).post(admin::post_settings)) + .route("/instance/locks/:instance", get(admin::locks_view).post(admin::post_locks)) .route("/api/history/:instance", post(admin::history_json_specific)) .route("/api/history", post(admin::history_json)) .route("/login", get(admin::login_view).post(admin::login).route_layer(rate_limit_layer)) + .route("/logs", get(admin::log_view)) .route("/logout", get(admin::logout)) - // .layer(ServiceBuilder::new().layer(SetResponseHeaderLayer::overriding(header::CACHE_CONTROL, "must-revalidate"))) + .layer(ServiceBuilder::new().layer(SetResponseHeaderLayer::overriding(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")))) .layer(session_service) ) .route("/about", get(website::about)) @@ -216,6 +222,10 @@ pub enum ServerError { MissingPermission, #[error("Failed to create CSV")] CSV(String), + #[error("Invalid override key")] + InvalidOverrideKey, + #[error("Failed getting system time: {0}")] + TimeError(#[from] SystemTimeError), } impl axum::response::IntoResponse for ServerError { @@ -223,16 +233,19 @@ impl axum::response::IntoResponse for ServerError { use ServerError::*; let msg = match &self { NoLogin => { - let mut resp = Redirect::temporary(LOGIN_URL).into_response(); // *resp.status_mut() = StatusCode::FOUND; // have to use a 301, [Redirect] 307 won't work for referrer - return resp; + return Redirect::temporary(LOGIN_URL).into_response(); } MissingPermission => ( StatusCode::FORBIDDEN, Cow::Borrowed("Missing permission to access this resource"), ), + InvalidOverrideKey => ( + StatusCode::BAD_REQUEST, + Cow::Borrowed("Invalid override key"), + ), CSV(_) | MutexFailure | Templating(_) | DBError(_) | SessionError(_) - | HostNotFound(_) => ( + | HostNotFound(_) | TimeError(_) => ( StatusCode::INTERNAL_SERVER_ERROR, Cow::Borrowed("Internal Server Error"), ), diff --git a/server/templates/admin.html.j2 b/server/templates/admin.html.j2 index 78c3adb..98a1b18 100644 --- a/server/templates/admin.html.j2 +++ b/server/templates/admin.html.j2 @@ -13,6 +13,7 @@

Admin Interface

Logout Add more instances

+ {% if is_admin %}

Settings Logs

{% endif %}

Instances

{% if is_admin %}

root mode

{% endif %} @@ -44,7 +45,7 @@ {{instance.domain}} {# History #} - Errors + Errors Settings {%- endfor %} diff --git a/server/templates/admin_logs.html.j2 b/server/templates/admin_logs.html.j2 new file mode 100644 index 0000000..e51a6b5 --- /dev/null +++ b/server/templates/admin_logs.html.j2 @@ -0,0 +1,44 @@ +{#- SPDX-License-Identifier: AGPL-3.0-only -#} + + + + + + + + + + Admin Logs + + + +
+

Override Logs

+

root mode

+

Home

+ + + + + + + + + + + + {% for entry in LOGS %} + + + + + + + + {% endfor %} + +
DateUserHostKeyNew Value
{{entry.time}}{{entry.by_user_host}}{{entry.for_host.domain}} {% if entry.for_host%}link{% endif %}{{entry.key}}{% if entry.value %}{{entry.value}}{% else %}null{% endif %}
+
+ + + \ No newline at end of file diff --git a/server/templates/instance_admin.html.j2 b/server/templates/instance_errors.html.j2 similarity index 100% rename from server/templates/instance_admin.html.j2 rename to server/templates/instance_errors.html.j2 diff --git a/server/templates/instance_locks.html.j2 b/server/templates/instance_locks.html.j2 new file mode 100644 index 0000000..9c85136 --- /dev/null +++ b/server/templates/instance_locks.html.j2 @@ -0,0 +1,38 @@ +{#- SPDX-License-Identifier: AGPL-3.0-only -#} + + + + + + + + + + Locks | {{HOST_DOMAIN}} + + + +
+

Override Locks | {{HOST_DOMAIN}}

+

root mode

+

Home

+
+ {% for key, value in OVERRIDES %} +
+
+ + +
+
+ {% endfor %} +
+ +
+
+
+ + + \ No newline at end of file diff --git a/server/templates/instance_settings.html.j2 b/server/templates/instance_settings.html.j2 new file mode 100644 index 0000000..ebfe56f --- /dev/null +++ b/server/templates/instance_settings.html.j2 @@ -0,0 +1,84 @@ +{#- SPDX-License-Identifier: AGPL-3.0-only -#} + + + + + + + + + + Settings | {{HOST_DOMAIN}} + + + +
+

Settings for {{HOST_DOMAIN}}

+

Home {% if IS_ADMIN %}Lockdown Settings{% endif %}

+ {% if IS_ADMIN %}

root mode

{% endif %} +
+ {%- set key = "BAD_HOST" -%} + {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} +
+
+ + +
+
+ +
+ +
+
+

Overrides

+

Leave settings blank to stop overriding them from their defaults.

+ +
+ {%- set key = "BEARER_TOKEN" -%} + {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} +
+ +
+ Authorization: Bearer + +
+
+ +
+ +
+
+
+ {%- set key = "HEALTH_PATH" -%} + {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} +
+ + +
+ +
+ +
+
+
+ {%- set key = "HEALTH_QUERY" -%} + {%- set disabled = OVERRIDES[key].locked and not IS_ADMIN -%} +
+ + +
+ +
+ +
+
+
+ + + \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index fbf64e3..f3127ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-only -use std::{env::var, time::Duration}; +use std::{collections::HashMap, env::var, time::Duration}; use entities::state::scanner::ScannerConfig; -use miette::{Context, IntoDiagnostic}; +use miette::{bail, Context, IntoDiagnostic}; use migration::MigratorTrait; use sea_orm::{ConnectOptions, ConnectionTrait, Database, DatabaseBackend, DatabaseConnection}; use std::sync::Arc; +use tracing::info; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -40,7 +41,7 @@ async fn _main() -> miette::Result<()> { var("RUST_LOG").unwrap_or_else(|_| { #[cfg(debug_assertions)] return format!( - "warn,tower_http=debug,migration=debug,scanner=trace,server=debug,{}=debug", + "warn,tower_http=debug,migration=debug,scanner=trace,server=trace,{}=trace", env!("CARGO_PKG_NAME") ) .into(); @@ -125,7 +126,6 @@ fn read_scanner_cfg() -> miette::Result { let additional_hosts: Vec = require_env_vec_str("ADDITIONAL_HOSTS")?; let additional_host_country = require_env_str("ADDITIONAL_HOSTS_COUNTRY")?; let rss_content = require_env_str("RSS_CONTENT")?; - let bad_hosts: Vec = require_env_vec_str("BAD_HOSTS")?; let auto_mute = require_env_str("AUTO_MUTE")? == "true"; let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?; let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?; @@ -157,7 +157,6 @@ fn read_scanner_cfg() -> miette::Result { auto_mute, source_git_branch, source_git_url, - bad_hosts, cleanup_interval: Duration::from_secs(cleanup_interval), error_retention_per_host, connectivity_path: String::from("/"), @@ -167,7 +166,7 @@ fn read_scanner_cfg() -> miette::Result { async fn test_init(db: &DatabaseConnection) -> miette::Result<()> { let res = db .query_one(sea_orm::Statement::from_string( - DatabaseBackend::Postgres, + DatabaseBackend::Sqlite, "SELECT sqlite_version() as version;".to_owned(), )) .await @@ -185,18 +184,41 @@ fn read_server_config(instance_ping_interval: usize) -> miette::Result = require_env_str("ADMIN_DOMAINS")? .split(",") .map(|v| v.trim().to_string()) .collect(); + let admin_keys: HashMap = require_env_str("ADMIN_KEYS")? + .split(",") + .map(|v| { + v.trim() + .split_once('=') + .map(|(a, b)| (a.to_owned(), b.to_owned())) + .expect("ADMIN_KEYS have to be in host=key format!") + }) + .collect(); let session_db_uri = require_env_str("SESSION_DB_URI")?; + // sanity check all hosts with an admin key are part of the admin hosts + for host in admin_keys.keys() { + if !admin_domains.contains(host) { + bail!("Found admin key for host {host} that is not in the admin hosts!"); + } + } + + info!( + "Loaded admin domains {:?}, loaded additional admin auth keys for {:?}", + admin_domains, + admin_keys.keys() + ); + Ok(server::Config { site_url, max_age: instance_ping_interval, session_ttl_seconds, login_token_name, admin_domains, + admin_keys, session_db_uri, }) }