diff --git a/.env.example b/.env.example index 2ee14b7..8ce65d9 100644 --- a/.env.example +++ b/.env.example @@ -29,4 +29,8 @@ PING_RANGE_H="3" # enable/disable startup scanning DISABLE_STARTUP_SCAN=false # don't emit errors when re-checking offline hosts -AUTO_MUTE=true \ No newline at end of file +AUTO_MUTE=true +# branch to use for fetching the latest nitter commit +ORIGIN_SOURCE_GIT_BRANCH="master" +# URL to use for fetching the latest nitter commit +ORIGIN_SOURCE_GIT_URL="https://github.com/zedeus/nitter.git" \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index de45dae..5847825 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1235,6 +1235,21 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "git2" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" +dependencies = [ + "bitflags 1.3.2", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "globset" version = "0.4.11" @@ -1661,6 +1676,20 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libgit2-sys" +version = "0.15.2+1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.7" @@ -1678,6 +1707,32 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2676,6 +2731,7 @@ version = "0.1.0" dependencies = [ "chrono", "entities", + "git2", "regex", "reqwest", "scraper", diff --git a/entities/src/host.rs b/entities/src/host.rs index 10ba640..c60a91c 100644 --- a/entities/src/host.rs +++ b/entities/src/host.rs @@ -22,6 +22,7 @@ pub struct Model { pub enabled: bool, pub rss: bool, pub version: Option, + pub version_url: Option, /// Last time the url and enabled were updated, *not* the rss pub updated: i64, } @@ -32,6 +33,7 @@ pub enum Column { Domain, Url, Version, + VersionUrl, Enabled, Rss, Updated, @@ -62,6 +64,7 @@ impl ColumnTrait for Column { Self::Domain => ColumnType::String(None).def(), Self::Url => ColumnType::String(None).def(), Self::Version => ColumnType::String(None).def().null(), + Self::VersionUrl => ColumnType::String(None).def().null(), Self::Enabled => ColumnType::Integer.def(), Self::Rss => ColumnType::Integer.def(), Self::Updated => ColumnType::Integer.def(), diff --git a/entities/src/state/mod.rs b/entities/src/state/mod.rs index f195137..9e45014 100644 --- a/entities/src/state/mod.rs +++ b/entities/src/state/mod.rs @@ -16,6 +16,7 @@ pub fn new() -> Cache { Arc::new(RwLock::new(CacheData { hosts: vec![], last_update: Utc::now(), + latest_commit: String::new(), })) } @@ -23,6 +24,7 @@ pub fn new() -> Cache { pub struct CacheData { pub hosts: Vec, pub last_update: DateTimeUtc, + pub latest_commit: String, } #[derive(Debug, Serialize)] @@ -36,6 +38,11 @@ pub struct CacheHost { pub ping_min: Option, pub ping_avg: Option, pub version: Option, + pub version_url: Option, pub healthy: bool, pub last_healthy: Option, + /// Whether the source is from the normal upstream repo + pub is_upstream: bool, + /// Whether the source is from the latest upstream commit + pub is_latest_version: bool, } diff --git a/entities/src/state/scanner.rs b/entities/src/state/scanner.rs index 40e2863..922131b 100644 --- a/entities/src/state/scanner.rs +++ b/entities/src/state/scanner.rs @@ -26,6 +26,10 @@ pub struct Config { pub ping_range: chrono::Duration, /// don't emit errors for hosts which are already listed as down pub auto_mute: bool, + /// Git URL for source fetching + pub source_git_url: String, + /// Git branch to fetch the current commit from + pub source_git_branch: String, } impl Config { @@ -43,6 +47,8 @@ impl Config { referer: String::from(""), ping_range: chrono::Duration::hours(3), auto_mute: true, + source_git_branch: String::from("master"), + source_git_url: String::from("https://github.com/zedeus/nitter.git"), }) } } diff --git a/migration/src/lib.rs b/migration/src/lib.rs index cf7f6ea..317cd1e 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -3,6 +3,7 @@ pub use sea_orm_migration::prelude::*; mod m20220101_000001_create_table; mod m20230729_010231_datetime_rowid; mod m20230729_230909_datetime_int_host; +mod m20230803_154714_version_url; pub struct Migrator; @@ -13,6 +14,7 @@ impl MigratorTrait for Migrator { Box::new(m20220101_000001_create_table::Migration), Box::new(m20230729_010231_datetime_rowid::Migration), Box::new(m20230729_230909_datetime_int_host::Migration), + Box::new(m20230803_154714_version_url::Migration), ] } } diff --git a/migration/src/m20230803_154714_version_url.rs b/migration/src/m20230803_154714_version_url.rs new file mode 100644 index 0000000..d950723 --- /dev/null +++ b/migration/src/m20230803_154714_version_url.rs @@ -0,0 +1,26 @@ +use sea_orm_migration::{ + prelude::*, + sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}, +}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cmd = r#"ALTER TABLE "host" ADD COLUMN "version_url" text;"#; + let db = manager.get_connection(); + db.execute_unprepared("BEGIN EXCLUSIVE").await?; + tracing::info!("adding version_url column.."); + db.execute_unprepared(cmd).await?; + db.execute_unprepared("COMMIT TRANSACTION").await?; + db.execute_unprepared("VACUUM").await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + panic!("Can't migrate down"); + } +} diff --git a/scanner/Cargo.toml b/scanner/Cargo.toml index cf8bad8..13de704 100644 --- a/scanner/Cargo.toml +++ b/scanner/Cargo.toml @@ -14,6 +14,7 @@ tokio = { workspace = true, features = ["full"] } scraper = "0.17.1" chrono = { workspace = true } regex = { workspace = true } +git2 = "0.17.2" [dev-dependencies] tracing-test = { workspace = true } diff --git a/scanner/src/about_parser.rs b/scanner/src/about_parser.rs index 5134960..9000b2f 100644 --- a/scanner/src/about_parser.rs +++ b/scanner/src/about_parser.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only use regex::{Regex, RegexBuilder}; +use reqwest::Url; use scraper::{Html, Selector}; use thiserror::Error; @@ -15,6 +16,8 @@ pub enum AboutParseError { NoCommitLinkFound, #[error("Missing test! Found '{0}'")] InvalidCommitFormat(String), + #[error("No valid href!")] + NoValidHref, } pub(crate) struct AboutParser { @@ -23,9 +26,14 @@ pub(crate) struct AboutParser { regex: Regex, } +pub struct AboutParsed { + pub version_name: String, + pub url: String, +} + impl AboutParser { /// Returns the text of the version element of nitters about site - pub fn parse_about_version(&self, html: &str) -> Result { + pub fn parse_about_version(&self, html: &str) -> Result { let fragment = Html::parse_fragment(html); // get all

elements let p_elem = fragment @@ -35,6 +43,8 @@ impl AboutParser { let mut a_elems = p_elem.select(&self.selector_a); let link = a_elems.next().ok_or(AboutParseError::NoCommitLinkFound)?; + let url = link.value().attr("href").map(|v|v.trim().to_owned()) + .ok_or(AboutParseError::NoValidHref)?; let link_text = link.text().fold(String::new(), |mut acc, text| { acc.push_str(text); acc @@ -42,7 +52,10 @@ impl AboutParser { if !self.regex.is_match(&link_text) { return Err(AboutParseError::InvalidCommitFormat(link_text)); } - Ok(link_text) + Ok(AboutParsed { + url, + version_name: link_text + }) } pub fn new() -> Self { @@ -64,6 +77,7 @@ mod test { let html = include_str!("../test_data/about.html"); let parser = AboutParser::new(); let res = parser.parse_about_version(html).unwrap(); - assert_eq!(&res, "2023.07.22-72d8f35"); + assert_eq!(&res.version_name, "2023.07.22-72d8f35"); + assert_eq!(res.url, String::from("https://github.com/zedeus/nitter/commit/72d8f35")); } } diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index e42e518..13c8ebc 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -14,6 +14,7 @@ use sea_orm::QueryOrder; use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}; use crate::LatestCheck; +use crate::version_check::fetch_git_state; use crate::{Result, Scanner}; #[derive(Debug, FromQueryResult)] @@ -49,10 +50,15 @@ impl Scanner { /// Generate host stats and returns a new CacheData pub(crate) async fn generate_cache_data(&self) -> Result { let hosts = self.query_hosts_enabled().await?; + let config_c = self.inner.config.clone(); + let current_version = tokio::task::spawn_blocking(move || { + fetch_git_state(config_c) + }).await.unwrap()?; if hosts.is_empty() { return Ok(CacheData { - hosts: vec![], + hosts: Vec::new(), last_update: Utc::now(), + latest_commit: current_version.version, }); } @@ -95,6 +101,9 @@ impl Scanner { // }; let points = (points * 100.0) as i32; + let latest_version = host.version_url.as_ref().map_or(false, |url|current_version.is_same_version(&url)); + let is_upstream = host.version_url.as_ref().map_or(false, |url|current_version.is_same_repo(&url)); + let host_ping_data = ping_data.remove(&host.id); host_statistics.push(CacheHost { last_healthy: last_healthy_check.remove(&host.id), @@ -108,6 +117,9 @@ impl Scanner { ping_min: host_ping_data.as_ref().and_then(|v|v.min), ping_avg: host_ping_data.as_ref().and_then(|v|v.avg), recent_pings: host_ping_data.map(|v|v.pings).unwrap_or_default(), + is_latest_version: latest_version, + is_upstream, + version_url: host.version_url, }) } host_statistics.sort_unstable_by_key(|k| k.points); @@ -115,6 +127,7 @@ impl Scanner { Ok(CacheData { hosts: host_statistics, last_update: time_now, + latest_commit: current_version.version, }) } diff --git a/scanner/src/instance_check.rs b/scanner/src/instance_check.rs index 72c7627..2702af8 100644 --- a/scanner/src/instance_check.rs +++ b/scanner/src/instance_check.rs @@ -17,6 +17,7 @@ use tracing::instrument; use crate::FetchError; use crate::Result; use crate::Scanner; +use crate::about_parser::AboutParsed; impl Scanner { /// Check uptime for host and create a new uptime entry in the database @@ -147,7 +148,7 @@ impl Scanner { } /// Check nitter version - pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option { + pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option { url.set_path(&self.inner.config.about_path); match self.fetch_url(url.as_str()).await { Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) { diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index a531380..22edb38 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -31,6 +31,7 @@ mod about_parser; mod cache_update; mod instance_check; mod instance_parser; +mod version_check; const CAPTCHA_TEXT: &'static str = "Enable JavaScript and cookies to continue"; const CAPTCHA_CODE: u16 = 403; @@ -58,6 +59,10 @@ pub enum ScannerError { InstanceParse(#[from] instance_parser::InstaceListError), #[error("Failed to fetch URL: {0}")] FetchError(#[from] FetchError), + #[error("Failed fetcing git: {0}")] + GitFetch(#[from] git2::Error), + #[error("Couldn't find git branch")] + GitBranch, } #[derive(Error, Debug)] @@ -263,25 +268,29 @@ impl Scanner { }; // tracing::trace!(muted_host,instance=?instance,last_status=?last_status); join_set.spawn(async move { - let (rss, version) = match Url::parse(&instance.url) { + let (rss, version, version_url) = match Url::parse(&instance.url) { Err(_) => { if !muted_host { tracing::info!(url = instance.url, "Instance URL invalid"); } - (false, None) + (false, None, None) } Ok(mut url) => { let rss = scanner_c.has_rss(&mut url, muted_host).await; - let version = scanner_c.nitter_version(&mut url, muted_host).await; - (rss, version) + match scanner_c.nitter_version(&mut url, muted_host).await { + Some(version) => (rss,Some(version.version_name),Some(version.url)), + None => (rss,None,None), + } } }; + host::ActiveModel { id: ActiveValue::NotSet, domain: ActiveValue::Set(instance.domain), url: ActiveValue::Set(instance.url), enabled: ActiveValue::Set(true), version: ActiveValue::Set(version), + version_url: ActiveValue::Set(version_url), rss: ActiveValue::Set(rss), updated: ActiveValue::Set(time.timestamp()), } diff --git a/scanner/src/version_check.rs b/scanner/src/version_check.rs new file mode 100644 index 0000000..87a0b35 --- /dev/null +++ b/scanner/src/version_check.rs @@ -0,0 +1,63 @@ +use entities::state::scanner::ScannerConfig; +use git2::{Remote, Direction}; + +use crate::{Result, ScannerError}; + +pub(crate) struct CurrentVersion { + pub version: String, + config: ScannerConfig, +} + +const SHORT_COMMIT_LEN: usize = 7; + +impl CurrentVersion { + /// Whether a nitter version URL is from the same repo + pub fn is_same_repo(&self, url: &str) -> bool { + url.starts_with(self.config.source_git_url.trim_end_matches(".git")) + } + /// Whether a nitter version URL is pointing to the same version as this one + pub fn is_same_version(&self, url: &str) -> bool { + // look for last path segment (don't parse) and look if that matches + // from the start + self.is_same_repo(url) && match url.split('/').last() { + Some(other_version) => { + if other_version.len() == self.version.len() { + // we get the long version + other_version.starts_with(&self.version) + } else if other_version.len() == SHORT_COMMIT_LEN { + other_version.starts_with(&self.version[..SHORT_COMMIT_LEN]) + } else { + // everything else is no short commit id.. + false + } + } + None => false + } + } +} + +pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result { + let mut remote = Remote::create_detached(config.source_git_url.as_str())?; + + remote.connect(Direction::Fetch)?; + + let reference = format!("refs/heads/{}",config.source_git_branch); + let commit = remote.list()?.into_iter().find(|v|v.name() == &reference) + .map(|v|v.oid().to_string()); + + remote.disconnect()?; + + Ok(commit.map(|commit|CurrentVersion { version: commit, config }).ok_or(ScannerError::GitBranch)?) +} + +#[cfg(test)] +mod test { + use entities::state::scanner::Config; + + use super::fetch_git_state; + + #[test] + fn test_git() { + fetch_git_state(Config::test_defaults()).unwrap(); + } +} \ No newline at end of file diff --git a/server/src/website.rs b/server/src/website.rs index f567e5a..21d5d2e 100644 --- a/server/src/website.rs +++ b/server/src/website.rs @@ -32,6 +32,7 @@ pub async fn instances( } pub async fn about( + State(ref cache): State, State(ref template): State>, State(ref scanner_config): State, ) -> Result { @@ -53,6 +54,10 @@ pub async fn about( "ping_avg_interval_h", &scanner_config.ping_range.num_hours(), ); + { + let guard = cache.read().map_err(|_| ServerError::MutexFailure)?; + context.insert("latest_commit", &guard.latest_commit); + } let mut res = Html(template.render("about.html.j2", &context)?).into_response(); res.headers_mut().insert( diff --git a/server/templates/about.html.j2 b/server/templates/about.html.j2 index f6559c0..7172fa2 100644 --- a/server/templates/about.html.j2 +++ b/server/templates/about.html.j2 @@ -14,12 +14,13 @@

Home

Uptime check interval: {{uptime_interval_s}}s.

Instance re-fetching interval, including RSS & version check: {{wiki_interval_s}}s.

+

Fetched latest commit: {{latest_commit}}

The following paths are checked per instance:

    - {% for path in checked_paths %} + {%- for path in checked_paths %}
  • {{path}}
    • - {% endfor %} + {%- endfor %}

The same data can also be fetched as JSON from /api/v1/instances

diff --git a/server/templates/instances.html.j2 b/server/templates/instances.html.j2 index 1cf5360..4f0308f 100644 --- a/server/templates/instances.html.j2 +++ b/server/templates/instances.html.j2 @@ -30,64 +30,81 @@ {% for host in instances -%} {{host.domain}} - {% if host.healthy -%} ✅ {% else -%} ❌ {% endif -%} + {% if host.healthy -%} ✅ {%- else -%} ❌ {%- endif -%} - {% if host.recent_pings|length > 0 -%} - {% set height = 57 -%} - {% set width = 182 -%} - {% set amount = host.recent_pings|length -%} - {% set width_bar = width / amount -%} - {% set offset = 5 -%} - {% set error_size = height / 3 %} - {% if host.ping_max and host.ping_min -%} + {%- if host.recent_pings|length > 0 -%} + {%- set height = 57 -%} + {%- set width = 182 -%} + {%- set amount = host.recent_pings|length -%} + {%- set width_bar = width / amount -%} + {%- set offset = 5 -%} + {%- set error_size = height / 3 %} + {%- if host.ping_max and host.ping_min -%} {# only one entry, so max = min => step_size would be 0 #} - {% if host.ping_max == host.ping_min -%} - {% set step_size = 5/(height-offset) -%} - {% else -%} - {% set step_size = (host.ping_max - host.ping_min)/(height-offset) -%} - {% endif -%} - {% else %} + {%- if host.ping_max == host.ping_min -%} + {%- set step_size = 5/(height-offset) -%} + {%- else -%} + {%- set step_size = (host.ping_max - host.ping_min)/(height-offset) -%} + {%- endif -%} + {%- else %} {# no valid entries, only nulls #} - {% set step_size = 5/(height-offset) -%} - {% endif %} + {%- set step_size = 5/(height-offset) -%} + {%- endif -%} {# #} - {% if host.ping_min -%} + {%- if host.ping_min -%} - {% if host.ping_avg and host.ping_min -%} - {% set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%} + {%- if host.ping_avg and host.ping_min -%} + {%- set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%} Average - {% endif -%} - {% for ping in host.recent_pings -%} - {% if ping -%} - {% set height_bar = (ping - host.ping_min )/step_size + offset -%} - {% set color_bar = "#a09df2" -%} - {% else %} - {% set height_bar = error_size -%} - {% set color_bar = "#cc0000" -%} - {% endif %} + {%- endif -%} + {%- for ping in host.recent_pings -%} + {%- if ping -%} + {%- set height_bar = (ping - host.ping_min )/step_size + offset -%} + {%- set color_bar = "#a09df2" -%} + {%- else -%} + {%- set height_bar = error_size -%} + {%- set color_bar = "#cc0000" -%} + {%- endif -%} {{ping | default(value="unhealthy") }} - {% endfor -%} + {%- endfor -%} - {% else -%} + {%- else -%} {# #} - {% if host.last_healthy -%} + {%- if host.last_healthy -%} Last seen healthy: {{host.last_healthy | truncate(length=10, end="") | replace(from="-", to=".")}} - {% else -%} + {%- else -%} Never seen healthy. - {% endif -%} - {% endif -%} - {% else -%} + {%- endif -%} + {%- endif -%} + {%- else -%} No recent data! - {% endif -%} + {%- endif -%} {{host.ping_avg}} - {% if host.rss -%} ✅ {% else -%} ❌ {% endif -%} - {{host.version}} + {% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%} + + {%- if host.version_url -%} + + {{host.version | default(value="missing version")}} + + {%- else -%} + missing version + {%- endif -%} + {%- if host.healthy and not host.version -%} + missing + {%- elif host.version and not host.is_upstream -%} + foreign repo + {%- elif host.is_latest_version -%} + latest + {%- elif host.version and not host.is_latest_version -%} + outdated + {%- endif -%} + {{host.points}} - {% endfor -%} + {%- endfor -%} diff --git a/src/main.rs b/src/main.rs index 151a4bf..bc5db63 100644 --- a/src/main.rs +++ b/src/main.rs @@ -105,6 +105,8 @@ fn read_scanner_cfg() -> miette::Result { .collect(); let referer = require_env_str("REFERER")?; let auto_mute = require_env_str("AUTO_MUTE")? == "true"; + let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?; + let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?; Ok(Arc::new(entities::state::scanner::Config { list_fetch_interval: Duration::from_secs(instance_list_interval), @@ -119,6 +121,8 @@ fn read_scanner_cfg() -> miette::Result { referer, ping_range: chrono::Duration::hours(ping_range as _), auto_mute, + source_git_branch, + source_git_url, })) }