diff --git a/.env.example b/.env.example index dfcf400..7744668 100644 --- a/.env.example +++ b/.env.example @@ -53,4 +53,6 @@ ERROR_RETENTION_PER_HOST=100 # session database URI SESSION_DB_URI="sqlite:./sessions.db?mode=rwc" # interval in seconds for gathering .health endpoint data -STATS_INTERVAL_S=900 \ No newline at end of file +STATS_INTERVAL_S=900 +# Folder to use for git operations during nitter version checks +GIT_SCRATCH_DIR="." \ No newline at end of file diff --git a/.gitignore b/.gitignore index a0498ca..60c8013 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /sqlite.db-journal /out.txt /sessions.db +/nitter_version_clone diff --git a/Cargo.lock b/Cargo.lock index 429ac15..ad60818 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1036,6 +1036,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -1220,9 +1231,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1419,9 +1430,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "git2" -version = "0.19.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +checksum = "3e2b37e2f62729cdada11f0e6b3b6fe383c69c29fc619e391223e12856af308c" dependencies = [ "bitflags 2.6.0", "libc", @@ -1779,6 +1790,87 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1807,12 +1899,23 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1953,9 +2056,9 @@ checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libgit2-sys" -version = "0.17.0+1.8.1" +version = "0.18.3+1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" dependencies = [ "cc", "libc", @@ -2020,6 +2123,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + [[package]] name = "lock_api" version = "0.4.12" @@ -2510,9 +2619,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" @@ -2740,6 +2849,15 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -4104,6 +4222,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4259,6 +4388,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -4756,13 +4895,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.1.0", "percent-encoding", + "serde", ] [[package]] @@ -4777,6 +4917,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -5154,6 +5300,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + [[package]] name = "wyz" version = "0.5.1" @@ -5163,6 +5315,29 @@ dependencies = [ "tap", ] +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -5184,12 +5359,66 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/entities/src/state/mod.rs b/entities/src/state/mod.rs index 8d8d9f2..ba15fe3 100644 --- a/entities/src/state/mod.rs +++ b/entities/src/state/mod.rs @@ -22,6 +22,33 @@ pub struct InnerState { pub cache: RwLock, } +/// Resolved information about an instances nitter source commit +#[derive(Debug, PartialEq, Eq, Clone, Serialize)] +pub enum CommitInfo { + /// Commit is behind main + Outdated, + /// Commit equals current main + Current, + /// Commit is inside a custom branch on main + CustomBranch, + /// Commit doesn't exist in the repo + UnknownCommit, + /// Missing commit (invalid URL etc) + Missing, +} + +impl CommitInfo { + pub fn is_latest_version(&self) -> bool { + *self == Self::Current + } + pub fn is_upstream(&self) -> bool { + match self { + CommitInfo::Outdated | CommitInfo::Current => true, + CommitInfo::CustomBranch | CommitInfo::UnknownCommit | CommitInfo::Missing => false, + } + } +} + pub fn new() -> AppState { Arc::new(InnerState { cache: RwLock::new(CacheData { @@ -53,9 +80,13 @@ pub struct CacheHost { pub version_url: Option, pub healthy: bool, pub last_healthy: Option, + /// State of the sources nitter version information + pub version_state: CommitInfo, /// Whether the source is from the normal upstream repo + #[deprecated = "use version_state"] pub is_upstream: bool, /// Whether the source is from the latest upstream commit + #[deprecated = "use version_state"] pub is_latest_version: bool, /// Whether this host is known to be bad (ip blocking) pub is_bad_host: bool, diff --git a/entities/src/state/scanner.rs b/entities/src/state/scanner.rs index 85c9e25..e2d6429 100644 --- a/entities/src/state/scanner.rs +++ b/entities/src/state/scanner.rs @@ -37,6 +37,8 @@ pub struct Config { pub source_git_url: String, /// Git branch to fetch the current commit from pub source_git_branch: String, + /// Folder to use for git operations during nitter version checks + pub git_scratch_folder: String, /// Interval to run cleanup operations in, to remove old data pub cleanup_interval: Duration, /// Amount of latest errors to keep per instance/host @@ -68,6 +70,7 @@ impl Config { cleanup_interval: Duration::from_secs(24 * 60 * 60), error_retention_per_host: 100, connectivity_path: String::from("/"), + git_scratch_folder: String::from("."), }) } } diff --git a/scanner/Cargo.toml b/scanner/Cargo.toml index 2b6c2d1..41342e5 100644 --- a/scanner/Cargo.toml +++ b/scanner/Cargo.toml @@ -26,7 +26,7 @@ tokio = { workspace = true, features = ["full"] } scraper = "0.17.1" chrono = { workspace = true } regex = { workspace = true } -git2 = "0.19.0" +git2 = "0.20" # testing serde = { workspace = true, features = ["derive"] } # health parsing diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index 0f5cd8d..4a84ecd 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -6,16 +6,16 @@ use chrono::{Days, Utc}; use chrono::{Duration, TimeZone}; use entities::host_overrides::keys::{KEY_BAD_HOST, VAL_BOOL_TRUE}; use entities::prelude::*; -use entities::state::CacheData; use entities::state::CacheHost; +use entities::state::{CacheData, CommitInfo}; use entities::{host, host_overrides}; use sea_orm::EntityTrait; use sea_orm::QueryFilter; use sea_orm::QueryOrder; use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}; use sea_orm::{ColumnTrait, QuerySelect}; +use tokio::task::spawn_blocking; -use crate::version_check::fetch_git_state; use crate::LatestCheck; use crate::{Result, Scanner}; @@ -59,15 +59,28 @@ impl Scanner { /// Generate host stats and returns a new CacheData pub(crate) async fn generate_cache_data(&self) -> Result { let hosts = self.query_hosts_enabled().await?; - let config_c = self.inner.config.clone(); - let current_version = tokio::task::spawn_blocking(move || fetch_git_state(config_c)) - .await - .unwrap()?; + let scanner_c = self.clone(); + if let Err(error) = spawn_blocking(move || { + scanner_c + .inner + .version_checker + .lock() + .unwrap() + .update_remote() + }) + .await + .unwrap() + { + tracing::error!(%error, "Unable to update nitter version, using old version"); + } + + let latest_commit = self.inner.version_checker.lock().unwrap().latest_commit()?; + if hosts.is_empty() { return Ok(CacheData { hosts: Vec::new(), last_update: Utc::now(), - latest_commit: current_version.version, + latest_commit, }); } @@ -123,17 +136,26 @@ impl Scanner { let last_check = latest_check.get(&host.id).unwrap_or(&default_health_check); let points = (points * 100.0) as i32; - let latest_version = host - .version_url - .as_ref() - .map_or(false, |url| current_version.is_same_version(&url)); - let is_upstream = host - .version_url - .as_ref() - .map_or(false, |url| current_version.is_same_repo(&url)); - let is_bad_host = bad_hosts.contains(&host.id); + let version_state = match host.version_url.clone() { + Some(url) => { + let scanner_c = self.clone(); + spawn_blocking(move || { + scanner_c + .inner + .version_checker + .lock() + .unwrap() + .check_url(&url) + }) + .await + .unwrap()? + } + None => CommitInfo::UnknownCommit, + }; + + // TODO: rename all of this from ping to response_time or similar let host_ping_data = ping_data.remove(&host.id); let last_healthy = last_healthy_check.remove(&host.id); let __show_last_seen = @@ -152,8 +174,11 @@ impl Scanner { ping_min: host_ping_data.as_ref().and_then(|v| v.min), ping_avg: host_ping_data.as_ref().and_then(|v| v.avg), recent_pings: host_ping_data.map(|v| v.pings).unwrap_or_default(), - is_latest_version: latest_version, - is_upstream, + #[allow(deprecated)] + is_latest_version: version_state.is_latest_version(), + #[allow(deprecated)] + is_upstream: version_state.is_upstream(), + version_state, version_url: host.version_url, is_bad_host, country: host.country, @@ -187,7 +212,7 @@ impl Scanner { Ok(CacheData { hosts: host_statistics, last_update: time_now, - latest_commit: current_version.version, + latest_commit, }) } @@ -372,7 +397,11 @@ impl Scanner { Ok(stats) } - /// Query latest health checks for the red/green only graph. Returns latest $amount in ascending order and formatted time. + /// Query latest health checks for the red/green only graph. + /// + /// amount: last amount of entries to retrieve + /// + /// Returns tuples of `(formatted time,healthy)` in ascending order of time. async fn query_latest_health_checks( &self, // How many to retrieve diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index 4472ddd..bca5416 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -19,6 +19,8 @@ use sea_orm::{ConnectionTrait, DatabaseConnection, DbBackend, FromQueryResult, S use thiserror::Error; use tokio::time::sleep; +use crate::version_check::VersionCheck; + type Result = std::result::Result; mod about_parser; @@ -164,6 +166,7 @@ struct InnerScanner { rss_check_regex: Regex, client_ipv4: Client, client_ipv6: Client, + version_checker: Mutex, } impl Scanner { @@ -193,6 +196,9 @@ impl Scanner { ) -> miette::Result { let mut builder_regex_rss = RegexBuilder::new(&config.rss_content); builder_regex_rss.case_insensitive(true); + + let version_checker = Mutex::new(VersionCheck::new(config.clone()).into_diagnostic()?); + let http_client = Self::client_builder(&config); let client_ipv4 = Scanner::client_builder(&config) .local_address("0.0.0.0".parse::().unwrap()) @@ -217,6 +223,7 @@ impl Scanner { inner: Arc::new(InnerScanner { db, app_state, + version_checker, client: http_client.build().unwrap(), config, client_ipv4, diff --git a/scanner/src/version_check.rs b/scanner/src/version_check.rs index 82fbe5f..22d83ac 100644 --- a/scanner/src/version_check.rs +++ b/scanner/src/version_check.rs @@ -1,72 +1,194 @@ -use entities::state::scanner::ScannerConfig; -use git2::{Direction, Remote}; +// SPDX-License-Identifier: AGPL-3.0-only -use crate::{Result, ScannerError}; +use std::{collections::HashMap, path::Path}; -pub(crate) struct CurrentVersion { - pub version: String, +use entities::state::{scanner::ScannerConfig, CommitInfo}; +use git2::Repository; + +use crate::Result; + +pub struct VersionCheck { config: ScannerConfig, + commit_cache: HashMap, + current_epoch: u8, + // TODO: this is ugly and unnecessary + repository: Repository, } -const SHORT_COMMIT_LEN: usize = 7; - -impl CurrentVersion { - /// Whether a nitter version URL is from the same repo - pub fn is_same_repo(&self, url: &str) -> bool { - url.starts_with(self.config.source_git_url.trim_end_matches(".git")) - } - /// Whether a nitter version URL is pointing to the same version as this one - pub fn is_same_version(&self, url: &str) -> bool { - // look for last path segment (don't parse) and look if that matches - // from the start - self.is_same_repo(url) - && match url.split('/').last() { - Some(other_version) => { - if other_version.len() == self.version.len() { - // we get the long version - other_version.starts_with(&self.version) - } else if other_version.len() == SHORT_COMMIT_LEN { - other_version.starts_with(&self.version[..SHORT_COMMIT_LEN]) - } else { - // everything else is no short commit id.. - false - } - } - None => false, - } - } +/// Internal cache entry +struct CommitCacheValue { + result: CommitInfo, + epoch: u8, } -pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result { - let mut remote = Remote::create_detached(config.source_git_url.as_str())?; - - remote.connect(Direction::Fetch)?; - - let reference = format!("refs/heads/{}", config.source_git_branch); - let commit = remote - .list()? - .into_iter() - .find(|v| v.name() == &reference) - .map(|v| v.oid().to_string()); - - remote.disconnect()?; - - Ok(commit - .map(|commit| CurrentVersion { - version: commit, +impl VersionCheck { + pub(crate) fn new(config: ScannerConfig) -> Result { + let temp_dir = Path::new(&config.git_scratch_folder).join(format!("nitter_version_clone")); + tracing::debug!(git_check_dir=?temp_dir); + let repository = Repository::init(temp_dir)?; + let mut checker = VersionCheck { config, - }) - .ok_or(ScannerError::GitBranchNotFound)?) + commit_cache: Default::default(), + current_epoch: 0, + repository, + }; + + checker.update_remote()?; + Ok(checker) + } + /// Increases the epoch, invalidating unused cache entries and removes them. + /// + /// Blocking for large caches. + pub fn cycle_epoch(&mut self) { + self.current_epoch = self.current_epoch.wrapping_add(1); + let curent_epoch = self.current_epoch; + self.commit_cache + .retain(|_key, entry| curent_epoch.wrapping_sub(entry.epoch) > 1); + } + /// Fetch remote, updating to new version, blocking + pub(crate) fn update_remote(&mut self) -> Result<()> { + self.cycle_epoch(); + let mut remote = match self.repository.find_remote(REMOTE_NAME) { + Ok(remote) => { + if remote.url() != Some(&self.config.source_git_url) { + tracing::warn!(found=?remote.url(),expected=self.config.source_git_url,"remote URL for {REMOTE_NAME} is unexpected"); + let mut config = self.repository.config()?; + config.set_str( + &format!("remote.{REMOTE_NAME}.url"), + &self.config.source_git_url, + )?; + } + remote + } + Err(_) => self + .repository + .remote(REMOTE_NAME, &self.config.source_git_url)?, + }; + + remote.fetch(&["refs/heads/*:refs/heads/*"], None, None)?; + Ok(()) + } + + /// Returns latest commit on main branch. + pub(crate) fn latest_commit(&self) -> Result { + let main_branch = self.repository.find_reference(&format!( + "refs/remotes/{REMOTE_NAME}/{}", + self.config.source_git_branch + ))?; + let current_main_commit = main_branch.peel_to_commit()?; + Ok(current_main_commit.id().to_string()) + } + + /// Check nitter git URL with SHA for its commit state. + /// + /// Example is `https://github.com/zedeus/nitter/commit/a92e79e` + pub(crate) fn check_url(&mut self, url: &str) -> Result { + match url.split('/').last() { + Some(commit) => self.check_commit(commit), + None => Ok(CommitInfo::UnknownCommit), + } + } + + pub(crate) fn check_commit(&mut self, commit_sha: &str) -> Result { + if let Some(value) = self.commit_cache.get_mut(commit_sha) { + if self.current_epoch.wrapping_sub(value.epoch) <= 1 { + value.epoch = self.current_epoch; + return Ok(value.result.clone()); + } + } + let result = self.check_commit_inner(commit_sha)?; + self.commit_cache.insert( + commit_sha.to_string(), + CommitCacheValue { + result: result.clone(), + epoch: self.current_epoch, + }, + ); + + Ok(result) + } + + fn check_commit_inner(&self, commit_sha: &str) -> Result { + let commit = match self.repository.revparse_single(commit_sha) { + Ok(commit) => commit, + Err(_) => return Ok(CommitInfo::UnknownCommit), + }; + + let main_branch = self.repository.find_reference(&format!( + "refs/remotes/{REMOTE_NAME}/{}", + self.config.source_git_branch + ))?; + let current_main_commit = main_branch.peel_to_commit()?; + + if current_main_commit.id() == commit.id() { + return Ok(CommitInfo::Current); + } + + let mut revwalk = self.repository.revwalk()?; + revwalk.push(main_branch.target().unwrap())?; + + let is_in_main_branch = + revwalk.any(|parent| parent.map(|v| v == commit.id()).unwrap_or_default()); + + match is_in_main_branch { + true => Ok(CommitInfo::Outdated), + false => Ok(CommitInfo::CustomBranch), + } + } } #[cfg(test)] mod test { use entities::state::scanner::Config; - use super::fetch_git_state; + use crate::version_check::{CommitInfo, VersionCheck}; #[test] - fn test_git() { - fetch_git_state(Config::test_defaults()).unwrap(); + fn test_git_commit_exists() { + let mut checker = VersionCheck::new(Config::test_defaults()).unwrap(); + assert_eq!( + CommitInfo::Outdated, + checker + .check_commit("064ec8808022abb071f93f0fc976a8aa123699dc",) + .unwrap(), + "long old hash should be outdated" + ); + assert_eq!( + CommitInfo::Outdated, + checker + .check_url("https://github.com/zedeus/nitter/commit/51b5485",) + .unwrap(), + "old URL should be outdated" + ); + assert_eq!( + CommitInfo::UnknownCommit, + checker + .check_url("https://github.com/zedeus/nitter/commit/3295bdb",) + .unwrap(), + "URL for unknown commit should be unknown" + ); + assert_eq!( + CommitInfo::Outdated, + checker.check_commit("064ec88",).unwrap(), + "short old hash should be outdated" + ); + assert_eq!( + CommitInfo::UnknownCommit, + checker + .check_commit("064ec8808022abb071f93f0fc976b8aa123699dc",) + .unwrap(), + "long invalid hash should be Unknown" + ); + // relies on https://github.com/zedeus/nitter/commits/tweets-parser/ + assert_eq!( + CommitInfo::CustomBranch, + checker + .check_commit("c9b261a79303189f61ef5f5c6bf2c2600cdba792",) + .unwrap(), + "long invalid hash should be Unknown" + ); + checker.update_remote().unwrap(); } } + +static REMOTE_NAME: &str = "origin"; diff --git a/server/templates/instances.html.j2 b/server/templates/instances.html.j2 index d495996..de523fa 100644 --- a/server/templates/instances.html.j2 +++ b/server/templates/instances.html.j2 @@ -95,22 +95,22 @@ {%- endif -%} {%- endif -%} + {%- if host.version_state == "Missing" -%} + {%- set version_order = "E" -%} + {%- elif host.version_state == "UnknownCommit" -%} + {%- set version_order = "D" -%} + {%- elif host.version_state == "CustomBranch" -%} + {%- set version_order = "C" -%} + {%- elif host.version_state == "Current" -%} + {%- set version_order = "A" -%} + {%- elif host.version_state == "Outdated" -%} + {%- set version_order = "B" -%} + {%- endif -%} {%- if not host.__show_last_seen -%}{{host.ping_avg}}ms{% endif %} {{host.healthy_percentage_overall}}% {% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%} - {%- if host.healthy and not host.version -%} - {%- set version = "Dmissing" -%} - {%- elif host.version and not host.is_upstream -%} - {%- set version = "Aforeign" -%} - {%- elif host.is_latest_version -%} - {%- set version = "Blatest" -%} - {%- elif host.version and not host.is_latest_version -%} - {%- set version = "Coutdated" -%} - {%- else -%} - {%- set version = "Eunknown" -%} - {%- endif -%} - + {%- if host.version_url -%} {{host.version | truncate(length=18) | default(value="missing version")}} @@ -118,13 +118,15 @@ {%- else -%} missing version {%- endif -%} - {%- if version == "Dmissing" -%} + {%- if host.version_state == "Missing" -%} missing - {%- elif version == "Aforeign" -%} - custom - {%- elif version == "Blatest" -%} + {%- elif host.version_state == "UnknownCommit" -%} + fork + {%- elif host.version_state == "CustomBranch" -%} + experimental + {%- elif host.version_state == "Current" -%} latest - {# {%- elif version == "Coutdated" -%} + {%- elif host.version_state == "Outdated" -%} outdated #} {%- endif -%} diff --git a/src/main.rs b/src/main.rs index f3127ea..93695da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -129,6 +129,7 @@ fn read_scanner_cfg() -> miette::Result { let auto_mute = require_env_str("AUTO_MUTE")? == "true"; let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?; let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?; + let git_scratch_folder: String = require_env_str("GIT_SCRATCH_DIR")?; let cleanup_interval: u64 = require_env_str("CLEANUP_INTERVAL_S")? .parse() .expect("CLEANUP_INTERVAL_S must be a number"); @@ -160,6 +161,7 @@ fn read_scanner_cfg() -> miette::Result { cleanup_interval: Duration::from_secs(cleanup_interval), error_retention_per_host, connectivity_path: String::from("/"), + git_scratch_folder, })) }