mirror of
https://github.com/4cuck/nitter-status.git
synced 2026-10-26 00:52:25 +00:00
overhaul version check
This commit is contained in:
4cuck committed
1 parent
709f1e5aa2
commit
a2f98bd522
11 files changed
+535
-107
No files matched your search
@@ -54,3 +54,5 @@ ERROR_RETENTION_PER_HOST=100
|
||||
SESSION_DB_URI="sqlite:./sessions.db?mode=rwc"
|
||||
# interval in seconds for gathering .health endpoint data
|
||||
STATS_INTERVAL_S=900
|
||||
# Folder to use for git operations during nitter version checks
|
||||
GIT_SCRATCH_DIR="."
|
||||
@@ -5,3 +5,4 @@
|
||||
/sqlite.db-journal
|
||||
/out.txt
|
||||
/sessions.db
|
||||
/nitter_version_clone
|
||||
Generated
+244
-15
@@ -1036,6 +1036,17 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.79",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dotenvy"
|
||||
version = "0.15.7"
|
||||
@@ -1220,9 +1231,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.1"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||
checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
@@ -1419,9 +1430,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
|
||||
|
||||
[[package]]
|
||||
name = "git2"
|
||||
version = "0.19.0"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"
|
||||
checksum = "3e2b37e2f62729cdada11f0e6b3b6fe383c69c29fc619e391223e12856af308c"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"libc",
|
||||
@@ -1779,6 +1790,87 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"potential_utf",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_core"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locale_core",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
@@ -1807,12 +1899,23 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1953,9 +2056,9 @@ checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
version = "0.17.0+1.8.1"
|
||||
version = "0.18.3+1.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224"
|
||||
checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -2020,6 +2123,12 @@ version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
@@ -2510,9 +2619,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pest"
|
||||
@@ -2740,6 +2849,15 @@ version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
|
||||
dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
@@ -4104,6 +4222,17 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.79",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.5.1"
|
||||
@@ -4259,6 +4388,16 @@ dependencies = [
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.8.0"
|
||||
@@ -4756,13 +4895,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.2"
|
||||
version = "2.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
|
||||
checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna 0.5.0",
|
||||
"idna 1.1.0",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4777,6 +4917,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
@@ -5154,6 +5300,12 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
|
||||
|
||||
[[package]]
|
||||
name = "wyz"
|
||||
version = "0.5.1"
|
||||
@@ -5163,6 +5315,29 @@ dependencies = [
|
||||
"tap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.79",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
@@ -5184,12 +5359,66 @@ dependencies = [
|
||||
"syn 2.0.79",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.79",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
|
||||
|
||||
[[package]]
|
||||
name = "zerotrie"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.79",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.2"
|
||||
|
||||
@@ -22,6 +22,33 @@ pub struct InnerState {
|
||||
pub cache: RwLock<CacheData>,
|
||||
}
|
||||
|
||||
/// Resolved information about an instances nitter source commit
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Serialize)]
|
||||
pub enum CommitInfo {
|
||||
/// Commit is behind main
|
||||
Outdated,
|
||||
/// Commit equals current main
|
||||
Current,
|
||||
/// Commit is inside a custom branch on main
|
||||
CustomBranch,
|
||||
/// Commit doesn't exist in the repo
|
||||
UnknownCommit,
|
||||
/// Missing commit (invalid URL etc)
|
||||
Missing,
|
||||
}
|
||||
|
||||
impl CommitInfo {
|
||||
pub fn is_latest_version(&self) -> bool {
|
||||
*self == Self::Current
|
||||
}
|
||||
pub fn is_upstream(&self) -> bool {
|
||||
match self {
|
||||
CommitInfo::Outdated | CommitInfo::Current => true,
|
||||
CommitInfo::CustomBranch | CommitInfo::UnknownCommit | CommitInfo::Missing => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new() -> AppState {
|
||||
Arc::new(InnerState {
|
||||
cache: RwLock::new(CacheData {
|
||||
@@ -53,9 +80,13 @@ pub struct CacheHost {
|
||||
pub version_url: Option<String>,
|
||||
pub healthy: bool,
|
||||
pub last_healthy: Option<DateTimeUtc>,
|
||||
/// State of the sources nitter version information
|
||||
pub version_state: CommitInfo,
|
||||
/// Whether the source is from the normal upstream repo
|
||||
#[deprecated = "use version_state"]
|
||||
pub is_upstream: bool,
|
||||
/// Whether the source is from the latest upstream commit
|
||||
#[deprecated = "use version_state"]
|
||||
pub is_latest_version: bool,
|
||||
/// Whether this host is known to be bad (ip blocking)
|
||||
pub is_bad_host: bool,
|
||||
|
||||
@@ -37,6 +37,8 @@ pub struct Config {
|
||||
pub source_git_url: String,
|
||||
/// Git branch to fetch the current commit from
|
||||
pub source_git_branch: String,
|
||||
/// Folder to use for git operations during nitter version checks
|
||||
pub git_scratch_folder: String,
|
||||
/// Interval to run cleanup operations in, to remove old data
|
||||
pub cleanup_interval: Duration,
|
||||
/// Amount of latest errors to keep per instance/host
|
||||
@@ -68,6 +70,7 @@ impl Config {
|
||||
cleanup_interval: Duration::from_secs(24 * 60 * 60),
|
||||
error_retention_per_host: 100,
|
||||
connectivity_path: String::from("/"),
|
||||
git_scratch_folder: String::from("."),
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -26,7 +26,7 @@ tokio = { workspace = true, features = ["full"] }
|
||||
scraper = "0.17.1"
|
||||
chrono = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
git2 = "0.19.0"
|
||||
git2 = "0.20"
|
||||
# testing
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
# health parsing
|
||||
|
||||
+49
-20
@@ -6,16 +6,16 @@ use chrono::{Days, Utc};
|
||||
use chrono::{Duration, TimeZone};
|
||||
use entities::host_overrides::keys::{KEY_BAD_HOST, VAL_BOOL_TRUE};
|
||||
use entities::prelude::*;
|
||||
use entities::state::CacheData;
|
||||
use entities::state::CacheHost;
|
||||
use entities::state::{CacheData, CommitInfo};
|
||||
use entities::{host, host_overrides};
|
||||
use sea_orm::EntityTrait;
|
||||
use sea_orm::QueryFilter;
|
||||
use sea_orm::QueryOrder;
|
||||
use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement};
|
||||
use sea_orm::{ColumnTrait, QuerySelect};
|
||||
use tokio::task::spawn_blocking;
|
||||
|
||||
use crate::version_check::fetch_git_state;
|
||||
use crate::LatestCheck;
|
||||
use crate::{Result, Scanner};
|
||||
|
||||
@@ -59,15 +59,28 @@ impl Scanner {
|
||||
/// Generate host stats and returns a new CacheData
|
||||
pub(crate) async fn generate_cache_data(&self) -> Result<CacheData> {
|
||||
let hosts = self.query_hosts_enabled().await?;
|
||||
let config_c = self.inner.config.clone();
|
||||
let current_version = tokio::task::spawn_blocking(move || fetch_git_state(config_c))
|
||||
.await
|
||||
.unwrap()?;
|
||||
let scanner_c = self.clone();
|
||||
if let Err(error) = spawn_blocking(move || {
|
||||
scanner_c
|
||||
.inner
|
||||
.version_checker
|
||||
.lock()
|
||||
.unwrap()
|
||||
.update_remote()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
tracing::error!(%error, "Unable to update nitter version, using old version");
|
||||
}
|
||||
|
||||
let latest_commit = self.inner.version_checker.lock().unwrap().latest_commit()?;
|
||||
|
||||
if hosts.is_empty() {
|
||||
return Ok(CacheData {
|
||||
hosts: Vec::new(),
|
||||
last_update: Utc::now(),
|
||||
latest_commit: current_version.version,
|
||||
latest_commit,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -123,17 +136,26 @@ impl Scanner {
|
||||
let last_check = latest_check.get(&host.id).unwrap_or(&default_health_check);
|
||||
let points = (points * 100.0) as i32;
|
||||
|
||||
let latest_version = host
|
||||
.version_url
|
||||
.as_ref()
|
||||
.map_or(false, |url| current_version.is_same_version(&url));
|
||||
let is_upstream = host
|
||||
.version_url
|
||||
.as_ref()
|
||||
.map_or(false, |url| current_version.is_same_repo(&url));
|
||||
|
||||
let is_bad_host = bad_hosts.contains(&host.id);
|
||||
|
||||
let version_state = match host.version_url.clone() {
|
||||
Some(url) => {
|
||||
let scanner_c = self.clone();
|
||||
spawn_blocking(move || {
|
||||
scanner_c
|
||||
.inner
|
||||
.version_checker
|
||||
.lock()
|
||||
.unwrap()
|
||||
.check_url(&url)
|
||||
})
|
||||
.await
|
||||
.unwrap()?
|
||||
}
|
||||
None => CommitInfo::UnknownCommit,
|
||||
};
|
||||
|
||||
// TODO: rename all of this from ping to response_time or similar
|
||||
let host_ping_data = ping_data.remove(&host.id);
|
||||
let last_healthy = last_healthy_check.remove(&host.id);
|
||||
let __show_last_seen =
|
||||
@@ -152,8 +174,11 @@ impl Scanner {
|
||||
ping_min: host_ping_data.as_ref().and_then(|v| v.min),
|
||||
ping_avg: host_ping_data.as_ref().and_then(|v| v.avg),
|
||||
recent_pings: host_ping_data.map(|v| v.pings).unwrap_or_default(),
|
||||
is_latest_version: latest_version,
|
||||
is_upstream,
|
||||
#[allow(deprecated)]
|
||||
is_latest_version: version_state.is_latest_version(),
|
||||
#[allow(deprecated)]
|
||||
is_upstream: version_state.is_upstream(),
|
||||
version_state,
|
||||
version_url: host.version_url,
|
||||
is_bad_host,
|
||||
country: host.country,
|
||||
@@ -187,7 +212,7 @@ impl Scanner {
|
||||
Ok(CacheData {
|
||||
hosts: host_statistics,
|
||||
last_update: time_now,
|
||||
latest_commit: current_version.version,
|
||||
latest_commit,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -372,7 +397,11 @@ impl Scanner {
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Query latest health checks for the red/green only graph. Returns latest $amount in ascending order and formatted time.
|
||||
/// Query latest health checks for the red/green only graph.
|
||||
///
|
||||
/// amount: last amount of entries to retrieve
|
||||
///
|
||||
/// Returns tuples of `(formatted time,healthy)` in ascending order of time.
|
||||
async fn query_latest_health_checks(
|
||||
&self,
|
||||
// How many to retrieve
|
||||
|
||||
@@ -19,6 +19,8 @@ use sea_orm::{ConnectionTrait, DatabaseConnection, DbBackend, FromQueryResult, S
|
||||
use thiserror::Error;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::version_check::VersionCheck;
|
||||
|
||||
type Result<T> = std::result::Result<T, ScannerError>;
|
||||
|
||||
mod about_parser;
|
||||
@@ -164,6 +166,7 @@ struct InnerScanner {
|
||||
rss_check_regex: Regex,
|
||||
client_ipv4: Client,
|
||||
client_ipv6: Client,
|
||||
version_checker: Mutex<VersionCheck>,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
@@ -193,6 +196,9 @@ impl Scanner {
|
||||
) -> miette::Result<Self> {
|
||||
let mut builder_regex_rss = RegexBuilder::new(&config.rss_content);
|
||||
builder_regex_rss.case_insensitive(true);
|
||||
|
||||
let version_checker = Mutex::new(VersionCheck::new(config.clone()).into_diagnostic()?);
|
||||
|
||||
let http_client = Self::client_builder(&config);
|
||||
let client_ipv4 = Scanner::client_builder(&config)
|
||||
.local_address("0.0.0.0".parse::<IpAddr>().unwrap())
|
||||
@@ -217,6 +223,7 @@ impl Scanner {
|
||||
inner: Arc::new(InnerScanner {
|
||||
db,
|
||||
app_state,
|
||||
version_checker,
|
||||
client: http_client.build().unwrap(),
|
||||
config,
|
||||
client_ipv4,
|
||||
|
||||
+176
-54
@@ -1,72 +1,194 @@
|
||||
use entities::state::scanner::ScannerConfig;
|
||||
use git2::{Direction, Remote};
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
use crate::{Result, ScannerError};
|
||||
use std::{collections::HashMap, path::Path};
|
||||
|
||||
pub(crate) struct CurrentVersion {
|
||||
pub version: String,
|
||||
use entities::state::{scanner::ScannerConfig, CommitInfo};
|
||||
use git2::Repository;
|
||||
|
||||
use crate::Result;
|
||||
|
||||
pub struct VersionCheck {
|
||||
config: ScannerConfig,
|
||||
commit_cache: HashMap<String, CommitCacheValue>,
|
||||
current_epoch: u8,
|
||||
// TODO: this is ugly and unnecessary
|
||||
repository: Repository,
|
||||
}
|
||||
|
||||
const SHORT_COMMIT_LEN: usize = 7;
|
||||
|
||||
impl CurrentVersion {
|
||||
/// Whether a nitter version URL is from the same repo
|
||||
pub fn is_same_repo(&self, url: &str) -> bool {
|
||||
url.starts_with(self.config.source_git_url.trim_end_matches(".git"))
|
||||
}
|
||||
/// Whether a nitter version URL is pointing to the same version as this one
|
||||
pub fn is_same_version(&self, url: &str) -> bool {
|
||||
// look for last path segment (don't parse) and look if that matches
|
||||
// from the start
|
||||
self.is_same_repo(url)
|
||||
&& match url.split('/').last() {
|
||||
Some(other_version) => {
|
||||
if other_version.len() == self.version.len() {
|
||||
// we get the long version
|
||||
other_version.starts_with(&self.version)
|
||||
} else if other_version.len() == SHORT_COMMIT_LEN {
|
||||
other_version.starts_with(&self.version[..SHORT_COMMIT_LEN])
|
||||
} else {
|
||||
// everything else is no short commit id..
|
||||
false
|
||||
}
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
/// Internal cache entry
|
||||
struct CommitCacheValue {
|
||||
result: CommitInfo,
|
||||
epoch: u8,
|
||||
}
|
||||
|
||||
pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result<CurrentVersion> {
|
||||
let mut remote = Remote::create_detached(config.source_git_url.as_str())?;
|
||||
|
||||
remote.connect(Direction::Fetch)?;
|
||||
|
||||
let reference = format!("refs/heads/{}", config.source_git_branch);
|
||||
let commit = remote
|
||||
.list()?
|
||||
.into_iter()
|
||||
.find(|v| v.name() == &reference)
|
||||
.map(|v| v.oid().to_string());
|
||||
|
||||
remote.disconnect()?;
|
||||
|
||||
Ok(commit
|
||||
.map(|commit| CurrentVersion {
|
||||
version: commit,
|
||||
impl VersionCheck {
|
||||
pub(crate) fn new(config: ScannerConfig) -> Result<Self> {
|
||||
let temp_dir = Path::new(&config.git_scratch_folder).join(format!("nitter_version_clone"));
|
||||
tracing::debug!(git_check_dir=?temp_dir);
|
||||
let repository = Repository::init(temp_dir)?;
|
||||
let mut checker = VersionCheck {
|
||||
config,
|
||||
})
|
||||
.ok_or(ScannerError::GitBranchNotFound)?)
|
||||
commit_cache: Default::default(),
|
||||
current_epoch: 0,
|
||||
repository,
|
||||
};
|
||||
|
||||
checker.update_remote()?;
|
||||
Ok(checker)
|
||||
}
|
||||
/// Increases the epoch, invalidating unused cache entries and removes them.
|
||||
///
|
||||
/// Blocking for large caches.
|
||||
pub fn cycle_epoch(&mut self) {
|
||||
self.current_epoch = self.current_epoch.wrapping_add(1);
|
||||
let curent_epoch = self.current_epoch;
|
||||
self.commit_cache
|
||||
.retain(|_key, entry| curent_epoch.wrapping_sub(entry.epoch) > 1);
|
||||
}
|
||||
/// Fetch remote, updating to new version, blocking
|
||||
pub(crate) fn update_remote(&mut self) -> Result<()> {
|
||||
self.cycle_epoch();
|
||||
let mut remote = match self.repository.find_remote(REMOTE_NAME) {
|
||||
Ok(remote) => {
|
||||
if remote.url() != Some(&self.config.source_git_url) {
|
||||
tracing::warn!(found=?remote.url(),expected=self.config.source_git_url,"remote URL for {REMOTE_NAME} is unexpected");
|
||||
let mut config = self.repository.config()?;
|
||||
config.set_str(
|
||||
&format!("remote.{REMOTE_NAME}.url"),
|
||||
&self.config.source_git_url,
|
||||
)?;
|
||||
}
|
||||
remote
|
||||
}
|
||||
Err(_) => self
|
||||
.repository
|
||||
.remote(REMOTE_NAME, &self.config.source_git_url)?,
|
||||
};
|
||||
|
||||
remote.fetch(&["refs/heads/*:refs/heads/*"], None, None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns latest commit on main branch.
|
||||
pub(crate) fn latest_commit(&self) -> Result<String> {
|
||||
let main_branch = self.repository.find_reference(&format!(
|
||||
"refs/remotes/{REMOTE_NAME}/{}",
|
||||
self.config.source_git_branch
|
||||
))?;
|
||||
let current_main_commit = main_branch.peel_to_commit()?;
|
||||
Ok(current_main_commit.id().to_string())
|
||||
}
|
||||
|
||||
/// Check nitter git URL with SHA for its commit state.
|
||||
///
|
||||
/// Example is `https://github.com/zedeus/nitter/commit/a92e79e`
|
||||
pub(crate) fn check_url(&mut self, url: &str) -> Result<CommitInfo> {
|
||||
match url.split('/').last() {
|
||||
Some(commit) => self.check_commit(commit),
|
||||
None => Ok(CommitInfo::UnknownCommit),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn check_commit(&mut self, commit_sha: &str) -> Result<CommitInfo> {
|
||||
if let Some(value) = self.commit_cache.get_mut(commit_sha) {
|
||||
if self.current_epoch.wrapping_sub(value.epoch) <= 1 {
|
||||
value.epoch = self.current_epoch;
|
||||
return Ok(value.result.clone());
|
||||
}
|
||||
}
|
||||
let result = self.check_commit_inner(commit_sha)?;
|
||||
self.commit_cache.insert(
|
||||
commit_sha.to_string(),
|
||||
CommitCacheValue {
|
||||
result: result.clone(),
|
||||
epoch: self.current_epoch,
|
||||
},
|
||||
);
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn check_commit_inner(&self, commit_sha: &str) -> Result<CommitInfo> {
|
||||
let commit = match self.repository.revparse_single(commit_sha) {
|
||||
Ok(commit) => commit,
|
||||
Err(_) => return Ok(CommitInfo::UnknownCommit),
|
||||
};
|
||||
|
||||
let main_branch = self.repository.find_reference(&format!(
|
||||
"refs/remotes/{REMOTE_NAME}/{}",
|
||||
self.config.source_git_branch
|
||||
))?;
|
||||
let current_main_commit = main_branch.peel_to_commit()?;
|
||||
|
||||
if current_main_commit.id() == commit.id() {
|
||||
return Ok(CommitInfo::Current);
|
||||
}
|
||||
|
||||
let mut revwalk = self.repository.revwalk()?;
|
||||
revwalk.push(main_branch.target().unwrap())?;
|
||||
|
||||
let is_in_main_branch =
|
||||
revwalk.any(|parent| parent.map(|v| v == commit.id()).unwrap_or_default());
|
||||
|
||||
match is_in_main_branch {
|
||||
true => Ok(CommitInfo::Outdated),
|
||||
false => Ok(CommitInfo::CustomBranch),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use entities::state::scanner::Config;
|
||||
|
||||
use super::fetch_git_state;
|
||||
use crate::version_check::{CommitInfo, VersionCheck};
|
||||
|
||||
#[test]
|
||||
fn test_git() {
|
||||
fetch_git_state(Config::test_defaults()).unwrap();
|
||||
fn test_git_commit_exists() {
|
||||
let mut checker = VersionCheck::new(Config::test_defaults()).unwrap();
|
||||
assert_eq!(
|
||||
CommitInfo::Outdated,
|
||||
checker
|
||||
.check_commit("064ec8808022abb071f93f0fc976a8aa123699dc",)
|
||||
.unwrap(),
|
||||
"long old hash should be outdated"
|
||||
);
|
||||
assert_eq!(
|
||||
CommitInfo::Outdated,
|
||||
checker
|
||||
.check_url("https://github.com/zedeus/nitter/commit/51b5485",)
|
||||
.unwrap(),
|
||||
"old URL should be outdated"
|
||||
);
|
||||
assert_eq!(
|
||||
CommitInfo::UnknownCommit,
|
||||
checker
|
||||
.check_url("https://github.com/zedeus/nitter/commit/3295bdb",)
|
||||
.unwrap(),
|
||||
"URL for unknown commit should be unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
CommitInfo::Outdated,
|
||||
checker.check_commit("064ec88",).unwrap(),
|
||||
"short old hash should be outdated"
|
||||
);
|
||||
assert_eq!(
|
||||
CommitInfo::UnknownCommit,
|
||||
checker
|
||||
.check_commit("064ec8808022abb071f93f0fc976b8aa123699dc",)
|
||||
.unwrap(),
|
||||
"long invalid hash should be Unknown"
|
||||
);
|
||||
// relies on https://github.com/zedeus/nitter/commits/tweets-parser/
|
||||
assert_eq!(
|
||||
CommitInfo::CustomBranch,
|
||||
checker
|
||||
.check_commit("c9b261a79303189f61ef5f5c6bf2c2600cdba792",)
|
||||
.unwrap(),
|
||||
"long invalid hash should be Unknown"
|
||||
);
|
||||
checker.update_remote().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
static REMOTE_NAME: &str = "origin";
|
||||
@@ -95,22 +95,22 @@
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
</td>
|
||||
{%- if host.version_state == "Missing" -%}
|
||||
{%- set version_order = "E" -%}
|
||||
{%- elif host.version_state == "UnknownCommit" -%}
|
||||
{%- set version_order = "D" -%}
|
||||
{%- elif host.version_state == "CustomBranch" -%}
|
||||
{%- set version_order = "C" -%}
|
||||
{%- elif host.version_state == "Current" -%}
|
||||
{%- set version_order = "A" -%}
|
||||
{%- elif host.version_state == "Outdated" -%}
|
||||
{%- set version_order = "B" -%}
|
||||
{%- endif -%}
|
||||
<td data-name="average">{%- if not host.__show_last_seen -%}{{host.ping_avg}}ms{% endif %}</td>
|
||||
<td>{{host.healthy_percentage_overall}}%</td>
|
||||
<td data-name="rss">{% if host.rss -%} <span role="img" aria-label="no rss">✅</span> {%- else -%} <span
|
||||
role="img" aria-label="has rss">❌</span> {%- endif -%}</td>
|
||||
{%- if host.healthy and not host.version -%}
|
||||
{%- set version = "Dmissing" -%}
|
||||
{%- elif host.version and not host.is_upstream -%}
|
||||
{%- set version = "Aforeign" -%}
|
||||
{%- elif host.is_latest_version -%}
|
||||
{%- set version = "Blatest" -%}
|
||||
{%- elif host.version and not host.is_latest_version -%}
|
||||
{%- set version = "Coutdated" -%}
|
||||
{%- else -%}
|
||||
{%- set version = "Eunknown" -%}
|
||||
{%- endif -%}
|
||||
<td data-name="version" data-sort="{{version}}{{host.version | default(value='')}}">
|
||||
<td data-name="version" data-sort="{{version_order}}{{host.version | default(value='')}}">
|
||||
{%- if host.version_url -%}
|
||||
<a rel="nofollow external" title="{{host.version}}" href="{{host.version_url}}">
|
||||
{{host.version | truncate(length=18) | default(value="missing version")}}
|
||||
@@ -118,13 +118,15 @@
|
||||
{%- else -%}
|
||||
<a rel="nofollow external" href="{{host.url ~ '/about'}}">missing version</a>
|
||||
{%- endif -%}
|
||||
{%- if version == "Dmissing" -%}
|
||||
{%- if host.version_state == "Missing" -%}
|
||||
<span class="ms-1 badge bg-warning">missing</span>
|
||||
{%- elif version == "Aforeign" -%}
|
||||
<span class="ms-1 badge bg-secondary">custom</span>
|
||||
{%- elif version == "Blatest" -%}
|
||||
{%- elif host.version_state == "UnknownCommit" -%}
|
||||
<span class="ms-1 badge bg-secondary">fork</span>
|
||||
{%- elif host.version_state == "CustomBranch" -%}
|
||||
<span class="ms-1 badge bg-secondary">experimental</span>
|
||||
{%- elif host.version_state == "Current" -%}
|
||||
<span class="ms-1 badge bg-success">latest</span>
|
||||
{# {%- elif version == "Coutdated" -%}
|
||||
{%- elif host.version_state == "Outdated" -%}
|
||||
<span class="ms-1 badge bg-info">outdated</span> #}
|
||||
{%- endif -%}
|
||||
</td>
|
||||
|
||||
@@ -129,6 +129,7 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
|
||||
let auto_mute = require_env_str("AUTO_MUTE")? == "true";
|
||||
let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?;
|
||||
let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?;
|
||||
let git_scratch_folder: String = require_env_str("GIT_SCRATCH_DIR")?;
|
||||
let cleanup_interval: u64 = require_env_str("CLEANUP_INTERVAL_S")?
|
||||
.parse()
|
||||
.expect("CLEANUP_INTERVAL_S must be a number");
|
||||
@@ -160,6 +161,7 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
|
||||
cleanup_interval: Duration::from_secs(cleanup_interval),
|
||||
error_retention_per_host,
|
||||
connectivity_path: String::from("/"),
|
||||
git_scratch_folder,
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user