fetch latest git branch

Closes #1
Closes #2
This commit is contained in:
4cuck committed 2023-08-04 17:31:43 +02:00
1 parent 533cc753db
commit bc156af23e
17 files changed
+284 -52

No files matched your search

+5 -1
View File
@@ -29,4 +29,8 @@ PING_RANGE_H="3"
# enable/disable startup scanning
DISABLE_STARTUP_SCAN=false
# don't emit errors when re-checking offline hosts
AUTO_MUTE=true
AUTO_MUTE=true
# branch to use for fetching the latest nitter commit
ORIGIN_SOURCE_GIT_BRANCH="master"
# URL to use for fetching the latest nitter commit
ORIGIN_SOURCE_GIT_URL="https://github.com/zedeus/nitter.git"
Generated
+56
View File
@@ -1235,6 +1235,21 @@ version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
[[package]]
name = "git2"
version = "0.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044"
dependencies = [
"bitflags 1.3.2",
"libc",
"libgit2-sys",
"log",
"openssl-probe",
"openssl-sys",
"url",
]
[[package]]
name = "globset"
version = "0.4.11"
@@ -1661,6 +1676,20 @@ version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "libgit2-sys"
version = "0.15.2+1.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa"
dependencies = [
"cc",
"libc",
"libssh2-sys",
"libz-sys",
"openssl-sys",
"pkg-config",
]
[[package]]
name = "libm"
version = "0.2.7"
@@ -1678,6 +1707,32 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "libssh2-sys"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee"
dependencies = [
"cc",
"libc",
"libz-sys",
"openssl-sys",
"pkg-config",
"vcpkg",
]
[[package]]
name = "libz-sys"
version = "1.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
@@ -2676,6 +2731,7 @@ version = "0.1.0"
dependencies = [
"chrono",
"entities",
"git2",
"regex",
"reqwest",
"scraper",
+3
View File
@@ -22,6 +22,7 @@ pub struct Model {
pub enabled: bool,
pub rss: bool,
pub version: Option<String>,
pub version_url: Option<String>,
/// Last time the url and enabled were updated, *not* the rss
pub updated: i64,
}
@@ -32,6 +33,7 @@ pub enum Column {
Domain,
Url,
Version,
VersionUrl,
Enabled,
Rss,
Updated,
@@ -62,6 +64,7 @@ impl ColumnTrait for Column {
Self::Domain => ColumnType::String(None).def(),
Self::Url => ColumnType::String(None).def(),
Self::Version => ColumnType::String(None).def().null(),
Self::VersionUrl => ColumnType::String(None).def().null(),
Self::Enabled => ColumnType::Integer.def(),
Self::Rss => ColumnType::Integer.def(),
Self::Updated => ColumnType::Integer.def(),
+7
View File
@@ -16,6 +16,7 @@ pub fn new() -> Cache {
Arc::new(RwLock::new(CacheData {
hosts: vec![],
last_update: Utc::now(),
latest_commit: String::new(),
}))
}
@@ -23,6 +24,7 @@ pub fn new() -> Cache {
pub struct CacheData {
pub hosts: Vec<CacheHost>,
pub last_update: DateTimeUtc,
pub latest_commit: String,
}
#[derive(Debug, Serialize)]
@@ -36,6 +38,11 @@ pub struct CacheHost {
pub ping_min: Option<i32>,
pub ping_avg: Option<i32>,
pub version: Option<String>,
pub version_url: Option<String>,
pub healthy: bool,
pub last_healthy: Option<DateTimeUtc>,
/// Whether the source is from the normal upstream repo
pub is_upstream: bool,
/// Whether the source is from the latest upstream commit
pub is_latest_version: bool,
}
+6
View File
@@ -26,6 +26,10 @@ pub struct Config {
pub ping_range: chrono::Duration,
/// don't emit errors for hosts which are already listed as down
pub auto_mute: bool,
/// Git URL for source fetching
pub source_git_url: String,
/// Git branch to fetch the current commit from
pub source_git_branch: String,
}
impl Config {
@@ -43,6 +47,8 @@ impl Config {
referer: String::from(""),
ping_range: chrono::Duration::hours(3),
auto_mute: true,
source_git_branch: String::from("master"),
source_git_url: String::from("https://github.com/zedeus/nitter.git"),
})
}
}
+2
View File
@@ -3,6 +3,7 @@ pub use sea_orm_migration::prelude::*;
mod m20220101_000001_create_table;
mod m20230729_010231_datetime_rowid;
mod m20230729_230909_datetime_int_host;
mod m20230803_154714_version_url;
pub struct Migrator;
@@ -13,6 +14,7 @@ impl MigratorTrait for Migrator {
Box::new(m20220101_000001_create_table::Migration),
Box::new(m20230729_010231_datetime_rowid::Migration),
Box::new(m20230729_230909_datetime_int_host::Migration),
Box::new(m20230803_154714_version_url::Migration),
]
}
}
@@ -0,0 +1,26 @@
use sea_orm_migration::{
prelude::*,
sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement},
};
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
let cmd = r#"ALTER TABLE "host" ADD COLUMN "version_url" text;"#;
let db = manager.get_connection();
db.execute_unprepared("BEGIN EXCLUSIVE").await?;
tracing::info!("adding version_url column..");
db.execute_unprepared(cmd).await?;
db.execute_unprepared("COMMIT TRANSACTION").await?;
db.execute_unprepared("VACUUM").await?;
Ok(())
}
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
panic!("Can't migrate down");
}
}
+1
View File
@@ -14,6 +14,7 @@ tokio = { workspace = true, features = ["full"] }
scraper = "0.17.1"
chrono = { workspace = true }
regex = { workspace = true }
git2 = "0.17.2"
[dev-dependencies]
tracing-test = { workspace = true }
+17 -3
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-only
use regex::{Regex, RegexBuilder};
use reqwest::Url;
use scraper::{Html, Selector};
use thiserror::Error;
@@ -15,6 +16,8 @@ pub enum AboutParseError {
NoCommitLinkFound,
#[error("Missing test! Found '{0}'")]
InvalidCommitFormat(String),
#[error("No valid href!")]
NoValidHref,
}
pub(crate) struct AboutParser {
@@ -23,9 +26,14 @@ pub(crate) struct AboutParser {
regex: Regex,
}
pub struct AboutParsed {
pub version_name: String,
pub url: String,
}
impl AboutParser {
/// Returns the text of the version <a> element of nitters about site
pub fn parse_about_version(&self, html: &str) -> Result<String> {
pub fn parse_about_version(&self, html: &str) -> Result<AboutParsed> {
let fragment = Html::parse_fragment(html);
// get all <p> elements
let p_elem = fragment
@@ -35,6 +43,8 @@ impl AboutParser {
let mut a_elems = p_elem.select(&self.selector_a);
let link = a_elems.next().ok_or(AboutParseError::NoCommitLinkFound)?;
let url = link.value().attr("href").map(|v|v.trim().to_owned())
.ok_or(AboutParseError::NoValidHref)?;
let link_text = link.text().fold(String::new(), |mut acc, text| {
acc.push_str(text);
acc
@@ -42,7 +52,10 @@ impl AboutParser {
if !self.regex.is_match(&link_text) {
return Err(AboutParseError::InvalidCommitFormat(link_text));
}
Ok(link_text)
Ok(AboutParsed {
url,
version_name: link_text
})
}
pub fn new() -> Self {
@@ -64,6 +77,7 @@ mod test {
let html = include_str!("../test_data/about.html");
let parser = AboutParser::new();
let res = parser.parse_about_version(html).unwrap();
assert_eq!(&res, "2023.07.22-72d8f35");
assert_eq!(&res.version_name, "2023.07.22-72d8f35");
assert_eq!(res.url, String::from("https://github.com/zedeus/nitter/commit/72d8f35"));
}
}
+14 -1
View File
@@ -14,6 +14,7 @@ use sea_orm::QueryOrder;
use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement};
use crate::LatestCheck;
use crate::version_check::fetch_git_state;
use crate::{Result, Scanner};
#[derive(Debug, FromQueryResult)]
@@ -49,10 +50,15 @@ impl Scanner {
/// Generate host stats and returns a new CacheData
pub(crate) async fn generate_cache_data(&self) -> Result<CacheData> {
let hosts = self.query_hosts_enabled().await?;
let config_c = self.inner.config.clone();
let current_version = tokio::task::spawn_blocking(move || {
fetch_git_state(config_c)
}).await.unwrap()?;
if hosts.is_empty() {
return Ok(CacheData {
hosts: vec![],
hosts: Vec::new(),
last_update: Utc::now(),
latest_commit: current_version.version,
});
}
@@ -95,6 +101,9 @@ impl Scanner {
// };
let points = (points * 100.0) as i32;
let latest_version = host.version_url.as_ref().map_or(false, |url|current_version.is_same_version(&url));
let is_upstream = host.version_url.as_ref().map_or(false, |url|current_version.is_same_repo(&url));
let host_ping_data = ping_data.remove(&host.id);
host_statistics.push(CacheHost {
last_healthy: last_healthy_check.remove(&host.id),
@@ -108,6 +117,9 @@ impl Scanner {
ping_min: host_ping_data.as_ref().and_then(|v|v.min),
ping_avg: host_ping_data.as_ref().and_then(|v|v.avg),
recent_pings: host_ping_data.map(|v|v.pings).unwrap_or_default(),
is_latest_version: latest_version,
is_upstream,
version_url: host.version_url,
})
}
host_statistics.sort_unstable_by_key(|k| k.points);
@@ -115,6 +127,7 @@ impl Scanner {
Ok(CacheData {
hosts: host_statistics,
last_update: time_now,
latest_commit: current_version.version,
})
}
+2 -1
View File
@@ -17,6 +17,7 @@ use tracing::instrument;
use crate::FetchError;
use crate::Result;
use crate::Scanner;
use crate::about_parser::AboutParsed;
impl Scanner {
/// Check uptime for host and create a new uptime entry in the database
@@ -147,7 +148,7 @@ impl Scanner {
}
/// Check nitter version
pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option<String> {
pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option<AboutParsed> {
url.set_path(&self.inner.config.about_path);
match self.fetch_url(url.as_str()).await {
Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) {
+13 -4
View File
@@ -31,6 +31,7 @@ mod about_parser;
mod cache_update;
mod instance_check;
mod instance_parser;
mod version_check;
const CAPTCHA_TEXT: &'static str = "Enable JavaScript and cookies to continue";
const CAPTCHA_CODE: u16 = 403;
@@ -58,6 +59,10 @@ pub enum ScannerError {
InstanceParse(#[from] instance_parser::InstaceListError),
#[error("Failed to fetch URL: {0}")]
FetchError(#[from] FetchError),
#[error("Failed fetcing git: {0}")]
GitFetch(#[from] git2::Error),
#[error("Couldn't find git branch")]
GitBranch,
}
#[derive(Error, Debug)]
@@ -263,25 +268,29 @@ impl Scanner {
};
// tracing::trace!(muted_host,instance=?instance,last_status=?last_status);
join_set.spawn(async move {
let (rss, version) = match Url::parse(&instance.url) {
let (rss, version, version_url) = match Url::parse(&instance.url) {
Err(_) => {
if !muted_host {
tracing::info!(url = instance.url, "Instance URL invalid");
}
(false, None)
(false, None, None)
}
Ok(mut url) => {
let rss = scanner_c.has_rss(&mut url, muted_host).await;
let version = scanner_c.nitter_version(&mut url, muted_host).await;
(rss, version)
match scanner_c.nitter_version(&mut url, muted_host).await {
Some(version) => (rss,Some(version.version_name),Some(version.url)),
None => (rss,None,None),
}
}
};
host::ActiveModel {
id: ActiveValue::NotSet,
domain: ActiveValue::Set(instance.domain),
url: ActiveValue::Set(instance.url),
enabled: ActiveValue::Set(true),
version: ActiveValue::Set(version),
version_url: ActiveValue::Set(version_url),
rss: ActiveValue::Set(rss),
updated: ActiveValue::Set(time.timestamp()),
}
+63
View File
@@ -0,0 +1,63 @@
use entities::state::scanner::ScannerConfig;
use git2::{Remote, Direction};
use crate::{Result, ScannerError};
pub(crate) struct CurrentVersion {
pub version: String,
config: ScannerConfig,
}
const SHORT_COMMIT_LEN: usize = 7;
impl CurrentVersion {
/// Whether a nitter version URL is from the same repo
pub fn is_same_repo(&self, url: &str) -> bool {
url.starts_with(self.config.source_git_url.trim_end_matches(".git"))
}
/// Whether a nitter version URL is pointing to the same version as this one
pub fn is_same_version(&self, url: &str) -> bool {
// look for last path segment (don't parse) and look if that matches
// from the start
self.is_same_repo(url) && match url.split('/').last() {
Some(other_version) => {
if other_version.len() == self.version.len() {
// we get the long version
other_version.starts_with(&self.version)
} else if other_version.len() == SHORT_COMMIT_LEN {
other_version.starts_with(&self.version[..SHORT_COMMIT_LEN])
} else {
// everything else is no short commit id..
false
}
}
None => false
}
}
}
pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result<CurrentVersion> {
let mut remote = Remote::create_detached(config.source_git_url.as_str())?;
remote.connect(Direction::Fetch)?;
let reference = format!("refs/heads/{}",config.source_git_branch);
let commit = remote.list()?.into_iter().find(|v|v.name() == &reference)
.map(|v|v.oid().to_string());
remote.disconnect()?;
Ok(commit.map(|commit|CurrentVersion { version: commit, config }).ok_or(ScannerError::GitBranch)?)
}
#[cfg(test)]
mod test {
use entities::state::scanner::Config;
use super::fetch_git_state;
#[test]
fn test_git() {
fetch_git_state(Config::test_defaults()).unwrap();
}
}
+5
View File
@@ -32,6 +32,7 @@ pub async fn instances(
}
pub async fn about(
State(ref cache): State<Cache>,
State(ref template): State<Arc<tera::Tera>>,
State(ref scanner_config): State<ScannerConfig>,
) -> Result<axum::response::Response> {
@@ -53,6 +54,10 @@ pub async fn about(
"ping_avg_interval_h",
&scanner_config.ping_range.num_hours(),
);
{
let guard = cache.read().map_err(|_| ServerError::MutexFailure)?;
context.insert("latest_commit", &guard.latest_commit);
}
let mut res = Html(template.render("about.html.j2", &context)?).into_response();
res.headers_mut().insert(
+3 -2
View File
@@ -14,12 +14,13 @@
<p><a href="/">Home</a></p>
<p>Uptime check interval: {{uptime_interval_s}}s.</p>
<p>Instance re-fetching interval, including RSS & version check: {{wiki_interval_s}}s.</p>
<p>Fetched latest commit: {{latest_commit}}</p>
<p>The following paths are checked per instance:</p>
<ul>
{% for path in checked_paths %}
{%- for path in checked_paths %}
<li><code>{{path}}</code></li>
{% endfor %}
{%- endfor %}
</ul>
<p>The same data can also be fetched as JSON from <code>/api/v1/instances</code></p>
+57 -40
View File
@@ -30,64 +30,81 @@
{% for host in instances -%}
<tr>
<td><a href="{{host.url}}">{{host.domain}}</a></td>
<td>{% if host.healthy -%} ✅ {% else -%} ❌ {% endif -%}</td>
<td>{% if host.healthy -%} ✅ {%- else -%} ❌ {%- endif -%}</td>
<td>
{% if host.recent_pings|length > 0 -%}
{% set height = 57 -%}
{% set width = 182 -%}
{% set amount = host.recent_pings|length -%}
{% set width_bar = width / amount -%}
{% set offset = 5 -%}
{% set error_size = height / 3 %}
{% if host.ping_max and host.ping_min -%}
{%- if host.recent_pings|length > 0 -%}
{%- set height = 57 -%}
{%- set width = 182 -%}
{%- set amount = host.recent_pings|length -%}
{%- set width_bar = width / amount -%}
{%- set offset = 5 -%}
{%- set error_size = height / 3 %}
{%- if host.ping_max and host.ping_min -%}
{# only one entry, so max = min => step_size would be 0 #}
{% if host.ping_max == host.ping_min -%}
{% set step_size = 5/(height-offset) -%}
{% else -%}
{% set step_size = (host.ping_max - host.ping_min)/(height-offset) -%}
{% endif -%}
{% else %}
{%- if host.ping_max == host.ping_min -%}
{%- set step_size = 5/(height-offset) -%}
{%- else -%}
{%- set step_size = (host.ping_max - host.ping_min)/(height-offset) -%}
{%- endif -%}
{%- else %}
{# no valid entries, only nulls #}
{% set step_size = 5/(height-offset) -%}
{% endif %}
{%- set step_size = 5/(height-offset) -%}
{%- endif -%}
{# <!-- step size {{step_size}} -->
<!-- ping_max {{host.ping_max}} -->
<!-- ping_min {{host.ping_min}} --> #}
{% if host.ping_min -%}
{%- if host.ping_min -%}
<svg viewBox="0 0 182 56" width="{{width - 1}}" height="{{height - 1}}">
{% if host.ping_avg and host.ping_min -%}
{% set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%}
{%- if host.ping_avg and host.ping_min -%}
{%- set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%}
<line x1="1" x2="181" y1="{{avg_pos}}" y2="{{avg_pos}}" stroke="black" stroke-width="0.72727275" stroke-linecap="butt" stroke-opacity="0.3"><title>Average</title></line>
{% endif -%}
{% for ping in host.recent_pings -%}
{% if ping -%}
{% set height_bar = (ping - host.ping_min )/step_size + offset -%}
{% set color_bar = "#a09df2" -%}
{% else %}
{% set height_bar = error_size -%}
{% set color_bar = "#cc0000" -%}
{% endif %}
{%- endif -%}
{%- for ping in host.recent_pings -%}
{%- if ping -%}
{%- set height_bar = (ping - host.ping_min )/step_size + offset -%}
{%- set color_bar = "#a09df2" -%}
{%- else -%}
{%- set height_bar = error_size -%}
{%- set color_bar = "#cc0000" -%}
{%- endif -%}
<rect x="{{(loop.index-1) * width_bar}}" y="{{height - height_bar - offset}}" width="{{width_bar}}" height="{{height_bar}}" fill="{{color_bar}}" fill-opacity="0.95"><title>{{ping | default(value="unhealthy") }}</title></rect>
{% endfor -%}
{%- endfor -%}
</svg>
{% else -%}
{%- else -%}
{# <!-- no healthy entries, show last time host was seen healthy --> #}
{% if host.last_healthy -%}
{%- if host.last_healthy -%}
Last seen healthy: {{host.last_healthy | truncate(length=10, end="") | replace(from="-", to=".")}}
{% else -%}
{%- else -%}
Never seen healthy.
{% endif -%}
{% endif -%}
{% else -%}
{%- endif -%}
{%- endif -%}
{%- else -%}
No recent data!
{% endif -%}
{%- endif -%}
</td>
<td>{{host.ping_avg}}</td>
<td>{% if host.rss -%} ✅ {% else -%} ❌ {% endif -%}</td>
<td>{{host.version}}</td>
<td>{% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%}</td>
<td>
{%- if host.version_url -%}
<a href="{{host.version_url}}">
{{host.version | default(value="missing version")}}
</a>
{%- else -%}
<a href="{{host.url ~ '/about'}}">missing version</a>
{%- endif -%}
{%- if host.healthy and not host.version -%}
<span class="ms-1 badge bg-warning">missing</span>
{%- elif host.version and not host.is_upstream -%}
<span class="ms-1 badge bg-secondary">foreign repo</span>
{%- elif host.is_latest_version -%}
<span class="ms-1 badge bg-success">latest</span>
{%- elif host.version and not host.is_latest_version -%}
<span class="ms-1 badge bg-info">outdated</span>
{%- endif -%}
</td>
<td>{{host.points}}</td>
</tr>
{% endfor -%}
{%- endfor -%}
</tbody>
</table>
</div>
+4
View File
@@ -105,6 +105,8 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
.collect();
let referer = require_env_str("REFERER")?;
let auto_mute = require_env_str("AUTO_MUTE")? == "true";
let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?;
let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?;
Ok(Arc::new(entities::state::scanner::Config {
list_fetch_interval: Duration::from_secs(instance_list_interval),
@@ -119,6 +121,8 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
referer,
ping_range: chrono::Duration::hours(ping_range as _),
auto_mute,
source_git_branch,
source_git_url,
}))
}