diff --git a/entities/src/health_check.rs b/entities/src/health_check.rs index 5a5ac37..e2a5aed 100644 --- a/entities/src/health_check.rs +++ b/entities/src/health_check.rs @@ -1,7 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 use sea_orm::{entity::prelude::*, FromQueryResult}; -use sea_query::{Query, Order, Alias, SimpleExpr}; +use sea_query::{Alias, Order, Query, SimpleExpr}; use serde::Serialize; use crate::check_errors; @@ -45,27 +45,31 @@ pub struct HealthyAmount { pub dead: i64, } -#[derive(Debug, FromQueryResult, Serialize)] -pub struct InstanceGraphEntry { - pub time: i64, - pub alive: i64, - pub dead: i64, -} - impl HealthyAmount { - /// Fetch health check graph data for all or selected hosts in the selected time range. - pub async fn fetch(db: &DatabaseConnection, from: Option, to: Option, hosts: Option<&[i32]>) -> Result,DbErr> { + /// Fetch health check graph data for all or selected hosts in the selected time range. + pub async fn fetch( + db: &DatabaseConnection, + from: Option, + to: Option, + hosts: Option<&[i32]>, + ) -> Result, DbErr> { let builder = db.get_database_backend(); let mut stmt: sea_query::SelectStatement = Query::select(); stmt.column(self::Column::Time) - .expr_as(SimpleExpr::Custom("SUM(healthy)".to_string()), Alias::new("alive")) - .expr_as(SimpleExpr::Custom("SUM(1-healthy)".to_string()), Alias::new("dead")) + .expr_as( + SimpleExpr::Custom("SUM(healthy)".to_string()), + Alias::new("alive"), + ) + .expr_as( + SimpleExpr::Custom("SUM(1-healthy)".to_string()), + Alias::new("dead"), + ) .group_by_col(self::Column::Time) .from(self::Entity); - if let (Some(from),Some(to)) = (from,to) { + if let (Some(from), Some(to)) = (from, to) { stmt.and_where(self::Column::Time.between(from.timestamp(), to.timestamp())); } - if let Some(hosts) = hosts { + if let Some(hosts) = hosts { stmt.and_where(self::Column::Host.is_in(hosts.iter().map(|v| *v))); } stmt.group_by_col(self::Column::Time) @@ -74,26 +78,6 @@ impl HealthyAmount { } } -impl InstanceGraphEntry { - /// Fetch health check graph data for all or selected hosts in the selected time range. - pub async fn fetch(db: &DatabaseConnection, to: Option) -> Result,DbErr> { - let builder = db.get_database_backend(); - let mut stmt: sea_query::SelectStatement = Query::select(); - stmt//.expr_as(SimpleExpr::Custom("strftime('%Y/%m/%d %H:%M:%S', datetime(time, 'unixepoch', 'utc'))".to_string()), Alias::new("time")) - .column(self::Column::Time) - .expr_as(SimpleExpr::Custom("SUM(healthy)".to_string()), Alias::new("alive")) - .expr_as(SimpleExpr::Custom("SUM(1-healthy)".to_string()), Alias::new("dead")) - .group_by_col(self::Column::Time) - .from(self::Entity); - if let Some(to) = to { - stmt.and_where(self::Column::Time.lte(to.timestamp())); - } - stmt.group_by_col(self::Column::Time) - .order_by(self::Column::Time, Order::Asc); - Self::find_by_statement(builder.build(&stmt)).all(db).await - } -} - #[cfg(test)] mod test { @@ -119,7 +103,9 @@ mod test { let time_3h = time_now .checked_sub_signed(chrono::Duration::days(14)) .unwrap(); - let hosts = vec![1,2,3]; - HealthyAmount::fetch(&db, Some(time_3h), Some(time_now),Some(&hosts)).await.unwrap(); + let hosts = vec![1, 2, 3]; + HealthyAmount::fetch(&db, Some(time_3h), Some(time_now), Some(&hosts)) + .await + .unwrap(); } -} \ No newline at end of file +} diff --git a/entities/src/instance_stats.rs b/entities/src/instance_stats.rs index eae4fda..32e5d4f 100644 --- a/entities/src/instance_stats.rs +++ b/entities/src/instance_stats.rs @@ -1,7 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 use sea_orm::{entity::prelude::*, FromQueryResult}; -use sea_query::{Query, SimpleExpr, Order, Alias}; +use sea_query::{Alias, Order, Query, SimpleExpr}; use serde::Serialize; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] @@ -75,24 +75,80 @@ pub struct StatsAmount { } impl StatsAmount { - /// Fetch health check graph data for all or selected hosts in the selected time range. - pub async fn fetch(db: &DatabaseConnection, from: DateTimeUtc, to: DateTimeUtc, hosts: Option<&[i32]>) -> Result,DbErr> { + /// Fetch health check graph data for all or selected hosts in the selected time range. + pub async fn fetch( + db: &DatabaseConnection, + from: DateTimeUtc, + to: DateTimeUtc, + hosts: Option<&[i32]>, + ) -> Result, DbErr> { let builder = db.get_database_backend(); - let columns = ["limited_accs","total_accs","total_requests","req_photo_rail","req_user_screen_name","req_search","req_list_tweets","req_user_media","req_tweet_detail","req_list","req_user_tweets","req_user_tweets_and_replies"]; + let columns = [ + "limited_accs", + "total_accs", + "total_requests", + "req_photo_rail", + "req_user_screen_name", + "req_search", + "req_list_tweets", + "req_user_media", + "req_tweet_detail", + "req_list", + "req_user_tweets", + "req_user_tweets_and_replies", + ]; let mut stmt: sea_query::SelectStatement = Query::select(); let col_stmt = stmt.column(self::Column::Time); for col in columns { - col_stmt.expr_as(SimpleExpr::Custom(format!("MAX({col})")), Alias::new(format!("{col}_max"))) - .expr_as(SimpleExpr::Custom(format!("CAST(ifnull(AVG({col}),0) as int)")), Alias::new(format!("{col}_avg"))); + col_stmt + .expr_as( + SimpleExpr::Custom(format!("MAX({col})")), + Alias::new(format!("{col}_max")), + ) + .expr_as( + SimpleExpr::Custom(format!("CAST(ifnull(AVG({col}),0) as int)")), + Alias::new(format!("{col}_avg")), + ); } - col_stmt.group_by_col(self::Column::Time) + col_stmt + .group_by_col(self::Column::Time) .from(self::Entity) .and_where(self::Column::Time.between(from.timestamp(), to.timestamp())); - if let Some(hosts) = hosts { + if let Some(hosts) = hosts { stmt.and_where(self::Column::Host.is_in(hosts.iter().map(|v| *v))); } stmt.group_by_col(self::Column::Time) .order_by(self::Column::Time, Order::Asc); - StatsAmount::find_by_statement(builder.build(&stmt)).all(db).await + StatsAmount::find_by_statement(builder.build(&stmt)) + .all(db) + .await } -} \ No newline at end of file +} + +#[derive(Debug, FromQueryResult, Serialize)] +pub struct StatsCSVEntry { + pub time: i64, + pub limited_accs_avg: i32, + pub total_accs_avg: i32, + pub total_requests_avg: i64, +} + +impl StatsCSVEntry { + /// Fetch health check graph data for all or selected hosts in the selected time range. + pub async fn fetch(db: &DatabaseConnection) -> Result, DbErr> { + let builder = db.get_database_backend(); + let columns = ["limited_accs", "total_accs", "total_requests"]; + let mut stmt: sea_query::SelectStatement = Query::select(); + stmt.column(self::Column::Time); + for col in columns { + stmt.expr_as( + SimpleExpr::Custom(format!("CAST (AVG({col}) as int)")), + Alias::new(format!("{col}_avg")), + ); + } + stmt.group_by_col(self::Column::Time).from(self::Entity); + stmt.group_by_col(self::Column::Time) + .order_by(self::Column::Time, Order::Asc); + Self::find_by_statement(builder.build(&stmt)).all(db).await + } +} diff --git a/server/src/api.rs b/server/src/api.rs index b01efe1..7dba5f1 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -3,8 +3,8 @@ use crate::{Result, ServerError}; use axum::response::IntoResponse; use axum::{extract::State, Json}; use chrono::{TimeZone, Utc}; -use entities::health_check; use entities::state::AppState; +use entities::{health_check, instance_stats}; use hyper::http::HeaderValue; use sea_orm::DatabaseConnection; use serde::Serialize; @@ -35,32 +35,33 @@ pub async fn instances( Ok(res) } -pub async fn graph( +pub async fn graph_csv_health( State(ref db): State, State(ref config): State>, ) -> Result { let start = std::time::Instant::now(); - let healthy_data = health_check::InstanceGraphEntry::fetch(db, None).await?; + let healthy_data = health_check::HealthyAmount::fetch(db, None, None, None).await?; let queried = std::time::Instant::now(); - let mut data = String::with_capacity(8*healthy_data.len()); + let mut data = String::with_capacity(8 * healthy_data.len()); data.push_str("Date,Healthy,Dead\n"); for entry in healthy_data { - let time = Utc.timestamp_opt(entry.time,0).unwrap().format("%Y-%m-%dT%H:%M:%SZ"); - writeln!(&mut data, "{},{},{}",time,entry.alive,entry.dead) - .map_err(|e|ServerError::CSV(e.to_string()))?; + let time = Utc + .timestamp_opt(entry.time, 0) + .unwrap() + .format("%Y-%m-%dT%H:%M:%SZ"); + writeln!(&mut data, "{time},{},{}", entry.alive, entry.dead) + .map_err(|e| ServerError::CSV(e.to_string()))?; } let formatted = std::time::Instant::now(); let query_time = queried - start; let format_time = formatted - queried; - tracing::debug!(?query_time,?format_time); + tracing::debug!(?query_time, ?format_time); let mut res = data.into_response(); - res.headers_mut().insert( - "content-type", - HeaderValue::from_str("text/csv").unwrap(), - ); + res.headers_mut() + .insert("content-type", HeaderValue::from_str("text/csv").unwrap()); res.headers_mut().insert( "cache-control", HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), @@ -70,4 +71,46 @@ pub async fn graph( HeaderValue::from_static("noindex, nofollow"), ); Ok(res) -} \ No newline at end of file +} + +pub async fn graph_csv_stats( + State(ref db): State, + State(ref config): State>, +) -> Result { + let start = std::time::Instant::now(); + let healthy_data = instance_stats::StatsCSVEntry::fetch(db).await?; + let queried = std::time::Instant::now(); + let mut data = String::with_capacity(8 * healthy_data.len()); + + data.push_str("Date,Tokens AVG,Limited Tokens AVG,Requests AVG\n"); + + for entry in healthy_data { + let time = Utc + .timestamp_opt(entry.time, 0) + .unwrap() + .format("%Y-%m-%dT%H:%M:%SZ"); + writeln!( + &mut data, + "{time},{},{},{}", + entry.total_accs_avg, entry.limited_accs_avg, entry.total_requests_avg + ) + .map_err(|e| ServerError::CSV(e.to_string()))?; + } + let formatted = std::time::Instant::now(); + let query_time = queried - start; + let format_time = formatted - queried; + tracing::debug!(?query_time, ?format_time); + + let mut res = data.into_response(); + res.headers_mut() + .insert("content-type", HeaderValue::from_str("text/csv").unwrap()); + res.headers_mut().insert( + "cache-control", + HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), + ); + res.headers_mut().insert( + "X-Robots-Tag", + HeaderValue::from_static("noindex, nofollow"), + ); + Ok(res) +} diff --git a/server/src/lib.rs b/server/src/lib.rs index 18b04fe..7a6a99f 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -145,11 +145,11 @@ pub async fn start( ServeDir::new("server/static").append_index_html_on_directories(false), ) .route("/api/v1/instances", get(api::instances)) - .route("/api/graph", get(api::graph)) + .route("/api/csv/health", get(api::graph_csv_health)) + .route("/api/csv/stats", get(api::graph_csv_stats)) .nest(ADMIN_OVERVIEW_URL, Router::new() .route("/", get(admin::overview)) .route("/instance/:instance", get(admin::instance_view)) - // .route("/history/:host", get(admin::history_view)) .route("/api/history/:instance", post(admin::history_json_specific)) .route("/api/history", post(admin::history_json)) .route("/login", get(admin::login_view).post(admin::login).route_layer(rate_limit_layer)) @@ -157,7 +157,6 @@ pub async fn start( // .layer(ServiceBuilder::new().layer(SetResponseHeaderLayer::overriding(header::CACHE_CONTROL, "must-revalidate"))) .layer(session_service) ) - // .route("/admin", get(admin::view)) .route("/about", get(website::about)) .route("/rip", get(website::rip)) .route( @@ -231,7 +230,8 @@ impl axum::response::IntoResponse for ServerError { StatusCode::FORBIDDEN, Cow::Borrowed("Missing permission to access this resource"), ), - CSV(_ )| MutexFailure | Templating(_) | DBError(_) | SessionError(_) | HostNotFound(_) => ( + CSV(_) | MutexFailure | Templating(_) | DBError(_) | SessionError(_) + | HostNotFound(_) => ( StatusCode::INTERNAL_SERVER_ERROR, Cow::Borrowed("Internal Server Error"), ), diff --git a/server/static/history.2.js b/server/static/history.2.js index f44867f..d6ad1cb 100644 --- a/server/static/history.2.js +++ b/server/static/history.2.js @@ -1,57 +1,129 @@ -document.addEventListener("DOMContentLoaded", function(event) { +document.addEventListener("DOMContentLoaded", function (event) { var loadGraphBtn = document.getElementById("loadGraphBtn"); if (loadGraphBtn) { - loadGraphBtn.addEventListener("click", function() { - var overlay = document.getElementById("overlay"); - if (overlay) { - overlay.style.display = "none"; - loadGraph(); - } + loadGraphBtn.addEventListener("click", function () { + var elements = document.querySelectorAll('.graph'); + elements.forEach(function (element) { + element.style.display = 'block'; + }); + loadGraphBtn.style.display = "none"; + loadHealthGraph(); + loadStatsGraph(); }); } }); -async function loadGraph() { - let graphDiv = document.getElementById('history'); - try { - let g = new Dygraph( - graphDiv, - "/api/graph", - { - title: 'Historic Instance Healthiness', - showRangeSelector: true, - width: '100%', - rangeSelectorPlotFillColor: 'MediumSlateBlue', - rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', - colorValue: 0.9, - fillAlpha: 0.4, - colors: ['#008000', '#ffa500'], - } - ); - g.ready(function() { - g.setAnnotations([ - { - series: "Healthy", - x: "2023-08-15T07:10:17Z", - shortText: "G", - text: "First API Change" - }, - { - series: "Dead", - x: "2023-10-21T14:37:44Z", - shortText: "C", - text: "Wiki Cleanup" - }, - { - series: "Healthy", - x: "2024-01-25T15:24:16Z", - shortText: "R", - text: "API Shutdown" - } - ]); - }); - } catch (error) { - console.error('Failed to fetch data:', error); - graphDiv.textContent = "Failed to load data."; - } +async function loadHealthGraph() { + let graphDiv = document.getElementById('graph-health'); + try { + let g = new Dygraph( + graphDiv, + "/api/csv/health", + { + title: 'Historic Instance Healthiness', + showRangeSelector: true, + width: '100%', + rangeSelectorPlotFillColor: 'MediumSlateBlue', + rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', + colorValue: 0.9, + fillAlpha: 0.4, + colors: ['#008000', '#ffa500'], + ylabel: 'Nitter Instances', + } + ); + g.ready(function () { + g.setAnnotations([ + { + series: "Healthy", + x: "2023-08-15T07:10:17Z", + shortText: "G", + text: "First API Change" + }, + { + series: "Dead", + x: "2023-10-21T14:37:44Z", + shortText: "C", + text: "Wiki Cleanup" + }, + { + series: "Healthy", + x: "2024-01-25T15:24:16Z", + shortText: "R", + text: "API Shutdown" + } + ]); + }); + } catch (error) { + console.error('Failed to fetch data:', error); + graphDiv.textContent = "Failed to load data."; + } +} +async function loadStatsGraph() { + let graphDiv = document.getElementById('graph-stats'); + try { + let g = new Dygraph( + graphDiv, + "/api/csv/stats", + { + title: 'Historic Average Instance Statistics', + showRangeSelector: true, + width: '100%', + rangeSelectorPlotFillColor: 'MediumSlateBlue', + rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', + colorValue: 0.9, + fillAlpha: 0.4, + colors: ['#008000', '#ffa500', '#6495ED'], + series: { + 'Tokens AVG': { + axis: 'y' + }, + 'Limited Tokens AVG': { + axis: 'y' + }, + 'Requests AVG': { + axis: 'y2' + }, + }, + axes: { + y: { + axisLabelWidth: 60, + logscale: "y log scale", + }, + y2: { + // set axis-related properties here + drawGrid: false, + drawAxis: false, + logscale: "y log scale", + }, + }, + ylabel: 'Tokens', + y2label: 'Requests', + } + ); + g.ready(function () { + g.setAnnotations([ + { + series: "Healthy", + x: "2023-08-15T07:10:17Z", + shortText: "G", + text: "First API Change" + }, + { + series: "Dead", + x: "2023-10-21T14:37:44Z", + shortText: "C", + text: "Wiki Cleanup" + }, + { + series: "Healthy", + x: "2024-01-25T15:24:16Z", + shortText: "R", + text: "API Shutdown" + } + ]); + }); + } catch (error) { + console.error('Failed to fetch data:', error); + graphDiv.textContent = "Failed to load data."; + } } \ No newline at end of file diff --git a/server/templates/instances.html.j2 b/server/templates/instances.html.j2 index 635c2dd..1b2c2c6 100644 --- a/server/templates/instances.html.j2 +++ b/server/templates/instances.html.j2 @@ -1,124 +1,136 @@ {# SPDX-License-Identifier: AGPL-3.0-only #} - - - - - - - - - Nitter Instance Health - - + + + Nitter Instance Health + + +

Nitter Instance Uptime & Health

+

About

-

Please use the API for bots. Please do NOT use these instances for scraping, host nitter yourself.

+

Please use the API for bots. Please do NOT use these instances for + scraping, host nitter yourself.

Last Updated {{last_updated}} UTC.

Customize the visible columns down below.

- + - - - - - - - - - - + + + + + + + + + + - - + + {% for host in instances -%} - - - - + + + - - - - {%- if host.healthy and not host.version -%} - {%- set version = "Dmissing" -%} - {%- elif host.version and not host.is_upstream -%} - {%- set version = "Aforeign" -%} - {%- elif host.is_latest_version -%} - {%- set version = "Blatest" -%} - {%- elif host.version and not host.is_latest_version -%} - {%- set version = "Coutdated" -%} - {%- else -%} - {%- set version = "Eunknown" -%} - {%- endif -%} - - - + + + + + {%- if host.healthy and not host.version -%} + {%- set version = "Dmissing" -%} + {%- elif host.version and not host.is_upstream -%} + {%- set version = "Aforeign" -%} + {%- elif host.is_latest_version -%} + {%- set version = "Blatest" -%} + {%- elif host.version and not host.is_latest_version -%} + {%- set version = "Coutdated" -%} + {%- else -%} + {%- set version = "Eunknown" -%} + {%- endif -%} + + + {%- endfor -%} - +
InstanceCountryHealthyHealth HistoryAverage TimeAll Time %RSSNitter VersionConnectivityPointsInstanceCountryHealthyHealth HistoryAverage TimeAll Time %RSSNitter VersionConnectivityPoints
{{host.domain}}{{host.country}} - {%- if host.is_bad_host -%} -
❓
- {%- elif host.healthy -%} - ✅ - {%- else -%} - ❌ - {%- endif -%}		 - {%- set height = 28 -%} - {%- set width = 110 -%} - {%- set width_bar = 5 -%} - {%- set offset = 5 -%} - {%- if not host.__show_last_seen -%} + {{host.domain}}{{host.country}} + {%- if host.is_bad_host -%} +
❓
+ {%- elif host.healthy -%} + ✅ + {%- else -%} + ❌ + {%- endif -%} + 		+ {%- set height = 28 -%} + {%- set width = 110 -%} + {%- set width_bar = 5 -%} + {%- set offset = 5 -%} + {%- if not host.__show_last_seen -%} {%- for check in host.recent_checks -%} - {%- if check.1 -%} - {%- set title = "Healthy " ~ check.0 -%} - {%- set color_bar = "#2fcc66" -%} - {%- else -%} - {%- set title = "Unhealthy " ~ check.0 -%} - {%- set color_bar = "#ff6225" -%} - {%- endif -%} - {{title}} + {%- if check.1 -%} + {%- set title = "Healthy " ~ check.0 -%} + {%- set color_bar = "#2fcc66" -%} + {%- else -%} + {%- set title = "Unhealthy " ~ check.0 -%} + {%- set color_bar = "#ff6225" -%} + {%- endif -%} + + {{title}} + {%- endfor -%} - {%- else -%} + {%- else -%} {# #} {%- if host.last_healthy -%} - LSH: {{host.last_healthy | truncate(length=16, end="") | replace(from="-", to=".") | replace(from="T", to=" ")}} + LSH: {{host.last_healthy | truncate(length=16, end="") | replace(from="-", to=".") | replace(from="T", + to=" ")}} UTC {%- else -%} Never seen healthy. {%- endif -%} - {%- endif -%} - {%- if not host.__show_last_seen -%}{{host.ping_avg}}ms{% endif %}{{host.healthy_percentage_overall}}%{% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%} - {%- if host.version_url -%} - - {{host.version | truncate(length=18) | default(value="missing version")}} - - {%- else -%} - missing version - {%- endif -%} - {%- if version == "Dmissing" -%} - missing - {%- elif version == "Aforeign" -%} - foreign repo - {%- elif version == "Blatest" -%} - latest - {%- elif version == "Coutdated" -%} - outdated {%- endif -%} - {{host.connectivity}}{{host.points}}{%- if not host.__show_last_seen -%}{{host.ping_avg}}ms{% endif %}{{host.healthy_percentage_overall}}%{% if host.rss -%} ✅ {%- else -%} ❌ {%- endif -%} + {%- if host.version_url -%} + + {{host.version | truncate(length=18) | default(value="missing version")}} + + {%- else -%} + missing version + {%- endif -%} + {%- if version == "Dmissing" -%} + missing + {%- elif version == "Aforeign" -%} + foreign repo + {%- elif version == "Blatest" -%} + latest + {%- elif version == "Coutdated" -%} + outdated + {%- endif -%} + {{host.connectivity}}{{host.points}}
@@ -156,9 +168,10 @@
- - - - - - + + + + + + + \ No newline at end of file diff --git a/server/templates/rip.html.j2 b/server/templates/rip.html.j2 index 1093cdf..fa4cecd 100644 --- a/server/templates/rip.html.j2 +++ b/server/templates/rip.html.j2 @@ -1,39 +1,28 @@ {# SPDX-License-Identifier: AGPL-3.0-only #} - - - - - - - - - - Nitter Is Dead - - + + + Nitter Is Dead + + +

All Nitter public instances shut down.

There are no public instances left for you to use.

@@ -43,32 +32,32 @@

In January 2024, the Twitter API used by Nitter to fetch data from Twitter got shut down.

-

Without this, there is no way to run instances on the scale needed for the public instances that got listed here.

+

Without this, there is no way to run instances on the scale needed for the public instances that got listed here. +

That's unfortunate. But I still want to use Nitter !

If you want to continue running Nitter, you can do so using a normal Twitter account.

Do note however, that this cannot and will not scale, and is in a legal grey-area. Proceed at your own risk.

-

See here for setting up nitter if you feel brave enough and don't fear the command line.

+

See here for setting up + Nitter if you feel brave enough and don't fear the command line.

- +
-
- -
-
+ +
+
+
+
+
-

Last Updated {{last_updated}} UTC.

Instances About

- - - - + + + + + \ No newline at end of file