Mirror-bot/nitter-status
1
0 Fork 0
mirror of https://github.com/4cuck/nitter-status.git synced 2026-10-26 00:52:25 +00:00
Code Issues Projects Releases Activity

cleanup

Browse Source
This commit is contained in:
4cuck committed 2023-10-16 23:32:10 +02:00
1 parent 759963e7b2
commit 0342427835
21 files changed
+318 -206
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

entities/src/host.rs
+3 -6
View File
@@ -1,10 +1,7 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
use sea_orm::{
entity::prelude::*,
sea_query::{Expr, SimpleExpr},
};
use serde::{Serialize, Deserialize};
use sea_orm::entity::prelude::*;
use serde::Serialize;
#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
pub struct Entity;
@@ -32,7 +29,7 @@ pub struct Model {
pub updated: i64,
}
#[derive(Copy, Clone, Debug, PartialEq ,Eq, EnumIter, DeriveActiveEnum, Serialize)]
#[derive(Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize)]
#[sea_orm(rs_type = "i32", db_type = "Integer")]
pub enum Connectivity {
#[sea_orm(num_value = 0)]
entities/src/state/error_cache.rs
+1 -3
View File
@@ -1,5 +1,3 @@
use std::collections::VecDeque;
use chrono::Utc;
use sea_orm::prelude::DateTimeUtc;
use serde::Serialize;
@@ -41,4 +39,4 @@ impl HostError {
http_status: Some(http_status as _),
}
}
}
}
entities/src/state/mod.rs
+2 -2
View File
@@ -10,9 +10,9 @@ use serde::Serialize;
use crate::host::Connectivity;
pub mod scanner;
/// Log for recent host errors
pub mod error_cache;
pub mod scanner;
pub type Cache = RwLock<CacheData>;
@@ -62,7 +62,7 @@ pub struct CacheHost {
/// Country from the wiki
pub country: String,
/// Last health checks time formatted, healthy
pub recent_checks: Vec<(String,bool)>,
pub recent_checks: Vec<(String, bool)>,
/// Percentage of healthy checks since first seen
pub healthy_percentage_overall: u8,
pub connectivity: Option<Connectivity>,
migration/src/lib.rs
-2
View File
@@ -8,8 +8,6 @@ mod m20230829_201916_country;
mod m20230914_231514_connectivity;
mod m20231011_231223_errors;
pub struct Migrator;
#[async_trait::async_trait]
migration/src/m20230729_010231_datetime_rowid.rs
+1 -3
View File
@@ -2,9 +2,7 @@ use std::borrow::Cow;
use sea_orm_migration::{
prelude::*,
sea_orm::{
prelude::DateTimeUtc, DbBackend, DeriveEntityModel, FromQueryResult, RuntimeErr, Statement,
},
sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, RuntimeErr, Statement},
};
#[derive(DeriveMigrationName)]
migration/src/m20230729_230909_datetime_int_host.rs
+1 -1
View File
@@ -90,4 +90,4 @@ impl MigrationTrait for Migration {
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
panic!("Can't migrate down");
}
}
}
migration/src/m20230803_154714_version_url.rs
+1 -4
View File
@@ -1,7 +1,4 @@
use sea_orm_migration::{
prelude::*,
sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement},
};
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
migration/src/m20230829_201916_country.rs
+1 -1
View File
@@ -19,4 +19,4 @@ impl MigrationTrait for Migration {
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
panic!("Can't migrate down");
}
}
}
migration/src/m20230914_231514_connectivity.rs
+1 -1
View File
@@ -19,4 +19,4 @@ impl MigrationTrait for Migration {
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
panic!("Can't migrate down");
}
}
}
migration/src/m20231011_231223_errors.rs
+1 -1
View File
@@ -27,4 +27,4 @@ impl MigrationTrait for Migration {
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
panic!("Can't migrate down");
}
}
}
scanner/src/about_parser.rs
-1
View File
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-only
use regex::{Regex, RegexBuilder};
use reqwest::Url;
use scraper::{Html, Selector};
use thiserror::Error;
scanner/src/cache_update.rs
+27 -23
View File
@@ -156,7 +156,7 @@ impl Scanner {
is_bad_host,
country: host.country,
healthy_percentage_overall: healthy_percentage_total.remove(&host.id).unwrap_or(0),
recent_checks: self.query_latest_health_checks(22,host.id).await?,
recent_checks: self.query_latest_health_checks(22, host.id).await?,
})
}
host_statistics.sort_unstable_by_key(|k| k.points);
@@ -318,19 +318,18 @@ impl Scanner {
}
/// Query total up percentage for all hosts
async fn query_healthy_percentage(
&self,
) -> Result<HashMap<i32, u8>> {
let stats: Vec<HostHealthyPercentage> = HostHealthyPercentage::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite,
r#"SELECT u.host, CAST(AVG(healthy) * 100 as INT) as healthy FROM health_check u
async fn query_healthy_percentage(&self) -> Result<HashMap<i32, u8>> {
let stats: Vec<HostHealthyPercentage> =
HostHealthyPercentage::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite,
r#"SELECT u.host, CAST(AVG(healthy) * 100 as INT) as healthy FROM health_check u
JOIN host h ON h.id = u.host
WHERE h.enabled = true
GROUP BY u.host"#,
[],
))
.all(&self.inner.db)
.await?;
[],
))
.all(&self.inner.db)
.await?;
let stats: HashMap<_, _> = stats.into_iter().map(|v| (v.host, v.healthy)).collect();
Ok(stats)
}
@@ -341,28 +340,33 @@ impl Scanner {
// How many to retrieve
amount: i32,
host: i32,
) -> Result<Vec<(String,bool)>> {
) -> Result<Vec<(String, bool)>> {
#[derive(Debug, FromQueryResult)]
pub struct HostHealthCheck {
healthy: bool,
time: i64,
}
let health_checks: Vec<HostHealthCheck> = HostHealthCheck::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite,
r#"SELECT healthy, time FROM health_check u
let health_checks: Vec<HostHealthCheck> =
HostHealthCheck::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite,
r#"SELECT healthy, time FROM health_check u
JOIN host h ON h.id = u.host
WHERE h.enabled = true AND host = $1
ORDER BY time DESC
LIMIT $2"#,
[host.into(), amount.into()],
))
.all(&self.inner.db)
.await?;
[host.into(), amount.into()],
))
.all(&self.inner.db)
.await?;
// transform to correct time format
let health_checks: Vec<_> = health_checks.into_iter().rev().map(|entry|{
let time = Utc.timestamp_opt(entry.time, 0).unwrap();
(time.format("%Y.%m.%d %H:%M").to_string(),entry.healthy)
}).collect();
let health_checks: Vec<_> = health_checks
.into_iter()
.rev()
.map(|entry| {
let time = Utc.timestamp_opt(entry.time, 0).unwrap();
(time.format("%Y.%m.%d %H:%M").to_string(), entry.healthy)
})
.collect();
Ok(health_checks)
}
}
scanner/src/instance_check.rs
+21 -10
View File
@@ -1,14 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! Instance health/uptime checking code
use std::net::IpAddr;
use std::time::Instant;
use chrono::Utc;
use entities::{health_check, check_errors};
use entities::host::Connectivity;
use entities::state::error_cache::HostError;
use entities::{check_errors, health_check};
use entities::{host, prelude::*};
use reqwest::{Url, ClientBuilder};
use reqwest::Url;
use sea_orm::prelude::DateTimeUtc;
use sea_orm::ColumnTrait;
use sea_orm::EntityTrait;
@@ -18,7 +16,6 @@ use tokio::task::JoinSet;
use tracing::instrument;
use crate::about_parser::AboutParsed;
use crate::FetchError;
use crate::Result;
use crate::Scanner;
@@ -63,8 +60,13 @@ impl Scanner {
if !muted {
tracing::error!(error=?e, url=host.url,"failed to parse instance URL");
}
self.insert_failed_health_check(host.id, now,HostError::new_message(format!("Not a valid URL")), None)
.await;
self.insert_failed_health_check(
host.id,
now,
HostError::new_message(format!("Not a valid URL")),
None,
)
.await;
return;
}
Ok(v) => v,
@@ -83,8 +85,13 @@ impl Scanner {
"couldn't ping host: {e}, marking as dead"
);
}
self.insert_failed_health_check(host.id, now, e.to_host_error(), Some(took_ms as _))
.await;
self.insert_failed_health_check(
host.id,
now,
e.to_host_error(),
Some(took_ms as _),
)
.await;
}
Ok((http_code, content)) => {
if !muted {
@@ -121,7 +128,11 @@ impl Scanner {
self.insert_failed_health_check(
host.id,
now,
HostError::new(format!("profile content mismatch"), content, http_code),
HostError::new(
format!("profile content mismatch"),
content,
http_code,
),
Some(took_ms as _),
)
.await;
scanner/src/instance_parser.rs
+4 -5
View File
@@ -196,10 +196,10 @@ impl InstanceParser {
#[cfg(test)]
mod test {
use std::{collections::HashMap, io::BufWriter};
use std::io::Write;
use tracing_test::traced_test;
use csv;
use std::io::Write;
use std::{collections::HashMap, io::BufWriter};
use tracing_test::traced_test;
use super::*;
#[test]
@@ -208,11 +208,10 @@ mod test {
let html = include_str!("../test_data/instancelist.html");
let parser = InstanceParser::new();
let res = parser.parse_instancelist(html, &[], "", true).unwrap();
// writeback for new tests
// write_data(res.values());
// adjust when updating the test html
let expected: HashMap<String, InstanceParsed> = expected_data()
.into_iter()
scanner/src/lib.rs
+81 -34
View File
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
use std::{
net::IpAddr,
sync::{Arc, Mutex},
time::Instant, net::IpAddr,
time::Instant,
};
use about_parser::AboutParser;
@@ -9,14 +10,14 @@ use chrono::Utc;
use entities::{
host,
prelude::*,
state::{scanner::ScannerConfig, Cache, AppState, error_cache::HostError},
state::{error_cache::HostError, scanner::ScannerConfig, AppState},
};
use instance_parser::InstanceParser;
use profile_parser::ProfileParser;
use regex::{Regex, RegexBuilder};
use reqwest::{
header::{HeaderMap, HeaderValue},
Client, Url, ClientBuilder,
Client, ClientBuilder, Url,
};
use sea_orm::{
sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait,
@@ -94,9 +95,15 @@ impl FetchError {
fn to_host_error(self) -> HostError {
match self {
FetchError::Reqwest(e) => HostError::new_message(e.to_string()),
FetchError::HttpResponseStatus(http_status, _code_msg, http_body) => HostError::new(format!("failed to fetch"), http_body, http_status),
FetchError::KnownHttpResponseStatus(http_status, _) => HostError::new_without_body(self.to_string(), http_status),
FetchError::RetrievingBody(_url, reqwest_error) => HostError::new_message(reqwest_error.to_string()),
FetchError::HttpResponseStatus(http_status, _code_msg, http_body) => {
HostError::new(format!("failed to fetch"), http_body, http_status)
}
FetchError::KnownHttpResponseStatus(http_status, _) => {
HostError::new_without_body(self.to_string(), http_status)
}
FetchError::RetrievingBody(_url, reqwest_error) => {
HostError::new_message(reqwest_error.to_string())
}
FetchError::Captcha => HostError::new_message(format!("Captcha detected")),
}
}
@@ -150,7 +157,7 @@ impl Scanner {
for header in HEADERS {
headers.insert(header[0], HeaderValue::from_static(header[1]));
}
let user_agent = format!("nitter-status (+{}/about)",config.website_url);
let user_agent = format!("nitter-status (+{}/about)", config.website_url);
let http_client = Client::builder()
.cookie_store(true)
.brotli(true)
@@ -164,12 +171,22 @@ impl Scanner {
http_client
}
async fn new(db: DatabaseConnection, config: ScannerConfig, app_state: AppState) -> Result<Self> {
async fn new(
db: DatabaseConnection,
config: ScannerConfig,
app_state: AppState,
) -> Result<Self> {
let mut builder_regex_rss = RegexBuilder::new(&config.rss_content);
builder_regex_rss.case_insensitive(true);
let http_client = Self::client_builder(&config);
let client_ipv4 = Scanner::client_builder(&config).local_address("0.0.0.0".parse::<IpAddr>().unwrap()).build().unwrap();
let client_ipv6 = Scanner::client_builder(&config).local_address("::".parse::<IpAddr>().unwrap()).build().unwrap();
let client_ipv4 = Scanner::client_builder(&config)
.local_address("0.0.0.0".parse::<IpAddr>().unwrap())
.build()
.unwrap();
let client_ipv6 = Scanner::client_builder(&config)
.local_address("::".parse::<IpAddr>().unwrap())
.build()
.unwrap();
let scanner = Self {
inner: Arc::new(InnerScanner {
db,
@@ -305,7 +322,12 @@ impl Scanner {
let connectivity = scanner_c.check_connectivity(url.as_str()).await;
let rss = scanner_c.has_rss(&mut url, muted_host).await;
match scanner_c.nitter_version(&mut url, muted_host).await {
Some(version) => (connectivity, rss, Some(version.version_name), Some(version.url)),
Some(version) => (
connectivity,
rss,
Some(version.version_name),
Some(version.url),
),
None => (connectivity, rss, None, None),
}
}
@@ -366,18 +388,26 @@ impl Scanner {
/// Check ipv4/6 connectivity of host
async fn check_connectivity(&self, url: &str) -> Option<host::Connectivity> {
let ipv4 = self.inner.client_ipv4.get(url).send().await.map_or(false, |res|{
res.status().is_success()
});
let ipv6 = self.inner.client_ipv6.get(url).send().await.map_or(false, |res|{
res.status().is_success()
});
let ipv4 = self
.inner
.client_ipv4
.get(url)
.send()
.await
.map_or(false, |res| res.status().is_success());
let ipv6 = self
.inner
.client_ipv6
.get(url)
.send()
.await
.map_or(false, |res| res.status().is_success());
match (ipv4,ipv6) {
(true,true) => Some(host::Connectivity::All),
(false,true) => Some(host::Connectivity::IPv6),
(true,false) => Some(host::Connectivity::IPv4),
(false,false) => None,
match (ipv4, ipv6) {
(true, true) => Some(host::Connectivity::All),
(false, true) => Some(host::Connectivity::IPv6),
(true, false) => Some(host::Connectivity::IPv4),
(false, false) => None,
}
}
@@ -457,11 +487,11 @@ mod test {
use super::*;
use chrono::Duration;
use entities::health_check;
use entities::state::scanner::Config;
use migration::MigratorTrait;
use sea_orm::{ConnectOptions, Database};
use tokio::{fs::File, io::AsyncWriteExt};
use entities::health_check;
async fn db_init() -> DatabaseConnection {
let db = Database::connect(ConnectOptions::new(
@@ -469,8 +499,7 @@ mod test {
))
.await
.unwrap();
migration::Migrator::up(&db, None)
.await.unwrap();
migration::Migrator::up(&db, None).await.unwrap();
db
}
@@ -482,7 +511,7 @@ mod test {
let db = db_init().await;
let time: chrono::DateTime<Utc> = Utc::now();
for v in 0..365 {
let entry_time = time.checked_sub_signed(Duration::minutes(15*v)).unwrap();
let entry_time = time.checked_sub_signed(Duration::minutes(15 * v)).unwrap();
health_check::ActiveModel {
time: ActiveValue::Set(entry_time.timestamp()),
host: ActiveValue::Set(208692),
@@ -491,7 +520,8 @@ mod test {
response_code: ActiveValue::Set(Some(200)),
}
.insert(&db)
.await.unwrap();
.await
.unwrap();
}
}
@@ -499,7 +529,9 @@ mod test {
#[ignore]
async fn update_instacelist() {
let db = db_init().await;
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
.await
.unwrap();
let res = scanner.fetch_instancelist().await.unwrap();
let mut file = File::create("test_data/instancelist.html").await.unwrap();
file.write_all(&res.as_bytes()).await.unwrap();
@@ -509,7 +541,9 @@ mod test {
#[ignore]
async fn fetch_test() {
let db = db_init().await;
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
.await
.unwrap();
let (_, res) = scanner.fetch_url("example.com/jack").await.unwrap();
let mut file = File::create("test_data/blocked.html").await.unwrap();
file.write_all(&res.as_bytes()).await.unwrap();
@@ -519,7 +553,9 @@ mod test {
#[ignore]
async fn stats_test() {
let db = db_init().await;
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
.await
.unwrap();
dbg!(scanner.generate_cache_data().await.unwrap());
}
@@ -527,10 +563,21 @@ mod test {
#[ignore]
async fn connectivity_test() {
let db = db_init().await;
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
assert_eq!(scanner.check_connectivity("https://v4.ipv6test.app").await,Some(host::Connectivity::IPv4));
assert_eq!(scanner.check_connectivity("https://ipv6test.app").await,Some(host::Connectivity::All));
assert_eq!(scanner.check_connectivity("https://v6.ipv6test.app").await,Some(host::Connectivity::IPv6));
let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new())
.await
.unwrap();
assert_eq!(
scanner.check_connectivity("https://v4.ipv6test.app").await,
Some(host::Connectivity::IPv4)
);
assert_eq!(
scanner.check_connectivity("https://ipv6test.app").await,
Some(host::Connectivity::All)
);
assert_eq!(
scanner.check_connectivity("https://v6.ipv6test.app").await,
Some(host::Connectivity::IPv6)
);
}
}
scanner/src/profile_parser.rs
-8
View File
@@ -1,7 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! Parse profile pages for verification
use regex::{Regex, RegexBuilder};
use reqwest::Url;
use scraper::{Html, Selector};
use thiserror::Error;
@@ -15,15 +13,12 @@ pub enum ProfileParseError {
NoProfileCard,
#[error("No timeline div found!")]
NoTimeline,
#[error("No timeline-item div found!")]
NoTimelineItem,
}
pub(crate) struct ProfileParser {
selector_profile_card_name: Selector,
selector_timeline: Selector,
selector_timeline_item: Selector,
regex: Regex,
}
#[derive(Debug)]
@@ -60,14 +55,11 @@ impl ProfileParser {
}
pub fn new() -> Self {
let mut builder = RegexBuilder::new(r#"^((\d+\.\d+\.\d+)|[a-zA-Z0-9]{7,})"#);
builder.case_insensitive(true);
Self {
selector_profile_card_name: Selector::parse(".profile-card-username")
.expect(EXPECT_CSS_SELCTOR),
selector_timeline: Selector::parse(".timeline").expect(EXPECT_CSS_SELCTOR),
selector_timeline_item: Selector::parse(".timeline-item").expect(EXPECT_CSS_SELCTOR),
regex: builder.build().expect("failed to generate regex"),
}
}
}
server/src/admin.rs
+126 -76
View File
@@ -1,48 +1,48 @@
use std::collections::HashSet;
use std::sync::Arc;
use axum::Form;
use axum::Json;
use axum::extract::Path;
use axum::extract::State;
use axum::response::Html;
use axum::response::IntoResponse;
use axum::response::Redirect;
use axum::Form;
use axum::Json;
use constant_time_eq::constant_time_eq;
use entities::check_errors;
use entities::health_check;
use entities::host;
use entities::state::AppState;
use hyper::header::REFERER;
use hyper::HeaderMap;
use hyper::StatusCode;
use hyper::header::REFERER;
use reqwest::Client;
use reqwest::Url;
use sea_orm::ColumnTrait;
use sea_orm::DatabaseConnection;
use sea_orm::EntityTrait;
use sea_orm::QueryFilter;
use sea_orm::QueryOrder;
use sea_orm::QuerySelect;
use serde::Deserialize;
use serde::Serialize;
use sha2::{Digest, Sha256};
use thiserror::Error;
use tower_sessions::Session;
use constant_time_eq::constant_time_eq;
use sha2::{Sha256, Digest};
use entities::host;
use sea_orm::ColumnTrait;
use sea_orm::QueryFilter;
use trust_dns_resolver::AsyncResolver;
use trust_dns_resolver::config::ResolverConfig;
use trust_dns_resolver::config::ResolverOpts;
use trust_dns_resolver::AsyncResolver;
use crate::ADMIN_OVERVIEW_URL;
use crate::Config;
use crate::LOGIN_URL;
use crate::Result;
use crate::ServerError;
use crate::ADMIN_OVERVIEW_URL;
use crate::LOGIN_URL;
#[derive(Serialize, Deserialize, Default)]
pub struct ActiveLogin {
/// Hosts this session has access to.
hosts: HashSet<i32>
hosts: HashSet<i32>,
}
const LOGIN_KEY: &'static str = "LOGIN";
@@ -53,7 +53,7 @@ pub enum LoginError {
#[error("Invalid instance URL '{0}'")]
InstanceUrl(String),
#[error("Invalid response at '{0}'")]
InvalidResponse(Url,String),
InvalidResponse(Url, String),
#[error("Key mismatch")]
KeyMismatch,
#[error("No public instance host found with domain '{0}'")]
@@ -80,34 +80,41 @@ pub async fn login(
State(ref login_client): State<Client>,
State(ref db): State<DatabaseConnection>,
session: Session,
Form(input): Form<LoginInput>
Form(input): Form<LoginInput>,
) -> Result<axum::response::Response> {
tracing::debug!(login=?input);
let host = host::Entity::find().filter(host::Column::Domain.eq(input.domain.trim()))
.one(db)
.await?;
let host = host::Entity::find()
.filter(host::Column::Domain.eq(input.domain.trim()))
.one(db)
.await?;
if host.as_ref().map(|host|get_session_login(&session).map(|login|login.hosts.contains(&host.id)).is_ok()).unwrap_or_default() {
if host
.as_ref()
.map(|host| {
get_session_login(&session)
.map(|login| login.hosts.contains(&host.id))
.is_ok()
})
.unwrap_or_default()
{
// already logged in
return Ok(Redirect::temporary(&input.referrer).into_response());
}
match login_inner(config, login_client, &input, host).await {
match login_inner(config, login_client, &input, host).await {
Ok(host) => {
let session_value = match get_session_login(&session) {
Ok(mut session) => {
session.hosts.insert(host.id);
session
},
}
Err(_) => {
let mut ids = HashSet::with_capacity(1);
ids.insert(host.id);
ActiveLogin {
hosts: ids,
}
ActiveLogin { hosts: ids }
}
};
session.insert(LOGIN_KEY,session_value)?;
session.insert(LOGIN_KEY, session_value)?;
let referrer = input.referrer.trim();
let location = match referrer.trim().is_empty() && referrer != LOGIN_URL {
true => ADMIN_OVERVIEW_URL,
@@ -116,7 +123,7 @@ pub async fn login(
let mut res = Redirect::temporary(location).into_response();
*res.status_mut() = StatusCode::FOUND;
Ok(res)
},
}
Err(e) => {
tracing::debug!(login_error=?e);
let mut context = tera::Context::new();
@@ -127,15 +134,15 @@ pub async fn login(
context.insert("VERIFY_TOKEN_NAME", &config.login_token_name);
match e {
LoginError::InvalidResponse(_, val)
| LoginError::ServerResponse(_,val)
| LoginError::DNSNoValidEntry(val)
| LoginError::InvalidHash(val) => context.insert("QUOTE", &val),
_ => (),
| LoginError::ServerResponse(_, val)
| LoginError::DNSNoValidEntry(val)
| LoginError::InvalidHash(val) => context.insert("QUOTE", &val),
_ => (),
}
let mut res = Html(template.render("login.html.j2", &context)?).into_response();
*res.status_mut() = StatusCode::FORBIDDEN;
Ok(res.into_response())
},
}
}
}
@@ -145,8 +152,8 @@ async fn login_inner(
input: &LoginInput,
host: Option<host::Model>,
) -> LoginResult<host::Model> {
let host = host.ok_or_else(||LoginError::HostNotFound(input.domain.clone()))?;
let host = host.ok_or_else(|| LoginError::HostNotFound(input.domain.clone()))?;
match input.verification_method {
VerificationMethod::DNS => {
let entries = fetch_host_dns(&host.domain, &config).await?;
@@ -156,10 +163,10 @@ async fn login_inner(
}
}
Err(LoginError::DNSNoValidEntry(entries.join(",")))
},
}
VerificationMethod::HTTP => {
let fetched_key = fetch_host_txt(&host.url, login_client, &config).await?;
verify_key(&fetched_key, &input.key).map(|_|host)
verify_key(&fetched_key, &input.key).map(|_| host)
}
}
}
@@ -169,30 +176,45 @@ pub struct LoginInput {
domain: String,
key: String,
referrer: String,
verification_method: VerificationMethod
verification_method: VerificationMethod,
}
#[derive(Serialize, Deserialize, Debug)]
enum VerificationMethod {
DNS,
HTTP
HTTP,
}
async fn fetch_host_txt(instance_url: &str, client: &Client, config: &Config) -> LoginResult<String> {
let mut request_url = Url::parse(&instance_url).map_err(|_|LoginError::InstanceUrl(instance_url.to_string()))?;
request_url.set_path(&[".",&config.login_token_name].concat());
async fn fetch_host_txt(
instance_url: &str,
client: &Client,
config: &Config,
) -> LoginResult<String> {
let mut request_url =
Url::parse(&instance_url).map_err(|_| LoginError::InstanceUrl(instance_url.to_string()))?;
request_url.set_path(&[".", &config.login_token_name].concat());
request_url.set_query(None);
let result = client.get(request_url.clone()).send().await.map_err(|e|LoginError::HttpFailure(request_url.clone(), e))?;
match true { // result.status().is_success()
true => Ok(result.text().await.map_err(|e: reqwest::Error|LoginError::InvalidResponse(request_url,e.to_string()))?),
false => Err(LoginError::ServerResponse(result.status().as_u16(),result.text().await.unwrap_or_default())),
let result = client
.get(request_url.clone())
.send()
.await
.map_err(|e| LoginError::HttpFailure(request_url.clone(), e))?;
match true {
// result.status().is_success()
true => Ok(result.text().await.map_err(|e: reqwest::Error| {
LoginError::InvalidResponse(request_url, e.to_string())
})?),
false => Err(LoginError::ServerResponse(
result.status().as_u16(),
result.text().await.unwrap_or_default(),
)),
}
}
async fn fetch_host_dns(instance_domain: &str, config: &Config) -> LoginResult<Vec<String>> {
// TODO: cache resolver ?
let resolver = AsyncResolver::tokio(ResolverConfig::cloudflare_tls(), ResolverOpts::default());
let hashed_key = resolver
.txt_lookup(format!("{}.{}.",&config.login_token_name, instance_domain))
.txt_lookup(format!("{}.{}.", &config.login_token_name, instance_domain))
.await?;
let mut entries = Vec::with_capacity(2);
@@ -212,14 +234,14 @@ fn verify_key(hash: &str, key: &str) -> LoginResult<()> {
let mut hasher: Sha256 = Sha256::new();
hasher.update(key);
let res = hasher.finalize();
let mut decoded_hash = [0;32];
base16ct::mixed::decode(hash, &mut decoded_hash).map_err(|e|{
let mut decoded_hash = [0; 32];
base16ct::mixed::decode(hash, &mut decoded_hash).map_err(|e| {
tracing::trace!(hash_error=?e);
LoginError::InvalidHash(hash.to_owned())
})?;
match constant_time_eq(&res,&decoded_hash) {
match constant_time_eq(&res, &decoded_hash) {
false => Err(LoginError::KeyMismatch),
true => Ok(())
true => Ok(()),
}
}
@@ -229,20 +251,19 @@ pub async fn login_view(
headers: HeaderMap,
) -> Result<axum::response::Response> {
tracing::debug!(headers=?headers);
let referrer = headers.get(REFERER).and_then(|v|v.to_str().ok());
let referrer = headers.get(REFERER).and_then(|v| v.to_str().ok());
let mut context = tera::Context::new();
context.insert("REFERRER", &referrer);// FIXME: won't work, handle this in the error part to extract the current situation
context.insert("REFERRER", &referrer); // FIXME: won't work, handle this in the error part to extract the current situation
context.insert("VERIFY_TOKEN_NAME", &config.login_token_name);
let res = Html(template.render("login.html.j2", &context)?).into_response();
Ok(res)
}
pub async fn overview(
State(ref app_state): State<AppState>,
State(ref template): State<Arc<tera::Tera>>,
State(ref db): State<DatabaseConnection>,
session: Session
session: Session,
) -> Result<axum::response::Response> {
tracing::info!(?session);
@@ -250,11 +271,14 @@ pub async fn overview(
let mut context = tera::Context::new();
let res = {
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
context.insert("last_updated", &time);
context.insert("instances", &hosts);
let res = Html(template.render("admin.html.j2", &context)?).into_response();
drop(guard);
res
@@ -265,12 +289,16 @@ pub async fn overview(
pub async fn history_json(
State(ref db): State<DatabaseConnection>,
Path(host): Path<i32>,
session: Session
session: Session,
) -> Result<axum::response::Response> {
let host = get_specific_login_host(host,&session,db).await?;
let host = get_specific_login_host(host, &session, db).await?;
let history: Vec<health_check::Model> = health_check::Entity::find().filter(health_check::Column::Host.eq(host.id))
.order_by_asc(health_check::Column::Time).limit(20).all(db).await?;
let history: Vec<health_check::Model> = health_check::Entity::find()
.filter(health_check::Column::Host.eq(host.id))
.order_by_asc(health_check::Column::Time)
.limit(20)
.all(db)
.await?;
Ok(Json(history).into_response())
}
@@ -280,19 +308,22 @@ pub async fn history_view(
State(ref template): State<Arc<tera::Tera>>,
State(ref db): State<DatabaseConnection>,
Path(host): Path<i32>,
session: Session
session: Session,
) -> Result<axum::response::Response> {
tracing::info!(?session);
let host = get_specific_login_host(host,&session,db).await?;
let host = get_specific_login_host(host, &session, db).await?;
let mut context = tera::Context::new();
let res = {
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
context.insert("last_updated", &time);
context.insert("HOST_DOMAIN", &host.domain);
let res = Html(template.render("errors_admin.html.j2", &context)?).into_response();
drop(guard);
res
@@ -305,23 +336,30 @@ pub async fn errors_view(
State(ref template): State<Arc<tera::Tera>>,
State(ref db): State<DatabaseConnection>,
Path(host): Path<i32>,
session: Session
session: Session,
) -> Result<axum::response::Response> {
tracing::info!(?session);
let host = get_specific_login_host(host,&session,db).await?;
let host = get_specific_login_host(host, &session, db).await?;
let errors = check_errors::Entity::find().filter(check_errors::Column::Host.eq(host.id))
.order_by_desc(check_errors::Column::Time).limit(20).all(db).await?;
let errors = check_errors::Entity::find()
.filter(check_errors::Column::Host.eq(host.id))
.order_by_desc(check_errors::Column::Time)
.limit(20)
.all(db)
.await?;
let mut context = tera::Context::new();
let res = {
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
context.insert("last_updated", &time);
context.insert("ERRORS", &errors);
context.insert("HOST_DOMAIN", &host.domain);
let res = Html(template.render("errors_admin.html.j2", &context)?).into_response();
drop(guard);
res
@@ -330,28 +368,40 @@ pub async fn errors_view(
}
/// Get all [host::Model] for current [Session]
async fn get_all_login_hosts(session: &Session, db: &DatabaseConnection) -> Result<Vec<host::Model>> {
async fn get_all_login_hosts(
session: &Session,
db: &DatabaseConnection,
) -> Result<Vec<host::Model>> {
let login = get_session_login(&session)?;
let host_res = host::Entity::find().filter(host::Column::Id.is_in(login.hosts.iter().map(|v|*v)))
.all(db).await?;
let host_res = host::Entity::find()
.filter(host::Column::Id.is_in(login.hosts.iter().map(|v| *v)))
.all(db)
.await?;
Ok(host_res)
}
/// Get wanted [host::Model] for current [Session] if valid for this user
async fn get_specific_login_host(wanted_host_id: i32, session: &Session, db: &DatabaseConnection) -> Result<host::Model> {
async fn get_specific_login_host(
wanted_host_id: i32,
session: &Session,
db: &DatabaseConnection,
) -> Result<host::Model> {
let login = get_session_login(&session)?;
if !login.hosts.contains(&wanted_host_id) {
return Err(ServerError::MissingPermission);
}
let host_res = host::Entity::find().filter(host::Column::Id.eq(wanted_host_id)).one(db).await?;
let host_res = host::Entity::find()
.filter(host::Column::Id.eq(wanted_host_id))
.one(db)
.await?;
match host_res {
None => {
session.delete();
return Err(ServerError::HostNotFound(wanted_host_id));
},
}
Some(host) => Ok(host),
}
}
@@ -359,9 +409,9 @@ async fn get_specific_login_host(wanted_host_id: i32, session: &Session, db: &Da
/// Check for valid session and return the stored [Login] data
fn get_session_login(session: &Session) -> Result<ActiveLogin> {
if session.active() {
if let Some(u) = session.get(LOGIN_KEY).map_err(|_|ServerError::NoLogin)? {
if let Some(u) = session.get(LOGIN_KEY).map_err(|_| ServerError::NoLogin)? {
return Ok(u);
}
}
Err(ServerError::NoLogin)
}
}
server/src/api.rs
+9 -3
View File
@@ -2,7 +2,7 @@
use crate::{Result, ServerError};
use axum::response::IntoResponse;
use axum::{extract::State, Json};
use entities::state::{Cache, AppState};
use entities::state::AppState;
use hyper::http::HeaderValue;
use std::sync::Arc;
@@ -11,7 +11,10 @@ pub async fn instances(
State(ref config): State<Arc<crate::Config>>,
) -> Result<axum::response::Response> {
let mut res = {
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
let res = Json(&*guard).into_response();
drop(guard);
res
@@ -20,6 +23,9 @@ pub async fn instances(
"cache-control",
HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(),
);
res.headers_mut().insert("X-Robots-Tag", HeaderValue::from_static("noindex, nofollow"));
res.headers_mut().insert(
"X-Robots-Tag",
HeaderValue::from_static("noindex, nofollow"),
);
Ok(res)
}
server/src/lib.rs
+25 -13
View File
@@ -1,7 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-only
use std::{borrow::Cow, net::SocketAddr, sync::Arc};
use axum::{extract::DefaultBodyLimit, http::HeaderValue, response::{Html, Redirect}, routing::{get, get_service}, Router, error_handling::HandleErrorLayer};
use axum::{
error_handling::HandleErrorLayer,
extract::DefaultBodyLimit,
http::HeaderValue,
response::{Html, Redirect},
routing::{get, get_service},
Router,
};
use entities::state::{scanner::ScannerConfig, AppState};
use hyper::{header, StatusCode};
use reqwest::Client;
@@ -10,14 +17,17 @@ use tera::Tera;
use thiserror::Error;
use tower::ServiceBuilder;
use tower_http::{
cors::CorsLayer, limit::RequestBodyLimitLayer, services::{ServeDir, ServeFile},
set_header::SetResponseHeaderLayer, trace::TraceLayer,
cors::CorsLayer,
limit::RequestBodyLimitLayer,
services::{ServeDir, ServeFile},
set_header::SetResponseHeaderLayer,
trace::TraceLayer,
};
use tower_sessions::{SessionManagerLayer, cookie::SameSite};
use tower_sessions::{cookie::SameSite, SessionManagerLayer};
mod admin;
mod api;
mod website;
mod admin;
const LOGIN_URL: &'static str = "/admin/login";
const ADMIN_OVERVIEW_URL: &'static str = "/admin";
@@ -70,8 +80,8 @@ pub async fn start(
.with_same_site(SameSite::Strict)
.with_max_age(time::Duration::seconds(config.session_ttl_seconds as _)),
);
let user_agent = format!("nitter-status (+{}/about)",scanner_config.website_url);
let user_agent = format!("nitter-status (+{}/about)", scanner_config.website_url);
let login_client = Client::builder()
.cookie_store(false)
.brotli(true)
@@ -81,7 +91,8 @@ pub async fn start(
.user_agent(user_agent)
.connect_timeout(std::time::Duration::from_secs(3))
.timeout(std::time::Duration::from_secs(10))
.build().unwrap();
.build()
.unwrap();
let config = Arc::new(config);
let mut tera = Tera::new("server/templates/*")?;
@@ -172,14 +183,15 @@ impl axum::response::IntoResponse for ServerError {
fn into_response(self) -> axum::response::Response {
use ServerError::*;
let msg = match &self {
NoLogin => {
NoLogin => {
let mut resp = Redirect::temporary(LOGIN_URL).into_response();
// *resp.status_mut() = StatusCode::FOUND; // have to use a 301, [Redirect] 307 won't work for referrer
return resp;
},
MissingPermission =>
(StatusCode::FORBIDDEN,Cow::Borrowed("Missing permission to access this resource"))
,
}
MissingPermission => (
StatusCode::FORBIDDEN,
Cow::Borrowed("Missing permission to access this resource"),
),
MutexFailure | Templating(_) | DBError(_) | SessionError(_) | HostNotFound(_) => (
StatusCode::INTERNAL_SERVER_ERROR,
Cow::Borrowed("Internal Server Error"),
server/src/website.rs
+9 -4
View File
@@ -6,9 +6,8 @@ use crate::Result;
use crate::ServerError;
use axum::response::IntoResponse;
use axum::{extract::State, response::Html};
use entities::state::AppState;
use entities::state::scanner::ScannerConfig;
use entities::state::Cache;
use entities::state::AppState;
use hyper::http::HeaderValue;
pub async fn instances(
@@ -18,7 +17,10 @@ pub async fn instances(
) -> Result<axum::response::Response> {
let mut context = tera::Context::new();
let mut res = {
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
context.insert("instances", &guard.hosts);
let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string();
context.insert("last_updated", &time);
@@ -61,7 +63,10 @@ pub async fn about(
&scanner_config.ping_range.num_hours(),
);
{
let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?;
let guard = app_state
.cache
.read()
.map_err(|_| ServerError::MutexFailure)?;
context.insert("latest_commit", &guard.latest_commit);
}
src/main.rs
+4 -5
View File
@@ -82,7 +82,8 @@ async fn _main() -> miette::Result<()> {
scanner_config.clone(),
cache.clone(),
disable_startup_scan,
).await
)
.await
.into_diagnostic()
.wrap_err("Failed starting background scanner")?;
@@ -115,12 +116,10 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
let profile_posts_min = require_env_str("PROFILE_POSTS_MIN")?
.parse()
.expect("PROFILE_POSTS_MIN must be a positive number");
let rss_content = require_env_str("RSS_CONTENT")?;
let additional_hosts: Vec<String> = require_env_vec_str("ADDITIONAL_HOSTS")?;
let additional_host_country = require_env_str("ADDITIONAL_HOSTS_COUNTRY")?;
let rss_content = require_env_str("RSS_CONTENT")?;
let bad_hosts: Vec<String> = require_env_vec_str("BAD_HOSTS")?;
let referrer = require_env_str("REFERER")?;
let auto_mute = require_env_str("AUTO_MUTE")? == "true";
let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?;
let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?;
@@ -164,8 +163,8 @@ async fn test_init(db: &DatabaseConnection) -> miette::Result<()> {
fn read_server_config(instance_ping_interval: usize) -> miette::Result<server::Config> {
let site_url = require_env_str("SITE_URL")?;
let session_ttl_seconds = require_env_str("SESSION_TTL_SECONDS")?
.parse()
.expect("SESSION_TTL_SECONDS must be a positive number");
.parse()
.expect("SESSION_TTL_SECONDS must be a positive number");
let login_token_name = require_env_str("LOGIN_TOKEN_NAME")?;
Ok(server::Config {