From 67ab47858113eff9862b392f0eaa8f31d2d96e16 Mon Sep 17 00:00:00 2001 From: 4cuck <40539855+4cuck@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:27:23 +0100 Subject: [PATCH] attempt to unify graphs --- entities/src/health_check.rs | 155 +++++++++++++++++++++++++++++------ server/src/api.rs | 8 +- server/static/history.js | 33 +++++++- 3 files changed, 165 insertions(+), 31 deletions(-) diff --git a/entities/src/health_check.rs b/entities/src/health_check.rs index 5a5ac37..5c02480 100644 --- a/entities/src/health_check.rs +++ b/entities/src/health_check.rs @@ -1,10 +1,12 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 -use sea_orm::{entity::prelude::*, FromQueryResult}; -use sea_query::{Query, Order, Alias, SimpleExpr}; +use sea_orm::{entity::prelude::*, FromQueryResult, Statement, StatementBuilder}; +use sea_query::{ + Alias, CommonTableExpression, Cond, Order, Query, QueryStatementWriter, SimpleExpr, SqliteQueryBuilder, WithClause +}; use serde::Serialize; -use crate::check_errors; +use crate::{check_errors, instance_stats}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] #[sea_orm(table_name = "health_check")] @@ -50,22 +52,36 @@ pub struct InstanceGraphEntry { pub time: i64, pub alive: i64, pub dead: i64, + pub limited_accs_avg: Option, + pub total_accs_avg: Option, + pub total_requests_avg: Option, } impl HealthyAmount { - /// Fetch health check graph data for all or selected hosts in the selected time range. - pub async fn fetch(db: &DatabaseConnection, from: Option, to: Option, hosts: Option<&[i32]>) -> Result,DbErr> { + /// Fetch health check graph data for all or selected hosts in the selected time range. + pub async fn fetch( + db: &DatabaseConnection, + from: Option, + to: Option, + hosts: Option<&[i32]>, + ) -> Result, DbErr> { let builder = db.get_database_backend(); let mut stmt: sea_query::SelectStatement = Query::select(); stmt.column(self::Column::Time) - .expr_as(SimpleExpr::Custom("SUM(healthy)".to_string()), Alias::new("alive")) - .expr_as(SimpleExpr::Custom("SUM(1-healthy)".to_string()), Alias::new("dead")) + .expr_as( + SimpleExpr::Custom("SUM(healthy)".to_string()), + Alias::new("alive"), + ) + .expr_as( + SimpleExpr::Custom("SUM(1-healthy)".to_string()), + Alias::new("dead"), + ) .group_by_col(self::Column::Time) .from(self::Entity); - if let (Some(from),Some(to)) = (from,to) { + if let (Some(from), Some(to)) = (from, to) { stmt.and_where(self::Column::Time.between(from.timestamp(), to.timestamp())); } - if let Some(hosts) = hosts { + if let Some(hosts) = hosts { stmt.and_where(self::Column::Host.is_in(hosts.iter().map(|v| *v))); } stmt.group_by_col(self::Column::Time) @@ -75,22 +91,105 @@ impl HealthyAmount { } impl InstanceGraphEntry { - /// Fetch health check graph data for all or selected hosts in the selected time range. - pub async fn fetch(db: &DatabaseConnection, to: Option) -> Result,DbErr> { + /// Fetch health check graph data for all or selected hosts in the selected time range. + pub async fn fetch( + db: &DatabaseConnection, + to: Option, + ) -> Result, DbErr> { let builder = db.get_database_backend(); - let mut stmt: sea_query::SelectStatement = Query::select(); - stmt//.expr_as(SimpleExpr::Custom("strftime('%Y/%m/%d %H:%M:%S', datetime(time, 'unixepoch', 'utc'))".to_string()), Alias::new("time")) - .column(self::Column::Time) - .expr_as(SimpleExpr::Custom("SUM(healthy)".to_string()), Alias::new("alive")) - .expr_as(SimpleExpr::Custom("SUM(1-healthy)".to_string()), Alias::new("dead")) - .group_by_col(self::Column::Time) - .from(self::Entity); - if let Some(to) = to { - stmt.and_where(self::Column::Time.lte(to.timestamp())); - } - stmt.group_by_col(self::Column::Time) - .order_by(self::Column::Time, Order::Asc); - Self::find_by_statement(builder.build(&stmt)).all(db).await + + let cte_stats = { + let mut stmt_stats = Query::select(); + + let columns: [&str; 3] = ["limited_accs", "total_accs", "total_requests"]; + stmt_stats.column(instance_stats::Column::Time); + for col in columns { + stmt_stats.expr_as( + Expr::cust(format!("AVG({col})")), + Alias::new(format!("{col}_avg")), + ); + } + stmt_stats + .from(instance_stats::Entity) + .group_by_col(self::Column::Time) + .order_by(self::Column::Time, Order::Asc); + let mut cte_stats = CommonTableExpression::from_select(stmt_stats); + cte_stats.table_name(Alias::new("stats")); + cte_stats + }; + + let mut cte_health = { + let mut stmt_health = Query::select(); + stmt_health //.expr_as(SimpleExpr::Custom("strftime('%Y/%m/%d %H:%M:%S', datetime(time, 'unixepoch', 'utc'))".to_string()), Alias::new("time")) + .column(self::Column::Time) + .expr_as( + SimpleExpr::Custom("SUM(healthy)".to_string()), + Alias::new("alive"), + ) + .expr_as( + SimpleExpr::Custom("SUM(1-healthy)".to_string()), + Alias::new("dead"), + ) + .group_by_col(self::Column::Time) + .from(self::Entity); + if let Some(to) = to { + stmt_health.and_where(self::Column::Time.lte(to.timestamp())); + } + stmt_health + .group_by_col(self::Column::Time) + .order_by(self::Column::Time, Order::Asc); + let mut cte = CommonTableExpression::from_select(stmt_health); + cte.table_name(Alias::new("health")); + cte + }; + + let with_clause = WithClause::new() + .recursive(false) + .cte(cte_health) + .cte(cte_stats) + .to_owned(); + + let select = Query::select() + .expr_as( + SimpleExpr::Custom("health.time".to_string()), + Alias::new("time"), + ) + .expr_as( + SimpleExpr::Custom("health.alive".to_string()), + Alias::new("alive"), + ) + .expr_as( + SimpleExpr::Custom("health.dead".to_string()), + Alias::new("dead"), + ) + .expr_as( + SimpleExpr::Custom("stats.limited_accs_avg".to_string()), + Alias::new("limited_accs_avg"), + ) + .expr_as( + SimpleExpr::Custom("stats.total_accs_avg".to_string()), + Alias::new("total_accs_avg"), + ) + .expr_as( + SimpleExpr::Custom("stats.total_requests_avg".to_string()), + Alias::new("total_requests_avg"), + ) + .left_join(Alias::new("stats"), Expr::cust("health.time = stats.time")) + .from(Alias::new("health")) + .to_owned(); + + let final_query = with_clause.query(select).to_owned(); + + // how to query this native, without this statement -> sql string -> query hack + // Self::find_by_statement(builder.build(&final_query)).all(db).await + let sql = final_query.build(SqliteQueryBuilder).0; + println!("{sql}"); + Self::find_by_statement(Statement::from_string( + db.get_database_backend(), + sql, + )) + .all(db) + .await } } @@ -119,7 +218,9 @@ mod test { let time_3h = time_now .checked_sub_signed(chrono::Duration::days(14)) .unwrap(); - let hosts = vec![1,2,3]; - HealthyAmount::fetch(&db, Some(time_3h), Some(time_now),Some(&hosts)).await.unwrap(); + let hosts = vec![1, 2, 3]; + HealthyAmount::fetch(&db, Some(time_3h), Some(time_now), Some(&hosts)) + .await + .unwrap(); } -} \ No newline at end of file +} diff --git a/server/src/api.rs b/server/src/api.rs index b01efe1..11270cf 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -44,11 +44,15 @@ pub async fn graph( let queried = std::time::Instant::now(); let mut data = String::with_capacity(8*healthy_data.len()); - data.push_str("Date,Healthy,Dead\n"); + data.push_str("Date,Healthy,Dead,Tokens AVG,Requests AVG,Limited Tokens AVG\n"); for entry in healthy_data { let time = Utc.timestamp_opt(entry.time,0).unwrap().format("%Y-%m-%dT%H:%M:%SZ"); - writeln!(&mut data, "{},{},{}",time,entry.alive,entry.dead) + if let (Some(total_accs_avg),Some(total_requests_avg),Some(limited_accs_avg)) = (entry.total_accs_avg,entry.total_requests_avg,entry.limited_accs_avg){ + writeln!(&mut data, "{time},{},{},{},{},{}",entry.alive,entry.dead,total_accs_avg,total_requests_avg,limited_accs_avg) + } else { + writeln!(&mut data, "{time},{},{},,,",entry.alive,entry.dead) + } .map_err(|e|ServerError::CSV(e.to_string()))?; } let formatted = std::time::Instant::now(); diff --git a/server/static/history.js b/server/static/history.js index d6d866d..5f6bd62 100644 --- a/server/static/history.js +++ b/server/static/history.js @@ -16,7 +16,36 @@ async function loadGraph() { rangeSelectorPlotFillGradientColor: 'rgba(123, 104, 238, 0)', colorValue: 0.9, fillAlpha: 0.4, - colors: ['#008000', '#ffa500'], + colors: ['#008000', '#ffa500','#008500','#058000'], + series: { + 'Date': { + axis: 'y' + }, + 'Healthy': { + axis: 'y' + }, + 'Tokens AVG': { + axis: 'y2' + }, + 'Limited Tokens AVG': { + axis: 'y2' + }, + 'Requests AVG': { + axis: 'y2' + }, + }, + axes: { + y: { + axisLabelWidth: 60 + }, + y2: { + // set axis-related properties here + // drawGrid: false, + // drawAxis: false, + }, + }, + ylabel: 'Instances', + y2label: 'Tokens', /*rollPeriod: 14, showRoller: true, customBars: true, @@ -38,7 +67,7 @@ async function loadGraph() { }, { series: "Healthy", - x: "2024-01-2400:00:01Z", + x: "2024-01-24T00:00:01Z", shortText: "D", text: "API Shutdown" }