Commit af63d6fe authored by Orhun Parmaksız's avatar Orhun Parmaksız
Browse files

Merge branch 'refactor/handle_errors' into 'main'

Improve error handling

Closes #4

See merge request artafinde/gitlab-exporter!9
parents 1755ee2e 1a2d4288
Pipeline #16072 passed with stages
in 1 minute and 24 seconds
......@@ -28,9 +28,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.52"
version = "1.0.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3"
checksum = "7a99269dff3bc004caa411f38845c20303f1e393ca2bd6581576fa3a7f59577d"
[[package]]
name = "arrayref"
......@@ -535,6 +535,7 @@ dependencies = [
"prometheus",
"reqwest",
"serde",
"thiserror",
"tokio",
"warp",
]
......
......@@ -24,6 +24,7 @@ futures ="0.3"
prometheus = "0.13"
clap = { version = "3.0", features = ["derive", "env"] }
warp = "0.3"
thiserror = "1.0.30"
[profile.dev]
opt-level = 0
......
use std::convert::TryInto;
use std::error::Error;
use graphql_client::{GraphQLQuery, Response};
use crate::error::{Error, Result};
use crate::graphql::{GraphQL, Metrics};
use graphql_client::{GraphQLQuery, Response};
use std::convert::TryInto;
type Time = String;
......@@ -20,13 +18,12 @@ macro_rules! gen_admin_query {
)]
pub struct $name;
impl $name {
pub async fn run(graphql: &GraphQL) -> Result<Metrics, Box<dyn Error + Send + Sync>> {
pub async fn run(graphql: &GraphQL) -> Result<Metrics> {
let mut metrics = Metrics::new();
if let Some(utm) = Self::query(graphql, $var::Variables {})
.await
.unwrap()
.await?
.data
.expect("missing response data")
.ok_or(Error::MissingData("response data"))?
.usage_trends_measurements
{
let mut count: i64 = 0;
......@@ -34,19 +31,20 @@ macro_rules! gen_admin_query {
count += e
.iter()
.flatten()
.map(|trends| trends.node.as_ref().unwrap())
.map(|trends| trends.node.as_ref())
.flatten()
.next()
.unwrap()
.ok_or(Error::MissingData("trends node"))?
.count
}
metrics.add_metric(String::from($metric), count.try_into().unwrap())
metrics.add_metric(String::from($metric), count.try_into()?)
}
Ok(metrics)
}
async fn query(
graphql: &GraphQL,
variables: $var::Variables,
) -> Result<Response<$var::ResponseData>, reqwest::Error> {
) -> Result<Response<$var::ResponseData>> {
let request_body = Self::build_query(variables);
let response = graphql
.client()
......
use thiserror::Error as ThisError;
#[derive(ThisError, Debug)]
pub enum Error<'a> {
#[error("request cannot be processed: `{0}`")]
FailedRequest(#[from] reqwest::Error),
#[error("invalid header: `{0}`")]
InvalidHeader(#[from] reqwest::header::InvalidHeaderValue),
#[error("conversion failed: `{0}`")]
FailedConversion(#[from] std::num::TryFromIntError),
#[error("no admin related queries can be performed")]
InsufficentPermission,
#[error("missing: `{0}`")]
MissingData(&'a str),
}
pub type Result<T> = std::result::Result<T, Error<'static>>;
use crate::error::Result;
use reqwest::Client;
use std::collections::HashMap;
use std::env;
......@@ -34,14 +35,13 @@ pub struct GraphQL {
}
impl GraphQL {
pub fn new(gitlab_url: String, gitlab_token: String) -> Result<GraphQL, anyhow::Error> {
pub fn new(gitlab_url: String, gitlab_token: String) -> Result<GraphQL> {
let client = Client::builder()
.user_agent(APP_USER_AGENT)
.default_headers(
std::iter::once((
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", gitlab_token))
.unwrap(),
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", gitlab_token))?,
))
.collect(),
)
......
#![warn(clippy::unwrap_used)]
mod admin_queries;
pub mod args;
mod error;
mod graphql;
mod queries;
use crate::admin_queries::{GroupsCount, ProjectsCount, UsersCount};
use crate::args::{Args, Command};
use crate::graphql::GraphQL;
use crate::queries::RepositorySize;
use anyhow::Result;
use futures::future::BoxFuture;
use indicatif::{HumanDuration, ProgressBar, ProgressStyle};
use lazy_static::lazy_static;
use log::{debug, error};
use prometheus::{Encoder, IntGauge, Opts, Registry, TextEncoder};
use std::convert::TryInto;
use std::fs;
use std::time::Instant;
use warp::{Rejection, Reply};
#[allow(dead_code)]
const DEFAULT_PAUSE_BETWEEN_QUERIES: u64 = 300; // 5 minutes
lazy_static! {
static ref TOTAL_BUILD_TRENDS_TOTAL_USER_GAUGE: IntGauge =
IntGauge::new("gitlab_exporter_trends_total_users", "trends total users")
.expect("metric can be created");
static ref TOTAL_BUILD_ARTIFACTS_SIZE_GAUGE: IntGauge = IntGauge::new(
"gitlab_exporter_total_build_artifacts_size",
"total build artifacts size"
)
.expect("metric can be created");
static ref REGISTRY: Registry = Registry::new();
}
pub async fn run(args: Args) -> Result<()> {
match args.command {
Command::Cli => {
let started = Instant::now();
let pb: Option<ProgressBar> = initialize_progress_bar(&args);
let graphql = GraphQL::new(args.api_url, args.api_token)?;
let handles: Vec<BoxFuture<_>> = vec![
Box::pin(UsersCount::run(&graphql)),
Box::pin(ProjectsCount::run(&graphql)),
Box::pin(GroupsCount::run(&graphql)),
Box::pin(RepositorySize::run(&graphql)),
];
let results = futures::future::join_all(handles).await.into_iter();
for metric in results {
let metric = metric?;
for (key, value) in metric.get_metrics() {
let help_text = key.to_string().replace('_', " ");
let gauge = IntGauge::with_opts(Opts::new(key, help_text))?;
REGISTRY.register(Box::new(gauge.clone()))?;
gauge.set(usize::try_into(value)?);
}
}
if let Some(pb) = pb {
pb.finish_with_message(format!("Done in {}", HumanDuration(started.elapsed())));
}
let mut buffer = vec![];
let encoder = TextEncoder::new();
let metric_families = REGISTRY.gather();
encoder.encode(&metric_families, &mut buffer)?;
match args.output {
Some(output) => fs::write(output, buffer)?,
None => println!("{}", String::from_utf8(buffer)?),
}
} // Command::Http => {
// panic!("Not implemented feature.");
// let health_route = warp::path!("health").map(|| StatusCode::PROCESSING);
// let metrics_route = warp::path!("metrics").and_then(metrics_handler);
//
// warp::serve(metrics_route.or(health_route))
// .run(([0, 0, 0, 0], 8080))
// .await;
// }
}
Ok(())
}
#[allow(dead_code)]
async fn metrics_handler() -> Result<impl Reply, Rejection> {
let encoder = TextEncoder::new();
let mut buffer = Vec::new();
if let Err(e) = encoder.encode(&REGISTRY.gather(), &mut buffer) {
error!("could not encode custom metrics: {}", e);
};
let res = match String::from_utf8(buffer.clone()) {
Ok(v) => v,
Err(e) => {
error!("custom metrics could not be from_utf8'd: {}", e);
String::default()
}
};
buffer.clear();
Ok(res)
}
fn initialize_progress_bar(args: &Args) -> Option<ProgressBar> {
if args.progress_bar {
debug!("Initializing progress indicator");
let pb = ProgressBar::new_spinner();
pb.enable_steady_tick(200);
pb.set_style(
ProgressStyle::default_spinner()
.tick_strings(&[". ", ".. ", "...", " ..", " .", " "])
.template("{spinner:.blue} {msg}"),
);
pb.set_message("Extracting...");
Some(pb)
} else {
None
}
}
use std::convert::TryInto;
use std::fs;
use std::time::Instant;
use crate::graphql::GraphQL;
use anyhow::Result;
use clap::Parser;
use env_logger::Env;
use futures::future::BoxFuture;
use indicatif::{HumanDuration, ProgressBar, ProgressStyle};
use lazy_static::lazy_static;
use gitlab_exporter::args::Args;
use log::{debug, error};
use prometheus::{Encoder, IntGauge, Opts, Registry, TextEncoder};
use warp::{Rejection, Reply};
use args::Args;
use crate::admin_queries::{GroupsCount, ProjectsCount, UsersCount};
use crate::args::Command;
use crate::queries::RepositorySize;
mod admin_queries;
mod args;
mod graphql;
mod queries;
#[allow(dead_code)]
const DEFAULT_PAUSE_BETWEEN_QUERIES: u64 = 300; // 5 minutes
lazy_static! {
pub static ref TOTAL_BUILD_TRENDS_TOTAL_USER_GAUGE: IntGauge =
IntGauge::new("gitlab_exporter_trends_total_users", "trends total users")
.expect("metric can be created");
pub static ref TOTAL_BUILD_ARTIFACTS_SIZE_GAUGE: IntGauge = IntGauge::new(
"gitlab_exporter_total_build_artifacts_size",
"total build artifacts size"
)
.expect("metric can be created");
pub static ref REGISTRY: Registry = Registry::new();
}
#[tokio::main]
async fn main() {
......@@ -51,7 +15,7 @@ async fn main() {
env_logger::init_from_env(Env::default().default_filter_or(logging));
debug!("{:?}", args);
if let Err(err) = run(args).await {
if let Err(err) = gitlab_exporter::run(args).await {
error!("Error: {:?}", err);
for cause in err.chain() {
error!("Caused by: {:?}", cause)
......@@ -60,84 +24,3 @@ async fn main() {
}
std::process::exit(0)
}
async fn run(args: Args) -> Result<()> {
match args.command {
Command::Cli => {
let started = Instant::now();
let pb: Option<ProgressBar> = initialize_progress_bar(&args);
let graphql = GraphQL::new(args.api_url, args.api_token)?;
let handles: Vec<BoxFuture<_>> = vec![
Box::pin(UsersCount::run(&graphql)),
Box::pin(ProjectsCount::run(&graphql)),
Box::pin(GroupsCount::run(&graphql)),
Box::pin(RepositorySize::run(&graphql)),
];
let results = futures::future::join_all(handles).await.into_iter();
results.flatten().for_each(|m| {
for (key, value) in m.get_metrics() {
let help_text = key.to_string().replace('_', " ");
let gauge = IntGauge::with_opts(Opts::new(key, help_text)).unwrap();
REGISTRY.register(Box::new(gauge.clone())).unwrap();
gauge.set(usize::try_into(value).unwrap());
}
});
if let Some(pb) = pb {
pb.finish_with_message(format!("Done in {}", HumanDuration(started.elapsed())));
}
let mut buffer = vec![];
let encoder = TextEncoder::new();
let metric_families = REGISTRY.gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
match args.output {
Some(output) => fs::write(output, buffer).expect("Unable to write file"),
None => println!("{}", String::from_utf8(buffer).unwrap()),
}
} // Command::Http => {
// panic!("Not implemented feature.");
// let health_route = warp::path!("health").map(|| StatusCode::PROCESSING);
// let metrics_route = warp::path!("metrics").and_then(metrics_handler);
//
// warp::serve(metrics_route.or(health_route))
// .run(([0, 0, 0, 0], 8080))
// .await;
// }
}
Ok(())
}
#[allow(dead_code)]
async fn metrics_handler() -> Result<impl Reply, Rejection> {
let encoder = TextEncoder::new();
let mut buffer = Vec::new();
if let Err(e) = encoder.encode(&REGISTRY.gather(), &mut buffer) {
error!("could not encode custom metrics: {}", e);
};
let res = match String::from_utf8(buffer.clone()) {
Ok(v) => v,
Err(e) => {
error!("custom metrics could not be from_utf8'd: {}", e);
String::default()
}
};
buffer.clear();
Ok(res)
}
fn initialize_progress_bar(args: &Args) -> Option<ProgressBar> {
if args.progress_bar {
debug!("Initializing progress indicator");
let pb = ProgressBar::new_spinner();
pb.enable_steady_tick(200);
pb.set_style(
ProgressStyle::default_spinner()
.tick_strings(&[". ", ".. ", "...", " ..", " .", " "])
.template("{spinner:.blue} {msg}"),
);
pb.set_message("Extracting...");
Some(pb)
} else {
None
}
}
use std::convert::TryInto;
use std::error::Error;
use graphql_client::{GraphQLQuery, Response};
use crate::error::{Error, Result};
use crate::graphql::{GraphQL, Metrics};
use graphql_client::{GraphQLQuery, Response};
use std::convert::TryInto;
#[derive(GraphQLQuery)]
#[graphql(
......@@ -16,7 +14,7 @@ use crate::graphql::{GraphQL, Metrics};
pub struct RepositorySize;
impl RepositorySize {
pub async fn run(graphql: &GraphQL) -> Result<Metrics, Box<dyn Error + Send + Sync>> {
pub async fn run(graphql: &GraphQL) -> Result<Metrics> {
let mut repository_size: f64 = 0.0;
let mut build_artifacts_size: f64 = 0.0;
let mut commit_count: f64 = 0.0;
......@@ -31,28 +29,30 @@ impl RepositorySize {
let mut end_cursor = String::new();
let mut projects = Self::query(graphql, end_cursor)
.await
.unwrap()
.await?
.data
.expect("missing response data")
.ok_or(Error::MissingData("response data"))?
.projects
.expect("missing projects");
.ok_or(Error::MissingData("projects"))?;
if let Some(e) = projects.edges {
e.iter()
.flatten()
.map(|edge| edge.node.as_ref().unwrap().statistics.as_ref().unwrap())
.for_each(|stats| {
build_artifacts_size += stats.build_artifacts_size;
commit_count += stats.commit_count;
lfs_objects_size += stats.lfs_objects_size;
packages_size += stats.packages_size;
pipeline_artifacts_size += stats.pipeline_artifacts_size.unwrap_or(0.0);
repository_size += stats.repository_size;
snippets_size += stats.snippets_size.unwrap_or(0.0);
storage_size += stats.storage_size;
uploads_size += stats.uploads_size.unwrap_or(0.0);
wiki_size += stats.wiki_size.unwrap_or(0.0);
})
for stats in e.iter().flatten().map(|edge| {
edge.node
.as_ref()
.and_then(|node| node.statistics.as_ref())
.ok_or(Error::InsufficentPermission)
}) {
let stats = stats?;
build_artifacts_size += stats.build_artifacts_size;
commit_count += stats.commit_count;
lfs_objects_size += stats.lfs_objects_size;
packages_size += stats.packages_size;
pipeline_artifacts_size += stats.pipeline_artifacts_size.unwrap_or(0.0);
repository_size += stats.repository_size;
snippets_size += stats.snippets_size.unwrap_or(0.0);
storage_size += stats.storage_size;
uploads_size += stats.uploads_size.unwrap_or(0.0);
wiki_size += stats.wiki_size.unwrap_or(0.0);
}
}
let mut more_data = projects.page_info.has_next_page;
while more_data {
......@@ -61,73 +61,75 @@ impl RepositorySize {
.end_cursor
.as_ref()
.map(String::from)
.unwrap();
.ok_or(Error::MissingData("end cursor"))?;
projects = Self::query(graphql, end_cursor)
.await
.unwrap()
.await?
.data
.expect("missing response data")
.ok_or(Error::MissingData("response data"))?
.projects
.expect("missing projects");
.ok_or(Error::MissingData("projects"))?;
if let Some(e) = projects.edges {
e.iter()
.flatten()
.map(|edge| edge.node.as_ref().unwrap().statistics.as_ref().unwrap())
.for_each(|stats| {
build_artifacts_size += stats.build_artifacts_size;
commit_count += stats.commit_count;
lfs_objects_size += stats.lfs_objects_size;
packages_size += stats.packages_size;
pipeline_artifacts_size += stats.pipeline_artifacts_size.unwrap_or(0.0);
repository_size += stats.repository_size;
snippets_size += stats.snippets_size.unwrap_or(0.0);
storage_size += stats.storage_size;
uploads_size += stats.uploads_size.unwrap_or(0.0);
wiki_size += stats.wiki_size.unwrap_or(0.0);
})
for stats in e.iter().flatten().map(|edge| {
edge.node
.as_ref()
.and_then(|node| node.statistics.as_ref())
.ok_or(Error::InsufficentPermission)
}) {
let stats = stats?;
build_artifacts_size += stats.build_artifacts_size;
commit_count += stats.commit_count;
lfs_objects_size += stats.lfs_objects_size;
packages_size += stats.packages_size;
pipeline_artifacts_size += stats.pipeline_artifacts_size.unwrap_or(0.0);
repository_size += stats.repository_size;
snippets_size += stats.snippets_size.unwrap_or(0.0);
storage_size += stats.storage_size;
uploads_size += stats.uploads_size.unwrap_or(0.0);
wiki_size += stats.wiki_size.unwrap_or(0.0);
}
}
more_data = projects.page_info.has_next_page;
}
let mut metrics = Metrics::new();
metrics.add_metric(
String::from("total_build_artifacts_size"),
(build_artifacts_size as i64).try_into().unwrap(),
(build_artifacts_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_commit_count"),
(commit_count as i64).try_into().unwrap(),
(commit_count as i64).try_into()?,
);
metrics.add_metric(
String::from("total_lfs_objects_size"),
(lfs_objects_size as i64).try_into().unwrap(),
(lfs_objects_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_packages_size"),
(packages_size as i64).try_into().unwrap(),
(packages_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_pipeline_artifacts_size"),
(pipeline_artifacts_size as i64).try_into().unwrap(),
(pipeline_artifacts_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_repository_size"),
(repository_size as i64).try_into().unwrap(),
(repository_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_snippets_size"),
(snippets_size as i64).try_into().unwrap(),
(snippets_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_storage_size"),
(storage_size as i64).try_into().unwrap(),
(storage_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_uploads_size"),
(uploads_size as i64).try_into().unwrap(),
(uploads_size as i64).try_into()?,
);
metrics.add_metric(
String::from("total_wiki_size"),
(wiki_size as i64).try_into().unwrap(),
(wiki_size as i64).try_into()?,
);
Ok(metrics)
}
......@@ -135,7 +137,7 @@ impl RepositorySize {
async fn query(
graphql: &GraphQL,
cursor: String,
) -> Result<Response<repository_size::ResponseData>, reqwest::Error> {
) -> Result<Response<repository_size::ResponseData>> {
let response = graphql
.client()
.post(graphql.gitlab_url())
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment