Request host metrics all at once

svelte/src/admin_panel/HostMetrics.svelte

@@ -1,14 +1,18 @@
 <script lang="ts">
-import { onMount } from "svelte";
+import { onDestroy, onMount } from "svelte";
 import HostMetricsGraph from "./HostMetricsGraph.svelte";
-import { load_host_names } from "./HostMetricsLib";
+import { get_host_metrics, load_host_names, type HostMetrics } from "./HostMetricsLib";
 import Expandable from "util/Expandable.svelte";
 import ToggleButton from "layout/ToggleButton.svelte";
+import { formatDate } from "util/Formatting";
+import { loading_finish, loading_start } from "lib/Loading";
 
 const groups: {
 	title: string,
 	graphs: {
 		metric: string,
+		agg_base?: string,
+		agg_divisor?: string,
 		data_type: string,
 	}[],
 }[] = [
@@ -17,6 +21,7 @@ const groups: {
 		graphs: [
 			{metric: "api_request", data_type: "number"},
 			{metric: "api_request_duration", data_type: "duration"},
+			{metric: "api_request_duration_avg", agg_base: "api_request_duration", agg_divisor: "api_request", data_type: "duration"},
 			{metric: "api_error", data_type: "number"},
 			{metric: "api_panic", data_type: "number"},
 		],
@@ -31,7 +36,9 @@ const groups: {
 		graphs: [
 			{metric: "database_query", data_type: "number"},
 			{metric: "database_query_duration", data_type: "duration"},
+			{metric: "database_query_duration_avg", agg_base: "database_query_duration", agg_divisor: "database_query", data_type: "duration"},
 			{metric: "database_query_rows", data_type: "number"},
+			{metric: "database_query_rows_avg", agg_base: "database_query_rows", agg_divisor: "database_query", data_type: "number"},
 			{metric: "database_query_retries", data_type: "number"},
 			{metric: "database_query_error", data_type: "number"},
 		],
@@ -56,6 +63,8 @@ const groups: {
 			{metric: "pixelstore_neighbour_read_size", data_type: "bytes"},
 			{metric: "pixelstore_reed_solomon_read", data_type: "number"},
 			{metric: "pixelstore_reed_solomon_read_size", data_type: "bytes"},
+			{metric: "pixelstore_read_retry_success", data_type: "number"},
+			{metric: "pixelstore_read_retry_error", data_type: "number"},
 		],
 	}, {
 		title: "Pixelstore shards",
@@ -67,23 +76,109 @@ const groups: {
 			{metric: "pixelstore_shard_move", data_type: "number"},
 			{metric: "pixelstore_shard_move_size", data_type: "bytes"},
 		],
+	}, {
+		title: "Pixelstore API",
+		graphs: [
+			{metric: "pixelstore_api_error_400", data_type: "number"},
+			{metric: "pixelstore_api_error_500", data_type: "number"},
+			{metric: "pixelstore_api_put_file", data_type: "number"},
+			{metric: "pixelstore_api_put_file_size", data_type: "bytes"},
+			{metric: "pixelstore_api_get_file", data_type: "number"},
+			{metric: "pixelstore_api_file_exists", data_type: "number"},
+			{metric: "pixelstore_api_status", data_type: "number"},
+		],
 	},
 ]
 
 let dataWindow: number = $state(60)
 let dataInterval: number = $state(1)
 let showAggregate: boolean = $state(false)
+let metrics: HostMetrics = $state({timestamps: [], metrics: {}})
+let metrics_timeout: NodeJS.Timeout = null
+
+const load_metrics = async (window: number, interval: number) => {
+	if (metrics_timeout !== null) { clearTimeout(metrics_timeout) }
+	metrics_timeout = setTimeout(() => { load_metrics(dataWindow, dataInterval) }, 10000)
+
+	let today = new Date()
+	let start = new Date()
+	start.setMinutes(start.getMinutes() - window)
+
+	let metrics_list: string[] = []
+	for (const group of groups) {
+		for (const graph of group.graphs) {
+			if (graph.metric !== undefined) {
+				metrics_list.push(graph.metric)
+			}
+		}
+	}
+
+	loading_start()
+	try {
+		metrics = await get_host_metrics(start, today, metrics_list, interval)
+
+		// Format the dates
+		metrics.timestamps.forEach((val: string, idx: number) => {
+			metrics.timestamps[idx] = formatDate(val, true, true, true)
+		});
+	} catch (error) {
+		alert(error)
+	} finally {
+		loading_finish()
+	}
+
+	// If the dataset uses the duration type, we need to convert the values
+	// to milliseconds
+	for (const group of groups) {
+		for (const graph of group.graphs) {
+			if (graph.data_type === "duration" && metrics.metrics[graph.metric] !== undefined) {
+				for (const host of Object.keys(metrics.metrics[graph.metric])) {
+					for (let i = 0; i < metrics.metrics[graph.metric][host].length; i++) {
+						// Go durations are expressed on nanoseconds, divide by
+						// 1 million to convert to milliseconds
+						metrics.metrics[graph.metric][host][i] /= 1000000
+					}
+				}
+			}
+
+			// If the graph is an aggregate, we'll need to create a new dataset
+			// with the child datasets as base. Here we create a new dataset by
+			// dividing one dataset by another
+			if (graph.agg_base !== undefined && graph.agg_divisor !== undefined) {
+				for (const host of Object.keys(metrics.metrics[graph.agg_base])) {
+					metrics.metrics[graph.metric][host] = []
+					for (let i = 0; i < metrics.metrics[graph.agg_base][host].length; i++) {
+						if (metrics.metrics[graph.agg_divisor][host][i] > 0) {
+							metrics.metrics[graph.metric][host].push(
+								metrics.metrics[graph.agg_base][host][i] / metrics.metrics[graph.agg_divisor][host][i]
+							)
+						} else {
+							metrics.metrics[graph.metric][host].push(0)
+						}
+					}
+				}
+			}
+		}
+	}
+}
 
 const setWindow = (window: number, interval: number) => {
 	dataWindow = window
 	dataInterval = interval
+	load_metrics(dataWindow, dataInterval);
 }
 
 let loaded = $state(false)
 onMount(async () => {
 	await load_host_names()
+	await load_metrics(dataWindow, dataInterval);
 	loaded = true
 })
+onDestroy(() => {
+	if (metrics_timeout !== null) {
+		clearTimeout(metrics_timeout)
+	}
+})
 </script>
 
 {#if loaded}
@@ -111,10 +206,10 @@ onMount(async () => {
 			<div class="grid">
 				{#each group.graphs as graph (graph.metric)}
 					<HostMetricsGraph
-						window={dataWindow}
-						interval={dataInterval}
 						metric={graph.metric}
 						data_type={graph.data_type}
+						timestamps={metrics.timestamps}
+						metrics={metrics.metrics[graph.metric]}
 						aggregate={showAggregate}
 					/>
 				{/each}