Add host metrics dashboard

svelte/src/admin_panel/HostMetrics.svelte

@@ -0,0 +1,131 @@
+<script lang="ts">
+import { onMount } from "svelte";
+import HostMetricsGraph from "./HostMetricsGraph.svelte";
+import { load_host_names } from "./HostMetricsLib";
+import Expandable from "util/Expandable.svelte";
+
+const groups: {
+	title: string,
+	graphs: {
+		metric: string,
+		data_type: string,
+	}[],
+}[] = [
+	{
+		title: "API",
+		graphs: [
+			{metric: "api_request", data_type: "number"},
+			{metric: "api_request_duration", data_type: "duration"},
+			{metric: "api_error", data_type: "number"},
+			{metric: "api_panic", data_type: "number"},
+		],
+	}, {
+		title: "Task scheduler",
+		graphs: [
+			{metric: "scheduler_filesystem_remove_orphan", data_type: "number"},
+			{metric: "scheduler_timeseries_delete", data_type: "number"},
+		],
+	}, {
+		title: "Database",
+		graphs: [
+			{metric: "database_query", data_type: "number"},
+			{metric: "database_query_duration", data_type: "duration"},
+			{metric: "database_query_rows", data_type: "number"},
+			{metric: "database_query_retries", data_type: "number"},
+			{metric: "database_query_error", data_type: "number"},
+		],
+	}, {
+		title: "Pixelstore",
+		graphs: [
+			{metric: "pixelstore_write", data_type: "number"},
+			{metric: "pixelstore_write_size", data_type: "bytes"},
+			{metric: "pixelstore_cache_fetch", data_type: "number"},
+			{metric: "pixelstore_cache_fetch_size", data_type: "bytes"},
+			{metric: "pixelstore_cache_delete", data_type: "number"},
+			{metric: "pixelstore_connection_error", data_type: "number"},
+			{metric: "pixelstore_peer_down", data_type: "number"},
+			{metric: "pixelstore_peer_up", data_type: "number"},
+		],
+	}, {
+		title: "Pixelstore reads",
+		graphs: [
+			{metric: "pixelstore_cache_read", data_type: "number"},
+			{metric: "pixelstore_cache_read_size", data_type: "bytes"},
+			{metric: "pixelstore_neighbour_read", data_type: "number"},
+			{metric: "pixelstore_neighbour_read_size", data_type: "bytes"},
+			{metric: "pixelstore_reed_solomon_read", data_type: "number"},
+			{metric: "pixelstore_reed_solomon_read_size", data_type: "bytes"},
+		],
+	}, {
+		title: "Pixelstore shards",
+		graphs: [
+			{metric: "pixelstore_shard_delete", data_type: "number"},
+			{metric: "pixelstore_shard_delete_size", data_type: "bytes"},
+			{metric: "pixelstore_shard_repair", data_type: "number"},
+			{metric: "pixelstore_shard_repair_size", data_type: "bytes"},
+			{metric: "pixelstore_shard_move", data_type: "number"},
+			{metric: "pixelstore_shard_move_size", data_type: "bytes"},
+		],
+	},
+]
+
+let dataWindow: number = $state(60)
+let dataInterval: number = $state(1)
+
+const setWindow = (window: number, interval: number) => {
+	dataWindow = window
+	dataInterval = interval
+}
+
+let loaded = $state(false)
+onMount(async () => {
+	await load_host_names()
+	loaded = true
+})
+</script>
+
+{#if loaded}
+	<div class="highlight_border" style="margin-bottom: 6px;">
+		<button onclick={() => setWindow(60, 1)}>Hour 1m</button>
+		<button onclick={() => setWindow(720, 1)}>Half Day 1m</button>
+		<button onclick={() => setWindow(1440, 60)}>Day 1h</button>
+		<button onclick={() => setWindow(10080, 60)}>Week 1h</button>
+		<button onclick={() => setWindow(43200, 60)}>Month 1h</button>
+		<button onclick={() => setWindow(131400, 1440)}>Quarter 1d</button>
+		<button onclick={() => setWindow(262800, 1440)}>Half-year 1d</button>
+		<button onclick={() => setWindow(525600, 1440)}>Year 1d</button>
+		<button onclick={() => setWindow(1051200, 1440)}>Two Years 1d</button>
+		<button onclick={() => setWindow(2628000, 1440)}>Five Years 1d</button>
+	</div>
+
+	{#each groups as group (group.title)}
+		<Expandable click_expand expanded>
+			{#snippet header()}
+				<div class="title">{group.title}</div>
+			{/snippet}
+
+			<div class="grid">
+				{#each group.graphs as graph (graph.metric)}
+					<HostMetricsGraph
+						window={dataWindow}
+						interval={dataInterval}
+						metric={graph.metric}
+						data_type={graph.data_type}
+					/>
+				{/each}
+			</div>
+		</Expandable>
+	{/each}
+{/if}
+
+<style>
+.title {
+	font-size: 1.6em;
+	padding: 4px;
+	text-align: center;
+}
+.grid {
+	display: grid;
+	grid-template-columns: repeat(auto-fit, minmax(500px, 1fr));
+}
+</style>

svelte/src/admin_panel/HostMetricsGraph.svelte

@@ -0,0 +1,101 @@
+<script lang="ts">
+import { onMount } from "svelte";
+import Chart from "util/Chart.svelte";
+import { get_endpoint } from "lib/PixeldrainAPI";
+    import { host_colour, host_label } from "./HostMetricsLib";
+
+let {
+	metric = "",
+	window = 0, // Size of the data window in minutes
+	interval = 0, // Interval of the datapoints in minutes
+	data_type = "number"
+}: {
+	metric: string;
+	window: number;
+	interval: number;
+	data_type?: string;
+} = $props();
+
+let chart: Chart = $state()
+let chartTimeout: NodeJS.Timeout = null
+
+const loadGraph = () => {
+	if (chartTimeout !== null) { clearTimeout(chartTimeout) }
+	chartTimeout = setTimeout(() => { loadGraph() }, 10000)
+
+	let today = new Date()
+	let start = new Date()
+	start.setMinutes(start.getMinutes() - window)
+
+	fetch(
+		get_endpoint() + "/admin/host_metrics" +
+		"?start=" + start.toISOString() +
+		"&end=" + today.toISOString() +
+		"&metric="+ metric +
+		"&interval=" + interval
+	).then(resp => {
+		if (!resp.ok) { return Promise.reject("Error: " + resp.status); }
+		return resp.json();
+	}).then(async resp => {
+
+		resp.timestamps.forEach((val: string, idx: number) => {
+			let date = new Date(val);
+			let dateStr: string = date.getFullYear().toString();
+			dateStr += "-" + ("00" + (date.getMonth() + 1)).slice(-2);
+			dateStr += "-" + ("00" + date.getDate()).slice(-2);
+			dateStr += " " + ("00" + date.getHours()).slice(-2);
+			dateStr += ":" + ("00" + date.getMinutes()).slice(-2);
+			resp.timestamps[idx] = " " + dateStr + " "; // Poor man's padding
+		});
+		chart.data().labels = resp.timestamps;
+
+		let i = 0
+		for (const host of Object.keys(resp.host_amounts).sort()) {
+			if (chart.data().datasets[i] === undefined) {
+				chart.data().datasets[i] = {
+					label: await host_label(host),
+					data: [],
+					borderWidth: 1,
+					pointRadius: 0,
+				}
+			}
+			chart.data().datasets[i].borderColor = host_colour(host)
+			chart.data().datasets[i].backgroundColor = host_colour(host)
+
+			if (data_type === "duration") {
+				for (let i = 0; i < resp.host_amounts[host].length; i++) {
+					// Go durations are expressed on nanoseconds, divide by 1
+					// million to convert to milliseconds
+					resp.host_amounts[host][i] /= 1000000
+				}
+			}
+
+			chart.data().datasets[i].data = resp.host_amounts[host]
+			i++
+		}
+
+		chart.update()
+	})
+}
+
+onMount(() => {
+	loadGraph();
+
+	return () => {
+		if (chartTimeout !== null) {
+			clearTimeout(chartTimeout)
+		}
+	}
+})
+</script>
+
+<div>
+	<div class="title">{metric}</div>
+	<Chart bind:this={chart} data_type={data_type} legend={false} ticks={false} animations={false} />
+</div>
+
+<style>
+.title {
+	font-size: 1.2em;
+}
+</style>

svelte/src/admin_panel/HostMetricsLib.ts

@@ -0,0 +1,50 @@
+import { loading_finish, loading_start } from "lib/Loading";
+import { get_endpoint } from "lib/PixeldrainAPI";
+import hsl2rgb from "pure-color/convert/hsl2rgb";
+import rgb2hex from "pure-color/convert/rgb2hex";
+
+let host_colours: { [key: string]: string } = {}
+export const host_colour = (id: string): string => {
+	let host_count: number = Object.keys(host_colours).length
+	if (host_colours[id] === undefined) {
+		host_colours[id] = ""
+	} else {
+		return host_colours[id]
+	}
+
+	const colour_interval = 360 / (host_count + 1)
+
+	var i = 0
+	for (const host of Object.keys(host_colours).sort()) {
+		host_colours[host] = rgb2hex(hsl2rgb([i * colour_interval, 100, 70]))
+		i++
+	}
+
+	return host_colours[id]
+}
+
+let host_names: { [key: string]: string } = {}
+export const host_label = async (id: string): Promise<string> => {
+	if (id === "00000000-0000-0000-0000-000000000000") {
+		return "task scheduler"
+	}
+	if (host_names[id] === undefined) {
+		return id
+	}
+
+	return host_names[id]
+}
+
+export const load_host_names = async () => {
+	loading_start()
+	try {
+		const req = await fetch(get_endpoint() + "/status")
+		const j = await req.json()
+
+		for (const peer of j.peers) {
+			host_names[peer.id] = peer.hostname
+		}
+	} finally {
+		loading_finish()
+	}
+}

svelte/src/admin_panel/Router.svelte

@@ -10,6 +10,7 @@ import MollieSettlements from "./MollieSettlements.svelte";
 import PayPalTaxes from "./PayPalTaxes.svelte";
 import UserBans from "./user_bans/UserBans.svelte";
 import InvoiceVat from "./InvoiceVAT.svelte";
+import HostMetrics from "./HostMetrics.svelte";
 
 let pages = [
 	{
@@ -47,6 +48,11 @@ let pages = [
 		title: "User Management",
 		icon: "person",
 		component: UserManagement,
+	}, {
+		path: "/admin/host_metrics",
+		title: "Host Metrics",
+		icon: "area_chart",
+		component: HostMetrics,
 	}, {
 		path: "/admin/mollie_settlements",
 		title: "Prepaid accounting",