Add admin file details page for debugging file entries

2026-09-10 00:32:49 +02:00
parent 050b5bbf99
commit 4eb0d9bedd
12 changed files with 302 additions and 69 deletions
--- a/svelte/package-lock.json
+++ b/svelte/package-lock.json
@@ -28,7 +28,6 @@
        "@rollup/plugin-typescript": "^11.1.6",
        "@types/jsmediatags": "^3.9.6",
        "@types/node": "^24.10.0",
-        "baseline-browser-mapping": "^2.10.13",
        "rollup": "^4.24.4",
        "rollup-plugin-livereload": "^2.0.5",
        "rollup-plugin-svelte": "^7.2.2",
@@ -2708,9 +2707,9 @@
      "license": "MIT"
    },
    "node_modules/baseline-browser-mapping": {
-      "version": "2.10.13",
-      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.13.tgz",
-      "integrity": "sha512-BL2sTuHOdy0YT1lYieUxTw/QMtPBC3pmlJC6xk8BBYVv6vcw3SGdKemQ+Xsx9ik2F/lYDO9tqsFQH1r9PFuHKw==",
+      "version": "2.11.12",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.12.tgz",
+      "integrity": "sha512-r7WnVImvVCeFpf2DOXfy41aPWzeNg3H/A2X4dKmy1QL0MSyyk/e7z8ihJ3N6Nn2PsdhkVlqnEfnUE4a05P2aTA==",
      "devOptional": true,
      "license": "Apache-2.0",
      "bin": {
@@ -2804,9 +2803,9 @@
      "license": "MIT"
    },
    "node_modules/caniuse-lite": {
-      "version": "1.0.30001749",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001749.tgz",
-      "integrity": "sha512-0rw2fJOmLfnzCRbkm8EyHL8SvI2Apu5UbnQuTsJ0ClgrH8hcwFooJ1s5R0EP8o8aVrFu8++ae29Kt9/gZAZp/Q==",
+      "version": "1.0.30001806",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz",
+      "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==",
      "devOptional": true,
      "funding": [
        {
--- a/svelte/package.json
+++ b/svelte/package.json
@@ -18,7 +18,6 @@
    "@rollup/plugin-typescript": "^11.1.6",
    "@types/jsmediatags": "^3.9.6",
    "@types/node": "^24.10.0",
-    "baseline-browser-mapping": "^2.10.13",
    "rollup": "^4.24.4",
    "rollup-plugin-livereload": "^2.0.5",
    "rollup-plugin-svelte": "^7.2.2",
--- a/svelte/src/admin_panel/FileInfo.svelte
+++ b/svelte/src/admin_panel/FileInfo.svelte
@@ -0,0 +1,180 @@
+<script lang="ts">
+import { onMount } from "svelte"
+import { formatDataVolume, formatDate, formatNumber } from "util/Formatting"
+import { get_endpoint, check_response } from "lib/NovaAPI"
+import { get_status, type Peer } from "lib/AdminAPI"
+
+let hash = $state("")
+let file = $state(null)
+let error = $state("")
+let loading = $state(false)
+
+const load = async () => {
+	const h = hash.trim().replace(/^0x/i, "")
+	file = null
+	error = ""
+	loading = true
+
+	try {
+		file = await check_response(
+			await fetch(get_endpoint() + "/admin/file/" + h + "/info")
+		)
+	} catch (err) {
+		error = err.message || JSON.stringify(err)
+	} finally {
+		loading = false
+	}
+}
+
+// Server UUID to peer, so shard locations can be shown as hostnames. A
+// decommissioned server is still in the shard map but no longer in the peer
+// list, so the UUID has to remain the fallback
+let peers: { [id: string]: Peer } = $state({})
+onMount(async () => {
+	const status = await get_status()
+	peers = Object.fromEntries(status.peers.map(p => [p.id, p]))
+})
+const server = (id: string) => {
+	const peer = peers[id]
+	return peer ? peer.hostname : id
+}
+
+// Go marshals a missing timestamp as the zero time instead of null
+const date = (d: string) => (!d || d.startsWith("0001-")) ? "never" : formatDate(d)
+</script>
+
+<section>
+	<h2>File inspector</h2>
+	<form onsubmit={e => { e.preventDefault(); load() }}>
+		<label for="hash_input">File hash (hex sha256)</label>
+		<input id="hash_input" bind:value={hash} onclick={e => e.currentTarget.select()}/>
+		<button type="submit" disabled={loading}>
+			<i class="icon">search</i> Look up
+		</button>
+	</form>
+
+	{#if error}
+		<p class="highlight_red">{error}</p>
+	{/if}
+</section>
+
+{#if file}
+<section>
+	<h3>File row</h3>
+	<div class="table_scroll">
+		<table>
+			<tbody>
+				<tr><td>Hash</td><td>{file.hash}</td></tr>
+				<tr><td>Hash UUID</td><td>{file.hash_uuid}</td></tr>
+				<tr><td>SHA1</td><td>{file.hash_sha1}</td></tr>
+				<tr><td>MD5</td><td>{file.hash_md5}</td></tr>
+				<tr><td>Size</td><td>{formatDataVolume(file.size, 4)} ({file.size} B)</td></tr>
+				<tr><td>Created</td><td>{date(file.created)}</td></tr>
+				<tr><td>Thumbnail</td><td>{file.thumbnail_status || "none"}</td></tr>
+				<tr>
+					<td>Abuse</td>
+					<td>
+						{#if file.abuse_type}
+							{file.abuse_type}, reported by {file.abuse_reporter_name}
+							&lt;{file.abuse_reporter_email}&gt;
+						{:else}
+							none
+						{/if}
+					</td>
+				</tr>
+				<tr><td>Last referenced</td><td>{date(file.last_referenced)}</td></tr>
+				<tr><td>Last dereferenced</td><td>{date(file.last_dereferenced)}</td></tr>
+				<tr>
+					<td>Collectable after</td>
+					<td>
+						{date(file.collectable_after)}
+						{#if file.live_references > 0}
+							(blocked by {file.live_references} live references)
+						{/if}
+					</td>
+				</tr>
+				<tr>
+					<td>Cache servers</td>
+					<td>{file.cache_servers?.map(server).join(", ") || "none"}</td>
+				</tr>
+				<tr>
+					<td>Reed-Solomon</td>
+					<td>
+						{file.data_shards} data + {file.parity_shards} parity,
+						{formatDataVolume(file.shard_size, 4)} per shard,
+						{formatDataVolume(file.shard_block_size, 4)} blocks
+					</td>
+				</tr>
+				<tr>
+					<td>Redundancy</td>
+					<td>
+						{formatNumber(file.redundancy, 3)}x
+						{#if file.missing_shards?.length}
+							<span class="highlight_red">
+								missing shards: {file.missing_shards.join(", ")}
+							</span>
+						{/if}
+					</td>
+				</tr>
+			</tbody>
+		</table>
+	</div>
+
+	<h3>Storage locations ({file.shards.length} shards)</h3>
+	<div class="table_scroll">
+		<table>
+			<thead>
+				<tr><td>Index</td><td>Shard hash</td><td>Server</td></tr>
+			</thead>
+			<tbody>
+				{#each file.shards as shard}
+					<tr>
+						<td>{shard.index}</td>
+						<td>{shard.hash}</td>
+						<td title={shard.server_id}>{server(shard.server_id)}</td>
+					</tr>
+				{/each}
+			</tbody>
+		</table>
+	</div>
+
+	<h3>
+		References ({file.references.length}{file.reference_limit ? "+" : ""},
+		{file.live_references} live)
+	</h3>
+	<div class="table_scroll">
+		<table>
+			<thead>
+				<tr>
+					<td>ID</td>
+					<td>Name</td>
+					<td>User</td>
+					<td>Size</td>
+					<td>Type</td>
+					<td>Views</td>
+					<td>Created</td>
+					<td>Modified</td>
+					<td>Deleted</td>
+					<td>Abuse</td>
+				</tr>
+			</thead>
+			<tbody>
+				{#each file.references as ref}
+					<tr>
+						<td><a href="/d/{ref.id}">{ref.id}</a></td>
+						<td>{ref.name}</td>
+						<td>{ref.user_id}</td>
+						<td>{formatDataVolume(ref.size, 3)}</td>
+						<td>{ref.type}</td>
+						<td>{ref.views}</td>
+						<td>{date(ref.created)}</td>
+						<td>{date(ref.modified)}</td>
+						<td>{ref.deleted ? formatDate(ref.deleted) : ""}</td>
+						<td>{ref.abuse_type}</td>
+					</tr>
+				{/each}
+			</tbody>
+		</table>
+	</div>
+</section>
+{/if}
--- a/svelte/src/admin_panel/Home.svelte
+++ b/svelte/src/admin_panel/Home.svelte
@@ -6,6 +6,7 @@ import { color_by_name } from "util/Util";
 import ServerDiagnostics from "./ServerDiagnostics.svelte";
 import PeerTable from "./PeerTable.svelte";
 import { get_endpoint } from "lib/NovaAPI";
+import { get_status, type ServerStatus } from "lib/AdminAPI";

 let graphEgress: Chart = $state()
 let graphDownloads: Chart = $state()
@@ -60,34 +61,32 @@ const loadGraph = (minutes: number, interval: number, live: boolean) => {
 // Load performance statistics

 let lastOrder: string = $state();
-let status = $state({
+let status: ServerStatus = $state({
+	running_since: "",
 	pid: 0,
 	cpu_profile_running_since: "",
 	db_latency: 0,
 	db_time: "",
+	query_statistics: [],
+	filesystem_watcher_threads: 0,
+	filesystem_watcher_listeners: 0,
+	peers: [],
 	cache_threshold: 0,
-	local_read_size: 0,
-	local_read_size_per_sec: 0,
 	local_reads: 0,
 	local_reads_per_sec: 0,
+	local_read_size: 0,
+	local_read_size_per_sec: 0,
 	local_readers: 0,
-	neighbour_read_size: 0,
-	neighbour_read_size_per_sec: 0,
 	neighbour_reads: 0,
 	neighbour_reads_per_sec: 0,
+	neighbour_read_size: 0,
+	neighbour_read_size_per_sec: 0,
 	neighbour_readers: 0,
-	remote_read_size: 0,
-	remote_read_size_per_sec: 0,
 	remote_reads: 0,
 	remote_reads_per_sec: 0,
+	remote_read_size: 0,
+	remote_read_size_per_sec: 0,
 	remote_readers: 0,
-	peers: [],
-	query_statistics: [],
-	running_since: "",
-	stats_watcher_listeners: 0,
-	stats_watcher_threads: 0,
-	filesystem_watcher_listeners: 0,
-	filesystem_watcher_threads: 0,
 })
 let total_reads = $derived(status.local_reads + status.neighbour_reads + status.remote_reads)
 let total_read_size = $derived(status.local_read_size + status.neighbour_read_size + status.remote_read_size)
@@ -95,9 +94,7 @@ let total_read_size = $derived(status.local_read_size + status.neighbour_read_si
 function getStats(order: string) {
 	lastOrder = order

-	fetch(get_endpoint() + "/status").then(
-		resp => resp.json()
-	).then(resp => {
+	get_status().then(resp => {
 		// Sort all queries by the selected sort column
 		resp.query_statistics.sort((a, b) => {
 			if (typeof (a[order]) === "number") {
--- a/svelte/src/admin_panel/HostMetrics.svelte
+++ b/svelte/src/admin_panel/HostMetrics.svelte
@@ -100,6 +100,14 @@ const groups: {
 			{metric: "pixelstore_reed_solomon_read_size", data_type: "bytes"},
 			{metric: "pixelstore_read_retry_success", data_type: "number"},
 			{metric: "pixelstore_read_retry_error", data_type: "number"},
+			{metric: "pixelstore_remote_read", data_type: "number"},
+			{metric: "pixelstore_remote_read_ttfb", data_type: "duration"},
+			{
+				metric: "pixelstore_remote_read_ttfb_avg",
+				agg_base: "pixelstore_remote_read_ttfb",
+				agg_divisor: "pixelstore_remote_read",
+				data_type: "duration",
+			},
 		],
 	}, {
 		title: "Pixelstore shards", expanded: false,
--- a/svelte/src/admin_panel/HostMetricsLib.ts
+++ b/svelte/src/admin_panel/HostMetricsLib.ts
@@ -1,4 +1,5 @@
-import { loading_finish, loading_start } from "lib/Loading";
+import { get_status } from "lib/AdminAPI";
+import { loading_run } from "lib/Loading";
 import { check_response, get_endpoint } from "lib/NovaAPI";
 import hsl2rgb from "pure-color/convert/hsl2rgb";
 import rgb2hex from "pure-color/convert/rgb2hex";
@@ -61,15 +62,10 @@ export const host_label = async (id: string): Promise<string> => {
 }

 export const load_host_names = async () => {
-	loading_start()
-	try {
-		const req = await fetch(get_endpoint() + "/status")
-		const j = await req.json()
-
-		for (const peer of j.peers) {
+	await loading_run(async () => {
+		const status = await get_status()
+		for (const peer of status.peers) {
 			host_names[peer.id] = peer.hostname
 		}
-	} finally {
-		loading_finish()
-	}
+	})
 }
--- a/svelte/src/admin_panel/PeerTable.svelte
+++ b/svelte/src/admin_panel/PeerTable.svelte
@@ -3,37 +3,17 @@ import { flip } from "svelte/animate";
 import { formatDataVolume } from "util/Formatting";
 import SortButton from "layout/SortButton.svelte";
 import { loading_run } from "lib/Loading";
-import { admin_decommission_node } from "lib/AdminAPI";
+import { admin_decommission_node, type Peer } from "lib/AdminAPI";

-let {
-	peers = $bindable([])
-}: {
-	peers: {
-		id: string
-		ip: string
-		port: number
-		hostname: string
-		role: string
-		reachable: boolean
-		unreachable_count: number
-		latency: number
-		last_seen: string
-		free_space: number
-		min_free_space: number
-		load_1_min: number
-		load_5_min: number
-		load_15_min: number
-		avg_network_tx: number
-		avg_network_rx: number
-		port_speed: number
-		cache_threshold: number
+// The computed columns are not part of the API response, they are filled in
+// by the effect below
+type TablePeer = Peer & {
+	avg_network_total?: number
+	usage_percent?: number
+	network_ratio?: number
+}

-		// Our props
-		avg_network_total?: number
-		usage_percent?: number
-		network_ratio?: number
-	}[]
-} = $props();
+let { peers = $bindable([]) }: { peers: TablePeer[] } = $props();

 // If not all peers are online we show the decommission button
 let all_reachable = $derived(peers.reduce((acc, val) => {
--- a/svelte/src/admin_panel/Router.svelte
+++ b/svelte/src/admin_panel/Router.svelte
@@ -11,6 +11,7 @@ import PayPalTaxes from "./PayPalTaxes.svelte";
 import UserBans from "./user_bans/UserBans.svelte";
 import InvoiceVat from "./InvoiceVAT.svelte";
 import HostMetrics from "./HostMetrics.svelte";
+import FileInfo from "./FileInfo.svelte";

 let pages = [
 	{
@@ -53,6 +54,11 @@ let pages = [
 		title: "Host Metrics",
 		icon: "area_chart",
 		component: HostMetrics,
+	}, {
+		path: "/admin/file_info",
+		title: "File Inspector",
+		icon: "search",
+		component: FileInfo,
 	}, {
 		path: "/admin/mollie_settlements",
 		title: "Prepaid accounting",
--- a/svelte/src/lib/AdminAPI.ts
+++ b/svelte/src/lib/AdminAPI.ts
@@ -39,3 +39,66 @@ export const admin_decommission_node = async (peer_id: string) => {
 		)
 	) as GenericResponse
 };
+
+export type Peer = {
+	id: string
+	ip: string
+	port: number
+	hostname: string
+	role: string
+	reachable: boolean
+	unreachable_count: number
+	latency: number
+	last_seen: string
+	free_space: number
+	min_free_space: number
+	load_1_min: number
+	load_5_min: number
+	load_15_min: number
+	avg_network_tx: number
+	avg_network_rx: number
+	port_speed: number
+	cache_threshold: number
+}
+
+export type QueryStats = {
+	query_name: string
+	calls: number
+	total_duration: number
+	average_duration: number
+	callers: { name: string, count: number }[]
+}
+
+export type ServerStatus = {
+	running_since: string
+	pid: number
+	cpu_profile_running_since: string
+	db_latency: number
+	db_time: string
+	query_statistics: QueryStats[]
+	filesystem_watcher_threads: number
+	filesystem_watcher_listeners: number
+	peers: Peer[]
+	cache_threshold: number
+	local_reads: number
+	local_reads_per_sec: number
+	local_read_size: number
+	local_read_size_per_sec: number
+	local_readers: number
+	neighbour_reads: number
+	neighbour_reads_per_sec: number
+	neighbour_read_size: number
+	neighbour_read_size_per_sec: number
+	neighbour_readers: number
+	remote_reads: number
+	remote_reads_per_sec: number
+	remote_read_size: number
+	remote_read_size_per_sec: number
+	remote_readers: number
+}
+
+export const get_status = async () => {
+	return await check_response(
+		await fetch(get_endpoint() + "/status")
+	) as ServerStatus
+};
--- a/svelte/src/util/Formatting.ts
+++ b/svelte/src/util/Formatting.ts
@@ -72,8 +72,13 @@ export const formatDuration = (ms: number, decimals: number) => {
 		res += Math.floor((ms % hour) / minute) + "m "
 		remainingDecimals--
 	}
+	if (ms >= second && remainingDecimals > 0) {
+		res += Math.floor((ms % minute) / second) + "s "
+		remainingDecimals--
+	}
 	if (remainingDecimals > 0) {
-		res += ((ms % minute) / second).toFixed(remainingDecimals) + "s"
+		res += (ms % second) + "ms "
+		remainingDecimals--
 	}
 	return res
 }
--- a/svelte/tsconfig.json
+++ b/svelte/tsconfig.json
@@ -1,7 +1,7 @@
 {
 	"compilerOptions": {
 		"lib": [
-			"es2017",
+			"es2019",
 			"dom"
 		],
 		"paths": {
@@ -10,7 +10,7 @@
 			]
 		},
 		"verbatimModuleSyntax": true,
-		"target": "ES2017",
+		"target": "ES2019",
 		"noUnusedLocals": true,
 		"module": "preserve",
 		"moduleResolution": "bundler",