From 4eb0d9bedd9e472b682fc649ee4cdcde9733d486 Mon Sep 17 00:00:00 2001 From: Wim Brand Date: Thu, 10 Sep 2026 00:32:49 +0200 Subject: [PATCH] Add admin file details page for debugging file entries --- svelte/package-lock.json | 13 +- svelte/package.json | 1 - svelte/src/admin_panel/FileInfo.svelte | 180 ++++++++++++++++++++++ svelte/src/admin_panel/Home.svelte | 31 ++-- svelte/src/admin_panel/HostMetrics.svelte | 8 + svelte/src/admin_panel/HostMetricsLib.ts | 16 +- svelte/src/admin_panel/PeerTable.svelte | 38 ++--- svelte/src/admin_panel/Router.svelte | 6 + svelte/src/lib/AdminAPI.ts | 63 ++++++++ svelte/src/util/Formatting.ts | 7 +- svelte/tsconfig.json | 4 +- webcontroller/style.go | 4 +- 12 files changed, 302 insertions(+), 69 deletions(-) create mode 100644 svelte/src/admin_panel/FileInfo.svelte diff --git a/svelte/package-lock.json b/svelte/package-lock.json index 7b61680..b42f134 100644 --- a/svelte/package-lock.json +++ b/svelte/package-lock.json @@ -28,7 +28,6 @@ "@rollup/plugin-typescript": "^11.1.6", "@types/jsmediatags": "^3.9.6", "@types/node": "^24.10.0", - "baseline-browser-mapping": "^2.10.13", "rollup": "^4.24.4", "rollup-plugin-livereload": "^2.0.5", "rollup-plugin-svelte": "^7.2.2", @@ -2708,9 +2707,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.13", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.13.tgz", - "integrity": "sha512-BL2sTuHOdy0YT1lYieUxTw/QMtPBC3pmlJC6xk8BBYVv6vcw3SGdKemQ+Xsx9ik2F/lYDO9tqsFQH1r9PFuHKw==", + "version": "2.11.12", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.12.tgz", + "integrity": "sha512-r7WnVImvVCeFpf2DOXfy41aPWzeNg3H/A2X4dKmy1QL0MSyyk/e7z8ihJ3N6Nn2PsdhkVlqnEfnUE4a05P2aTA==", "devOptional": true, "license": "Apache-2.0", "bin": { @@ -2804,9 +2803,9 @@ "license": "MIT" }, "node_modules/caniuse-lite": { - "version": "1.0.30001749", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001749.tgz", - "integrity": "sha512-0rw2fJOmLfnzCRbkm8EyHL8SvI2Apu5UbnQuTsJ0ClgrH8hcwFooJ1s5R0EP8o8aVrFu8++ae29Kt9/gZAZp/Q==", + "version": "1.0.30001806", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", + "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", "devOptional": true, "funding": [ { diff --git a/svelte/package.json b/svelte/package.json index 436a7c0..68a548d 100644 --- a/svelte/package.json +++ b/svelte/package.json @@ -18,7 +18,6 @@ "@rollup/plugin-typescript": "^11.1.6", "@types/jsmediatags": "^3.9.6", "@types/node": "^24.10.0", - "baseline-browser-mapping": "^2.10.13", "rollup": "^4.24.4", "rollup-plugin-livereload": "^2.0.5", "rollup-plugin-svelte": "^7.2.2", diff --git a/svelte/src/admin_panel/FileInfo.svelte b/svelte/src/admin_panel/FileInfo.svelte new file mode 100644 index 0000000..1457f3c --- /dev/null +++ b/svelte/src/admin_panel/FileInfo.svelte @@ -0,0 +1,180 @@ + + +
+

File inspector

+
{ e.preventDefault(); load() }}> + + e.currentTarget.select()}/> + +
+ + {#if error} +

{error}

+ {/if} +
+ +{#if file} +
+

File row

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Hash{file.hash}
Hash UUID{file.hash_uuid}
SHA1{file.hash_sha1}
MD5{file.hash_md5}
Size{formatDataVolume(file.size, 4)} ({file.size} B)
Created{date(file.created)}
Thumbnail{file.thumbnail_status || "none"}
Abuse + {#if file.abuse_type} + {file.abuse_type}, reported by {file.abuse_reporter_name} + <{file.abuse_reporter_email}> + {:else} + none + {/if} +
Last referenced{date(file.last_referenced)}
Last dereferenced{date(file.last_dereferenced)}
Collectable after + {date(file.collectable_after)} + {#if file.live_references > 0} + (blocked by {file.live_references} live references) + {/if} +
Cache servers{file.cache_servers?.map(server).join(", ") || "none"}
Reed-Solomon + {file.data_shards} data + {file.parity_shards} parity, + {formatDataVolume(file.shard_size, 4)} per shard, + {formatDataVolume(file.shard_block_size, 4)} blocks +
Redundancy + {formatNumber(file.redundancy, 3)}x + {#if file.missing_shards?.length} + + missing shards: {file.missing_shards.join(", ")} + + {/if} +
+
+ +

Storage locations ({file.shards.length} shards)

+
+ + + + + + {#each file.shards as shard} + + + + + + {/each} + +
IndexShard hashServer
{shard.index}{shard.hash}{server(shard.server_id)}
+
+ +

+ References ({file.references.length}{file.reference_limit ? "+" : ""}, + {file.live_references} live) +

+
+ + + + + + + + + + + + + + + + + {#each file.references as ref} + + + + + + + + + + + + + {/each} + +
IDNameUserSizeTypeViewsCreatedModifiedDeletedAbuse
{ref.id}{ref.name}{ref.user_id}{formatDataVolume(ref.size, 3)}{ref.type}{ref.views}{date(ref.created)}{date(ref.modified)}{ref.deleted ? formatDate(ref.deleted) : ""}{ref.abuse_type}
+
+
+{/if} diff --git a/svelte/src/admin_panel/Home.svelte b/svelte/src/admin_panel/Home.svelte index 5764b77..0b9db5f 100644 --- a/svelte/src/admin_panel/Home.svelte +++ b/svelte/src/admin_panel/Home.svelte @@ -6,6 +6,7 @@ import { color_by_name } from "util/Util"; import ServerDiagnostics from "./ServerDiagnostics.svelte"; import PeerTable from "./PeerTable.svelte"; import { get_endpoint } from "lib/NovaAPI"; +import { get_status, type ServerStatus } from "lib/AdminAPI"; let graphEgress: Chart = $state() let graphDownloads: Chart = $state() @@ -60,34 +61,32 @@ const loadGraph = (minutes: number, interval: number, live: boolean) => { // Load performance statistics let lastOrder: string = $state(); -let status = $state({ +let status: ServerStatus = $state({ + running_since: "", pid: 0, cpu_profile_running_since: "", db_latency: 0, db_time: "", + query_statistics: [], + filesystem_watcher_threads: 0, + filesystem_watcher_listeners: 0, + peers: [], cache_threshold: 0, - local_read_size: 0, - local_read_size_per_sec: 0, local_reads: 0, local_reads_per_sec: 0, + local_read_size: 0, + local_read_size_per_sec: 0, local_readers: 0, - neighbour_read_size: 0, - neighbour_read_size_per_sec: 0, neighbour_reads: 0, neighbour_reads_per_sec: 0, + neighbour_read_size: 0, + neighbour_read_size_per_sec: 0, neighbour_readers: 0, - remote_read_size: 0, - remote_read_size_per_sec: 0, remote_reads: 0, remote_reads_per_sec: 0, + remote_read_size: 0, + remote_read_size_per_sec: 0, remote_readers: 0, - peers: [], - query_statistics: [], - running_since: "", - stats_watcher_listeners: 0, - stats_watcher_threads: 0, - filesystem_watcher_listeners: 0, - filesystem_watcher_threads: 0, }) let total_reads = $derived(status.local_reads + status.neighbour_reads + status.remote_reads) let total_read_size = $derived(status.local_read_size + status.neighbour_read_size + status.remote_read_size) @@ -95,9 +94,7 @@ let total_read_size = $derived(status.local_read_size + status.neighbour_read_si function getStats(order: string) { lastOrder = order - fetch(get_endpoint() + "/status").then( - resp => resp.json() - ).then(resp => { + get_status().then(resp => { // Sort all queries by the selected sort column resp.query_statistics.sort((a, b) => { if (typeof (a[order]) === "number") { diff --git a/svelte/src/admin_panel/HostMetrics.svelte b/svelte/src/admin_panel/HostMetrics.svelte index c3d0da4..77480dc 100644 --- a/svelte/src/admin_panel/HostMetrics.svelte +++ b/svelte/src/admin_panel/HostMetrics.svelte @@ -100,6 +100,14 @@ const groups: { {metric: "pixelstore_reed_solomon_read_size", data_type: "bytes"}, {metric: "pixelstore_read_retry_success", data_type: "number"}, {metric: "pixelstore_read_retry_error", data_type: "number"}, + {metric: "pixelstore_remote_read", data_type: "number"}, + {metric: "pixelstore_remote_read_ttfb", data_type: "duration"}, + { + metric: "pixelstore_remote_read_ttfb_avg", + agg_base: "pixelstore_remote_read_ttfb", + agg_divisor: "pixelstore_remote_read", + data_type: "duration", + }, ], }, { title: "Pixelstore shards", expanded: false, diff --git a/svelte/src/admin_panel/HostMetricsLib.ts b/svelte/src/admin_panel/HostMetricsLib.ts index 2eb9ed7..8a80dd9 100644 --- a/svelte/src/admin_panel/HostMetricsLib.ts +++ b/svelte/src/admin_panel/HostMetricsLib.ts @@ -1,4 +1,5 @@ -import { loading_finish, loading_start } from "lib/Loading"; +import { get_status } from "lib/AdminAPI"; +import { loading_run } from "lib/Loading"; import { check_response, get_endpoint } from "lib/NovaAPI"; import hsl2rgb from "pure-color/convert/hsl2rgb"; import rgb2hex from "pure-color/convert/rgb2hex"; @@ -61,15 +62,10 @@ export const host_label = async (id: string): Promise => { } export const load_host_names = async () => { - loading_start() - try { - const req = await fetch(get_endpoint() + "/status") - const j = await req.json() - - for (const peer of j.peers) { + await loading_run(async () => { + const status = await get_status() + for (const peer of status.peers) { host_names[peer.id] = peer.hostname } - } finally { - loading_finish() - } + }) } diff --git a/svelte/src/admin_panel/PeerTable.svelte b/svelte/src/admin_panel/PeerTable.svelte index 454e2b2..b78c6d2 100644 --- a/svelte/src/admin_panel/PeerTable.svelte +++ b/svelte/src/admin_panel/PeerTable.svelte @@ -3,37 +3,17 @@ import { flip } from "svelte/animate"; import { formatDataVolume } from "util/Formatting"; import SortButton from "layout/SortButton.svelte"; import { loading_run } from "lib/Loading"; -import { admin_decommission_node } from "lib/AdminAPI"; +import { admin_decommission_node, type Peer } from "lib/AdminAPI"; -let { - peers = $bindable([]) -}: { - peers: { - id: string - ip: string - port: number - hostname: string - role: string - reachable: boolean - unreachable_count: number - latency: number - last_seen: string - free_space: number - min_free_space: number - load_1_min: number - load_5_min: number - load_15_min: number - avg_network_tx: number - avg_network_rx: number - port_speed: number - cache_threshold: number +// The computed columns are not part of the API response, they are filled in +// by the effect below +type TablePeer = Peer & { + avg_network_total?: number + usage_percent?: number + network_ratio?: number +} - // Our props - avg_network_total?: number - usage_percent?: number - network_ratio?: number - }[] -} = $props(); +let { peers = $bindable([]) }: { peers: TablePeer[] } = $props(); // If not all peers are online we show the decommission button let all_reachable = $derived(peers.reduce((acc, val) => { diff --git a/svelte/src/admin_panel/Router.svelte b/svelte/src/admin_panel/Router.svelte index 5f1940e..d91351c 100644 --- a/svelte/src/admin_panel/Router.svelte +++ b/svelte/src/admin_panel/Router.svelte @@ -11,6 +11,7 @@ import PayPalTaxes from "./PayPalTaxes.svelte"; import UserBans from "./user_bans/UserBans.svelte"; import InvoiceVat from "./InvoiceVAT.svelte"; import HostMetrics from "./HostMetrics.svelte"; +import FileInfo from "./FileInfo.svelte"; let pages = [ { @@ -53,6 +54,11 @@ let pages = [ title: "Host Metrics", icon: "area_chart", component: HostMetrics, + }, { + path: "/admin/file_info", + title: "File Inspector", + icon: "search", + component: FileInfo, }, { path: "/admin/mollie_settlements", title: "Prepaid accounting", diff --git a/svelte/src/lib/AdminAPI.ts b/svelte/src/lib/AdminAPI.ts index 59b2c3a..c30c5b0 100644 --- a/svelte/src/lib/AdminAPI.ts +++ b/svelte/src/lib/AdminAPI.ts @@ -39,3 +39,66 @@ export const admin_decommission_node = async (peer_id: string) => { ) ) as GenericResponse }; + +export type Peer = { + id: string + ip: string + port: number + hostname: string + role: string + reachable: boolean + unreachable_count: number + latency: number + last_seen: string + free_space: number + min_free_space: number + load_1_min: number + load_5_min: number + load_15_min: number + avg_network_tx: number + avg_network_rx: number + port_speed: number + cache_threshold: number +} + +export type QueryStats = { + query_name: string + calls: number + total_duration: number + average_duration: number + callers: { name: string, count: number }[] +} + +export type ServerStatus = { + running_since: string + pid: number + cpu_profile_running_since: string + db_latency: number + db_time: string + query_statistics: QueryStats[] + filesystem_watcher_threads: number + filesystem_watcher_listeners: number + peers: Peer[] + cache_threshold: number + local_reads: number + local_reads_per_sec: number + local_read_size: number + local_read_size_per_sec: number + local_readers: number + neighbour_reads: number + neighbour_reads_per_sec: number + neighbour_read_size: number + neighbour_read_size_per_sec: number + neighbour_readers: number + remote_reads: number + remote_reads_per_sec: number + remote_read_size: number + remote_read_size_per_sec: number + remote_readers: number +} + +export const get_status = async () => { + return await check_response( + await fetch(get_endpoint() + "/status") + ) as ServerStatus +}; diff --git a/svelte/src/util/Formatting.ts b/svelte/src/util/Formatting.ts index 4bfdeb1..0c45499 100644 --- a/svelte/src/util/Formatting.ts +++ b/svelte/src/util/Formatting.ts @@ -72,8 +72,13 @@ export const formatDuration = (ms: number, decimals: number) => { res += Math.floor((ms % hour) / minute) + "m " remainingDecimals-- } + if (ms >= second && remainingDecimals > 0) { + res += Math.floor((ms % minute) / second) + "s " + remainingDecimals-- + } if (remainingDecimals > 0) { - res += ((ms % minute) / second).toFixed(remainingDecimals) + "s" + res += (ms % second) + "ms " + remainingDecimals-- } return res } diff --git a/svelte/tsconfig.json b/svelte/tsconfig.json index 9f6686d..85bba05 100644 --- a/svelte/tsconfig.json +++ b/svelte/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": [ - "es2017", + "es2019", "dom" ], "paths": { @@ -10,7 +10,7 @@ ] }, "verbatimModuleSyntax": true, - "target": "ES2017", + "target": "ES2019", "noUnusedLocals": true, "module": "preserve", "moduleResolution": "bundler", diff --git a/webcontroller/style.go b/webcontroller/style.go index fc824eb..54dce64 100644 --- a/webcontroller/style.go +++ b/webcontroller/style.go @@ -548,6 +548,7 @@ footer, .footer_content, .upload_task, .add_button, .expandable, +.bubble, .upload_widget { border: none !important; border-radius: 0 !important; @@ -596,7 +597,6 @@ code { background-color: #fff !important; } -.file_preview, hr, .toolbar > .separator, .toolbar > .grid > .separator, @@ -676,6 +676,7 @@ footer, .footer_content, .upload_task, .add_button, .expandable, +.bubble, .upload_widget, .highlight_shaded { border: none !important; @@ -721,7 +722,6 @@ code { border-radius: 0 !important; } -.file_preview, hr, .toolbar > .separator, .toolbar > .grid > .separator {