Update uploading widget

svelte/src/filesystem/Filesystem.svelte

@@ -5,7 +5,6 @@ import Toolbar from "./Toolbar.svelte";
 import Breadcrumbs from "./Breadcrumbs.svelte";
 import DetailsWindow from "./DetailsWindow.svelte";
 import FilePreview from "./viewers/FilePreview.svelte";
-import FSUploadWidget from "./upload_widget/FSUploadWidget.svelte";
 import { global_navigator } from "./FSNavigator"
 import { css_from_path } from "filesystem/edit_window/Branding";
 import { update_favicon } from "util/NovaLogo.svelte";
@@ -15,7 +14,6 @@ import ErrorPage from "./ErrorPage.svelte";
 
 let file_preview: FilePreview = $state()
 let toolbar: Toolbar = $state()
-let upload_widget: FSUploadWidget = $state()
 let details_visible = $state(false)
 let edit_window: EditWindow = $state()
 let edit_visible = $state(false)
@@ -142,7 +140,6 @@ const keydown = (e: KeyboardEvent) => {
 		<FilePreview
 			bind:this={file_preview}
 			nav={nav}
-			upload_widget={upload_widget}
 			edit_window={edit_window}
 			details_window={details_window}
 		/>
@@ -165,10 +162,6 @@ const keydown = (e: KeyboardEvent) => {
 
 <EditWindow nav={nav} bind:this={edit_window} bind:visible={edit_visible} />
 
-<!-- This one is included at the highest level so uploads can keep running
-	even when the user navigates to a different directory -->
-<FSUploadWidget nav={nav} bind:this={upload_widget} />
-
 <AffiliatePrompt/>
 
 <style>

svelte/src/filesystem/filemanager/FileManager.svelte

@@ -14,7 +14,7 @@ import Button from "layout/Button.svelte";
 import { formatDate } from "util/Formatting";
 import { drop_target } from "lib/DropTarget"
 import { drag_click, move_target, type FSNavigator } from "filesystem/FSNavigator";
-import FsUploadWidget from "filesystem/upload_widget/FSUploadWidget.svelte";
+import { pick_files, upload_files } from "filesystem/upload_widget/FSUploadWidget.svelte";
 import EditWindow from "filesystem/edit_window/EditWindow.svelte";
 import { FileAction, type FileActionHandler } from "./FileManagerLib";
 import FileMenu from "./FileMenu.svelte";
@@ -25,20 +25,17 @@ import { toolbar_store } from "wrap/HeaderStore";
 
 let {
 	nav = $bindable(),
-	upload_widget,
 	edit_window = $bindable(),
 	directory_view = $bindable(localStorage.getItem("directory_view") || "list"),
 	children
 }: {
 	nav: FSNavigator;
-	upload_widget: FsUploadWidget;
 	edit_window: EditWindow;
 	directory_view?: string;
 	children?: import('svelte').Snippet;
 } = $props();
 
 let large_icons = $state(localStorage.getItem("large_icons") === "true")
-let uploader!: FsUploadWidget
 // In selection mode a click selects. Moving is the part of selection mode where
 // the user browses to the directory which the selection has to go to
 let moving = $state(false)
@@ -49,10 +46,6 @@ $effect(() => {
 	window.scrollTo(0, scroll_positions.get($nav.base.path) ?? 0)
 })
 
-export const upload = (files: File[]) => {
-	return uploader.upload_files(files)
-}
-
 // Navigation functions
 const file_event: FileActionHandler = (action: FileAction, index: number, orig: Event) => {
 	const node = nav.children[index]
@@ -244,7 +237,7 @@ onMount(() => {
 
 			<div class="toolbar_right">
 				{#if $nav.permissions.write}
-					<button onclick={() => upload_widget.pick_files()} title="Upload files to this directory">
+					<button onclick={() => pick_files()} title="Upload files to this directory">
 						<i class="icon">cloud_upload</i>
 					</button>
 
@@ -296,7 +289,7 @@ onMount(() => {
 	class="container"
 	use:move_target={{nav, node: () => nav.base}}
 	use:drop_target={{
-		upload: (files) => upload_widget.upload_files(files),
+		upload: files => upload_files(files),
 		shadow: "var(--accent_color) 0 0 10px 2px inset",
 	}}
 >

svelte/src/filesystem/upload_widget/FSUploadWidget.svelte

@@ -1,157 +1,363 @@
 <script lang="ts" module>
-export type UploadJob = {
+// A file, and its path relative to the directory which it is uploaded to
-	task_id: number,
+export type Upload = { file: File, path: string }
-	file: File,
+
-	path: string,
+// The widget is mounted once, in App.svelte, so that uploads keep running when
-	component: UploadProgress,
+// the user opens another page. It sets these functions when it mounts
-	status: string,
+export let upload_files: (files: Upload[] | Promise<Upload[]>) => void
-	total_size: number,
+export let pick_files: () => void
-	loaded_size: number,
-};
 </script>
+
 <script lang="ts">
-import { tick } from "svelte";
+import { global_navigator as nav } from "filesystem/FSNavigator";
-import UploadProgress from "./UploadProgress.svelte";
+import { fs_path_url } from "lib/FilesystemAPI.svelte";
-import type { FSNavigator } from "filesystem/FSNavigator";
+import { loading_run } from "lib/Loading";
+import { user } from "lib/UserStore";
+import { formatDataVolume, formatDuration } from "util/Formatting";
+import ProgressBar from "util/ProgressBar.svelte";
+import Button from "layout/Button.svelte";
 
-let { nav }: {
+type Job = Upload & {
-	nav: FSNavigator;
+	dir: string,
-} = $props();
+	loaded: number,
-
+	// What loaded was at the previous update, the difference gives the speed
-let file_input_field: HTMLInputElement = $state()
+	prev: number,
-let file_input_change = (e: Event) => {
+	speed: number,
-	// Start uploading the files async
+	xhr?: XMLHttpRequest,
-	upload_files((e.target as HTMLInputElement).files)
-
-	// This resets the file input field
-	file_input_field.nodeValue = ""
-}
-export const pick_files = () => {
-	file_input_field.click()
-}
-
-let visible = $state(false)
-let upload_queue: UploadJob[] = $state([]);
-let task_id_counter = 0
-
-export const upload_files = async (files: File[]|FileList) => {
-	if (files.length === 0) {
-		return
-	} else if (nav.base.type !== "dir") {
-		alert("Can only upload to directory")
-		return
-	}
-
-	// Add files to the queue
-	for (let i = 0; i < files.length; i++) {
-		await upload_file(files[i])
-	}
-}
-
-export const upload_file = async (file: File) => {
-	if (nav.base.type !== "dir") {
-		alert("Can only upload to directory")
-		return
-	} else if (file.type === "" && file.size === 0) {
-		return
-	}
-
-	let path = nav.base.path + "/"
-	if (file.webkitRelativePath) {
-		path += file.webkitRelativePath
-	} else {
-		path += file.name
-	}
-
-	upload_queue.push({
-		task_id: task_id_counter,
-		file: file,
-		path: path,
-		component: null,
-		status: "queued",
-		total_size: file.size,
-		loaded_size: 0,
-	})
-	task_id_counter++
-
-	// Reassign array and wait for tick to complete. After the tick is completed
-	// each upload progress bar will have bound itself to its array item
-	upload_queue = upload_queue
-
-	if (active_uploads === 0 && status !== "uploading") {
-		status = "uploading"
-		visible = true
-		await tick()
-		await start_upload()
-	}
 }
 
 const min_uploads = 2
 const max_uploads = 20
+// Milliseconds between two updates of the widget. The progress bars take as
+// long to move to their new position
+const update_interval = 250
 
-let active_uploads = 0
+// The uploads are deliberately not reactive. A dropped directory can hold
-let status = $state("idle")
+// hundreds of thousands of files, and every running upload reports progress
+// many times per second. Every update_interval, update() copies what the
+// widget shows to view, so the cost of rendering does not depend on the size of
+// the queue
+let queue: Job[] = []
+let active: Job[] = []
+let failed: { path: string, message: string }[] = []
+let files_total = 0
+let files_done = 0
+// In bytes. Files which failed or were cancelled are taken off the total
+let size_total = 0
+let size_done = 0
+// Every byte which was sent, also for uploads which were aborted later
+let transferred = 0
+let transferred_prev = 0
+let speed = 0
+let updated = 0
+let timer = 0
+// What was transferred at each update of the last ten seconds, and how many
+// bytes the running uploads may have left to send together, see schedule()
+let samples: { time: number, sent: number }[] = []
+let threshold = 0
+// The title which the page set, and the one we replaced it with
+let page_title = ""
+let title = ""
 
-const start_upload = async () => {
+let visible = $state(false)
-	active_uploads = 0
+let collapsed = $state(false)
-	let uploading_size = 0
+let paused = $state(false)
-	for (let i = 0; i < upload_queue.length; i++) {
+// The timer which waits to try again after the connection was lost
-		if (upload_queue[i]) {
+let retry = $state(0)
-			// If this file is queued, start the upload
+let view = $state.raw({
-			if (upload_queue[i].status === "queued") {
+	uploading: false,
-				upload_queue[i].component.start()
+	running: 0,
-				upload_queue[i].status = "uploading"
+	percent: 0,
+	loaded: 0,
+	total: 0,
+	speed: 0,
+	eta: "",
+	files_done: 0,
+	files_total: 0,
+	rows: [] as { job: Job, loaded: number, speed: number, queued: boolean }[],
+	failed,
+	failed_count: 0,
+})
+
+let file_input: HTMLInputElement
+pick_files = () => file_input.click()
+const file_input_change = () => {
+	upload_files(Array.from(file_input.files, file => ({ file, path: file.name })))
+
+	// Without this the same file can't be picked twice in a row
+	file_input.value = ""
 }
 
-			// If this file is already uploading (or just started), count it
+upload_files = async files => {
-			if (upload_queue[i].status === "uploading") {
+	// Reading a large directory takes a while. The files go to the directory
-				uploading_size += upload_queue[i].total_size
+	// which they were dropped on, also if the user has left it by then
-				active_uploads++
+	const dir = nav.base.path
+	try {
+		files = await loading_run(async () => files)
+	} catch (err) {
+		alert("Failed to read the files: " + err)
+		return
+	}
+	if (files.length === 0) {
+		return
 	}
 
-			// If the size threshold or the concurrent upload limit is reached
+	if (active.length + queue.length === 0) {
-			// we break the loop. The system tries to keep an upload queue of
+		files_total = files_done = size_total = size_done = speed = 0
-			// 100 MB and a minimum of two concurrent uploads.
+		timer = window.setInterval(() => {
-			if (
+			update()
-				(uploading_size >= 100e6 && active_uploads >= min_uploads) ||
+			schedule()
-				active_uploads >= max_uploads
+		}, update_interval)
+	}
+
+	const limit = $user?.subscription.file_size_limit || 20e9
+	for (const { file, path } of files) {
+		if (file.size > limit) {
+			failed.push({ path, message: "This file is too large. Check out " +
+				"the Pro subscription to increase the file size limit" })
+			continue
+		}
+		queue.push({ file, path, dir, loaded: 0, prev: 0, speed: 0 })
+		files_total++
+		size_total += file.size
+	}
+
+	visible = true
+	schedule()
+	update()
+}
+
+// Starts queued uploads until enough are running: at least two, and as many
+// as it takes to have ten seconds of work at the speed of the last ten seconds.
+// Small files are sent in large numbers at a time that way, and a slow
+// connection is not divided over more large files than it can handle. A new
+// connection starts with two uploads, and gets more as its speed shows
+const schedule = () => {
+	let size = 0
+	for (const job of active) {
+		size += job.file.size - job.loaded
+	}
+
+	// Empty files don't count towards the speed, a directory full of them
+	// never gets over a threshold of zero. Hence the <=
+	while (
+		!paused && retry === 0 &&
+		queue.length > 0 && active.length < max_uploads &&
+		(size <= threshold || active.length < min_uploads)
 	) {
-				console.debug("Current uploads", active_uploads, "uploads size", uploading_size)
+		const job = queue.shift()
-				break
+		size += job.file.size
+		active.push(job)
+
+		// Takes the job off the running uploads. Without an error it's done
+		const end = (error?: string) => {
+			job.xhr = undefined
+			active.splice(active.indexOf(job), 1)
+			if (error === undefined) {
+				transferred += job.file.size - job.loaded
+				files_done++
+				size_done += job.file.size
+			} else {
+				failed.push({ path: job.path, message: error })
+				files_total--
+				size_total -= job.file.size
 			}
+			schedule()
+		}
+
+		const xhr = job.xhr = new XMLHttpRequest()
+		xhr.open("PUT", fs_path_url(job.dir + "/" + job.path) + "?make_parents=true")
+		xhr.upload.onprogress = e => {
+			transferred += e.loaded - job.loaded
+			job.loaded = e.loaded
+		}
+		xhr.onloadend = () => {
+			if (job.xhr !== xhr) {
+				return // Aborted by stop or cancel
+			} else if (xhr.status >= 400) {
+				// A proxy in front of the API does not answer with JSON
+				let error = xhr.statusText
+				try {
+					error = JSON.parse(xhr.response).message
+				} catch {}
+				end(error)
+			} else if (xhr.status !== 0) {
+				end()
+			} else {
+				// No response, the connection is gone. The file goes back to
+				// the front of the queue, and the queue waits. Without the
+				// wait every file in it would fail within seconds
+				job.xhr = undefined
+				active.splice(active.indexOf(job), 1)
+				job.loaded = job.prev = job.speed = 0
+				queue.unshift(job)
+				clearTimeout(retry)
+				retry = window.setTimeout(() => {
+					retry = 0
+					schedule()
+				}, 10e3)
 			}
 		}
 
-	if (active_uploads === 0) {
+		// Firefox never ends the request for a file which was removed or
-		status = "finished"
+		// truncated after it was picked: there is no error, it hangs. Reading
+		// the last byte of the file first shows if all of it is still there.
+		// Chromium refuses to read a file which changed in any way, with a
+		// NotReadableError, but does upload it. That one is not a reason to
+		// give up
+		job.file.slice(-1).arrayBuffer().then(
+			last => last.byteLength === Math.min(1, job.file.size),
+			err => err.name === "NotReadableError",
+		).then(readable => {
+			if (job.xhr !== xhr) {
+				return
+			} else if (readable) {
+				xhr.send(job.file)
+			} else {
+				end("The file was changed or removed after it was added")
+			}
+		})
+	}
+
+	if (timer !== 0 && active.length + queue.length === 0) {
+		clearInterval(timer)
+		timer = 0
+		paused = false
+		update()
+
+		// Show the new files. The other pages don't use the navigator, there a
+		// reload would put the directory in the address bar
+		if (window.location.pathname.startsWith("/d/")) {
 			nav.reload()
+		}
 
-		// Empty the queue to free any references to lingering components
+		// Errors stay until the user has closed them
-		upload_queue = []
-
-		// In ten seconds we close the popup
 		setTimeout(() => {
-			if (status === "finished") {
+			if (timer === 0 && failed.length === 0) {
 				visible = false
 			}
-		}, 10000)
+		}, 10e3)
-	} else {
-		status = "uploading"
 	}
 }
 
-const finish_upload = () => {
+// The request is taken off the job first, that is how its loadend handler knows
-	// Update the queue so the status updates are properly rendered
+// that it was aborted on purpose
-	upload_queue = upload_queue
+const abort = (job: Job) => {
-	start_upload()
+	const xhr = job.xhr
+	job.xhr = undefined
+	xhr.abort()
+}
+
+// A pause only holds the queue, the uploads which are running finish
+const toggle_pause = () => {
+	paused = !paused
+	schedule()
+	update()
+}
+
+const stop = () => {
+	clearTimeout(retry)
+	retry = 0
+	active.forEach(abort)
+	active = []
+	queue = []
+	failed = []
+	visible = false
+	schedule()
+}
+
+const cancel = (job: Job) => {
+	const list = job.xhr === undefined ? queue : active
+	const index = list.indexOf(job)
+	if (index === -1) {
+		return // The row is older than the job, it has finished since
+	}
+	if (list === active) {
+		abort(job)
+	}
+	list.splice(index, 1)
+	files_total--
+	size_total -= job.file.size
+	schedule()
+	update()
+}
+
+const update = () => {
+	const now = performance.now()
+	const dt = (now - updated) / 1000
+	updated = now
+
+	// Speeds are moving averages over about the last three seconds. A sample
+	// weighs as much as the time which it covers, because the timer is not
+	// steady: browsers slow it down in a background tab, to once per minute even
+	const weight = 1 - Math.exp(-dt / 3)
+	const average = (speed: number, bytes: number) =>
+		dt > 0 ? speed + (bytes / dt - speed) * weight : speed
+
+	speed = average(speed, transferred - transferred_prev)
+	transferred_prev = transferred
+
+	// Ten seconds at the average speed of the last ten seconds is what was
+	// sent in that time. One sample from before it stays as the start of the
+	// period: in a background tab the timer runs as little as once per minute,
+	// and the period would be empty. It is longer than ten seconds then, so
+	// the amount is scaled back to ten
+	samples.push({ time: now, sent: transferred })
+	while (samples.length > 2 && now - samples[1].time >= 10e3) {
+		samples.shift()
+	}
+	threshold = (transferred - samples[0].sent) * 10e3 /
+		Math.max(10e3, now - samples[0].time)
+
+	let loaded = size_done
+	const rows = active.map(job => {
+		job.speed = average(job.speed, job.loaded - job.prev)
+		job.prev = job.loaded
+		loaded += job.loaded
+		return { job, loaded: job.loaded, speed: job.speed, queued: false }
+	})
+	// The start of the queue is shown too. With only the running uploads the
+	// list changes height all the time, and the widget jumps with it
+	for (const job of queue.slice(0, 50)) {
+		rows.push({ job, loaded: 0, speed: 0, queued: true })
+	}
+
+	const uploading = active.length + queue.length > 0
+	const percent = Math.floor(loaded / size_total * 100) || 0
+	// There is no sensible estimate while nothing is being sent, nor when it
+	// is over a hundred days, which is what a stalled connection gives.
+	// ponytail: estimated from bytes alone, which is too low when small files
+	// with their overhead per request are mixed with large ones. Weigh in the
+	// files per second if that turns out to matter
+	const left = (Math.floor((size_total - loaded) / speed) + 1) * 1000
+	const eta = !paused && active.length > 0 && left < 100 * 86400e3
+		? formatDuration(left, left < 60e3 ? 1 : 2)
+		: ""
+
+	view = {
+		uploading, percent, loaded, speed, eta, rows, files_done, files_total,
+		running: active.length,
+		total: size_total,
+		failed: failed.slice(0, 100),
+		failed_count: failed.length,
+	}
+
+	// Pages set the title when the user navigates. A title which is not the one
+	// we set is a new one, the progress goes in front of it
+	if (document.title !== title) {
+		page_title = document.title
+	}
+	document.title = !uploading ? page_title
+		: (paused ? `Paused ${percent}% ` : `${percent}% ${eta}`) + "/ " + page_title
+
+	// The browser collapses the whitespace in a title, so what we get back
+	// is not always what we set
+	title = document.title
 }
 
 const leave_confirmation = (e: BeforeUnloadEvent) => {
-	if (status === "uploading") {
+	if (active.length + queue.length > 0) {
 		e.preventDefault()
-		return "If you close this page your files will stop uploading. Do you want to continue?"
-	} else {
-		return null
 	}
 }
 </script>
@@ -159,33 +365,94 @@ const leave_confirmation = (e: BeforeUnloadEvent) => {
 <svelte:window onbeforeunload={leave_confirmation} />
 
 <input
-	bind:this={file_input_field}
+	bind:this={file_input}
 	onchange={file_input_change}
 	class="upload_input" type="file" name="file" multiple
 />
 
 {#if visible}
-	<div class="upload_widget">
+	<div class="upload_widget bubble">
 		<div class="header">
-			{#if status === "idle"}
+			<div class="grow">
-				Waiting for files
+				<div class="ellipsis">
-			{:else if status === "uploading"}
+					{#if !view.uploading}
-				Uploading files...
+						Done{#if view.failed_count > 0}, {view.failed_count} failed{/if}
-			{:else if status === "finished"}
+					{:else}
-				Done
+						{retry !== 0 ? "Connection lost" : paused ? "Paused" : "Uploading"}
+						{view.percent}%{#if view.running > 0}, {formatDataVolume(view.speed, 3)}/s{/if}{#if view.eta !== ""}, {view.eta}left{/if}
+					{/if}
+				</div>
+				<ProgressBar total={view.total} used={view.loaded} speed={update_interval} animation="linear"/>
+			</div>
+			{#if view.uploading}
+				<Button
+					flat
+					icon={paused ? "play_arrow" : "pause"}
+					title={paused
+						? "Resume: start uploading the files in the queue again"
+						: "Pause: finish the uploads which are running, but don't start the files in the queue"}
+					click={toggle_pause}
+				/>
+				<Button
+					flat icon="stop"
+					title="Stop: abort the uploads which are running and empty the queue"
+					click={stop}
+				/>
+			{:else}
+				<Button flat icon="close" title="Close" click={stop}/>
+			{/if}
+			<Button
+				flat
+				icon={collapsed ? "expand_less" : "expand_more"}
+				title={collapsed ? "Expand" : "Collapse"}
+				click={() => collapsed = !collapsed}
+			/>
+		</div>
+
+		{#if !collapsed}
+			<div class="stats">
+				{view.files_done} / {view.files_total} files,
+				{formatDataVolume(view.loaded, 3)} /
+				{formatDataVolume(view.total, 3)}
+				{#if retry !== 0}
+					<br/>
+					Trying again every 10 seconds
 				{/if}
 			</div>
 			<div class="body">
-			{#each upload_queue as job, i}
+				{#each view.rows as row (row.job)}
-				{#if job.status !== "finished"}
+					<div class="row" class:queued={row.queued}>
-					<UploadProgress
+						<div class="grow">
-						bind:this={upload_queue[i].component}
+							<div class="ellipsis path" title={row.job.path}>
-						bind:job={upload_queue[i]}
+								<span dir="ltr">{row.job.path}</span>
-						finish={finish_upload}
-					/>
-				{/if}
-			{/each}
 							</div>
+							{#if row.queued}
+								{formatDataVolume(row.job.file.size, 3)}
+							{:else}
+								{formatDataVolume(row.loaded, 3)} /
+								{formatDataVolume(row.job.file.size, 3)},
+								{formatDataVolume(row.speed, 3)}/s
+								<ProgressBar total={row.job.file.size} used={row.loaded} speed={update_interval} animation="linear"/>
+							{/if}
+						</div>
+						<Button icon="cancel" title="Cancel" click={() => cancel(row.job)}/>
+					</div>
+				{/each}
+				{#each view.failed as file}
+					<div class="error">
+						<div class="ellipsis path" title={file.path}>
+							<span dir="ltr">{file.path}</span>
+						</div>
+						{file.message}
+					</div>
+				{/each}
+				{#if view.failed_count > view.failed.length}
+					<div class="error">
+						And {view.failed_count - view.failed.length} more
+					</div>
+				{/if}
+			</div>
+		{/if}
 	</div>
 {/if}
 
@@ -200,34 +467,59 @@ const leave_confirmation = (e: BeforeUnloadEvent) => {
 	position: fixed;
 	display: flex;
 	flex-direction: column;
-	width: auto;
+	width: 600px;
-	min-width: 400px;
+	max-width: calc(100% - 40px);
-	max-width: 80%;
-	height: auto;
 	max-height: 50%;
 	right: 20px;
 	bottom: 20px;
-	border-radius: 8px;
 	overflow: hidden;
+	color: var(--body_text_color);
 	box-shadow: 1px 1px 8px var(--shadow_color);
 }
-
 .header {
 	flex: 0 0 auto;
-	background: var(--background_color);
+	display: flex;
-	color: var(--background_text_color);
+	align-items: center;
-	text-align: center;
+	padding: 0 0 0 8px;
+	border-bottom: 1px solid var(--separator);
+}
+.header .ellipsis {
 	font-size: 1.2em;
-	padding: 4px;
+}
-	white-space: nowrap;
+.stats {
-	text-overflow: ellipsis;
+	flex: 0 0 auto;
-	overflow: hidden;
+	padding: 4px 8px;
+	text-align: center;
 }
 .body {
 	flex: 1 1 auto;
-	background: var(--body_color);
-	color: var(--body_text_color);
 	overflow-y: auto;
+}
+.row {
+	display: flex;
+	align-items: center;
+	padding: 2px 8px;
+}
+.grow {
+	flex: 1 1 auto;
+	min-width: 0;
+}
+.ellipsis {
+	overflow: hidden;
+	white-space: nowrap;
+	text-overflow: ellipsis;
+}
+/* A path which does not fit loses its start, the file name is the part which
+   matters. Text is cut off where it overflows, which is on the left in a box
+   which runs from right to left. The path inside it still runs left to right */
+.path {
+	direction: rtl;
 	text-align: left;
 }
+.error {
+	padding: 2px 8px;
+	background: var(--red_tint);
+	color: var(--red_color);
+	overflow-wrap: break-word;
+}
 </style>

svelte/src/filesystem/upload_widget/UploadFunc.ts

@@ -1,83 +0,0 @@
-// Uploads a file to the logged in user's Nova account. If no user is
-// logged in the file is uploaded anonymously.
-//
-// on_progress reports progress on the file upload, parameter 1 is the uploaded
-// file size and parameter 2 is the total file size
-//
-// on_success is called when the upload is done, the only parameter is the file
-// ID
-//
-// on_error is called when the upload has failed. The parameters are the error
-
-import { fs_path_url, type GenericResponse } from "lib/FilesystemAPI.svelte"
-import { get_user } from "lib/NovaAPI"
-
-// code and an error message
-export const upload_file = async (
-	file: Blob,
-	path: string,
-	on_progress: (loaded: number, total: number) => void,
-	on_success: (id: string) => void,
-	on_error: (code: string, message: string) => void,
-) => {
-	const user = await get_user()
-	// Check the file size limit. For free accounts it's 20 GB
-	if (user !== null && user.subscription.file_size_limit === 0) {
-		user.subscription.file_size_limit = 20e9
-	}
-
-	if (file.size > user.subscription.file_size_limit) {
-		on_error(
-			"file_too_large",
-			"This file is too large. Check out the Pro subscription to increase the file size limit"
-		)
-		return
-	}
-
-	console.debug("Uploading file to", fs_path_url(path))
-
-	let xhr = new XMLHttpRequest();
-	xhr.open("PUT", fs_path_url(path) + "?make_parents=true", true);
-	xhr.timeout = 86400000; // 24 hours, to account for slow connections
-
-	xhr.upload.addEventListener("progress", evt => {
-		if (on_progress && evt.lengthComputable) {
-			on_progress(evt.loaded, evt.total)
-		}
-	});
-
-	xhr.onreadystatechange = () => {
-		// readystate 4 means the upload is done
-		if (xhr.readyState !== 4) {
-			return
-		}
-
-		if (xhr.status >= 100 && xhr.status < 400) {
-			// Request is a success
-			on_success(JSON.parse(xhr.response).id)
-		} else if (xhr.status >= 400) {
-			// Request failed
-			console.log("Upload error. status: " + xhr.status + " response: " + xhr.response);
-
-			let resp: GenericResponse
-			if (xhr.status === 429) {
-				resp = {
-					value: "too_many_requests",
-					message: "Too many requests. Please wait a few seconds",
-				}
-			} else {
-				resp = JSON.parse(xhr.response) as GenericResponse
-			}
-
-			on_error(resp.value, resp.message)
-		} else if (xhr.status === 0) {
-			on_error("request_failed", "Your request did not arrive, check your network connection")
-		} else {
-			on_error(xhr.responseText, xhr.responseText)
-		}
-	};
-
-	xhr.send(file);
-
-	return xhr
-}

svelte/src/filesystem/upload_widget/UploadProgress.svelte

@@ -1,105 +0,0 @@
-<script lang="ts">
-import { upload_file } from "./UploadFunc";
-import ProgressBar from "util/ProgressBar.svelte";
-import Button from "layout/Button.svelte"
-import type { UploadJob } from "./FSUploadWidget.svelte";
-
-let {
-	job = $bindable(),
-	total = $bindable(0),
-	loaded = $bindable(0),
-	finish,
-}: {
-	job: UploadJob
-	total?: number
-	loaded?: number
-	finish: () => void
-} = $props();
-
-let error_code = $state("")
-let error_message = $state("")
-let xhr: XMLHttpRequest = null
-
-export const start = () => {
-	upload_file(
-		job.file,
-		job.path,
-		(prog_loaded, prog_total) => {
-			loaded = prog_loaded
-			total = prog_total
-		},
-		async () => {
-			job.status = "finished"
-			finish()
-		},
-		(code, message) => {
-			if (job.status === "finished") {
-				return
-			}
-
-			console.log("Upload error", code, message)
-			error_code = code
-			error_message = message
-			job.status = "error"
-
-			// We don't send the finished signal so the user can read the error
-			// messsage and manually dismiss it through the cancel button
-		},
-	).then(result => {
-		xhr = result
-		job.status = "uploading"
-	})
-}
-
-const cancel = () => {
-	job.status = "finished"
-
-	if (xhr && xhr.abort) {
-		xhr.abort()
-	}
-
-	xhr = null
-	finish()
-}
-</script>
-
-<div class="prog" class:error={job.status === "error"}>
-	<div class="bar">
-		{job.file.name}<br/>
-		{#if error_code !== ""}
-			{error_message}<br/>
-			{error_code}<br/>
-		{/if}
-		<ProgressBar total={total} used={loaded} speed={500}/>
-	</div>
-	<div class="cancel">
-		<Button icon="cancel" click={cancel}/>
-	</div>
-</div>
-
-<style>
-.prog {
-	display: flex;
-	flex-direction: row;
-	width: 100%;
-	overflow: hidden;
-}
-.bar {
-	flex: 1 1 auto;
-	padding: 2px 4px 1px 4px;
-	margin: 4px;
-	border-radius: 4px;
-	overflow: hidden;
-	line-break: anywhere;
-}
-.cancel {
-	flex: 0 0 content;
-	display: flex;
-	justify-content: center;
-	align-items: center; /* Stop stretching the button */
-}
-.error {
-	background: var(--red_tint);
-	color: var(--red_color);
-}
-</style>

svelte/src/filesystem/viewers/FilePreview.svelte

@@ -14,18 +14,15 @@ import Torrent from "./Torrent.svelte";
 import Zip from "./Zip.svelte";
 import CustomBanner from "./CustomBanner.svelte";
 import type { FSNavigator } from "filesystem/FSNavigator";
-import FsUploadWidget from "filesystem/upload_widget/FSUploadWidget.svelte";
 import EditWindow from "filesystem/edit_window/EditWindow.svelte";
 import DetailsWindow from "filesystem/DetailsWindow.svelte";
 
 let {
 	nav,
-	upload_widget,
 	edit_window,
 	details_window,
 }: {
 	nav: FSNavigator
-	upload_widget: FsUploadWidget
 	edit_window: EditWindow
 	details_window: DetailsWindow
 } = $props();
@@ -89,7 +86,7 @@ export const seek = (delta: number) => {
 {:else if viewer_type === "dir" && $nav.base.name === ".Trash"}
 	<TrashBin nav={nav} root_path={$nav.path[$nav.path.length - 2].path}/>
 {:else if viewer_type === "dir"}
-	<FileManager nav={nav} upload_widget={upload_widget} edit_window={edit_window}>
+	<FileManager nav={nav} edit_window={edit_window}>
 		<CustomBanner path={$nav.path}/>
 	</FileManager>
 {:else if viewer_type === "audio"}

svelte/src/home_page/HomePage.svelte

@@ -1,12 +1,9 @@
 <script>
 import Footer from "layout/Footer.svelte";
-import { drop_target } from "lib/DropTarget";
 import GetStarted from "./GetStarted.svelte";
 import Pricing from "./Pricing.svelte";
 import SubscriptionPlans from "layout/checkout/SubscriptionPlans.svelte";
 import NovaLogo from "util/NovaLogo.svelte";
-
-let upload_widget
 </script>
 
 <header class="logo_header">

svelte/src/lib/DropTarget.ts

@@ -1,7 +1,11 @@
+import type { Upload } from "filesystem/upload_widget/FSUploadWidget.svelte"
+
 export const drop_target = (
 	node: HTMLElement,
 	args: {
-		upload: (files: File[]) => void,
+		// Called as soon as the files are dropped, the promise resolves when
+		// all the dropped directories have been read
+		upload: (files: Promise<Upload[]>) => void,
 		shadow: string | undefined,
 	},
 ) => {
@@ -39,7 +43,7 @@ export const drop_target = (
 	const dragleave = (e: DragEvent) => {
 		node.style.boxShadow = ""
 	}
-	const drop = async (e: DragEvent) => {
+	const drop = (e: DragEvent) => {
 		node.style.boxShadow = ""
 
 		if (can_upload(e)) {
@@ -49,39 +53,59 @@ export const drop_target = (
 			return
 		}
 
-		// if directory support is available
+		// The items are only valid until this function returns, so every entry
-		if (e.dataTransfer && e.dataTransfer.items && e.dataTransfer.items.length > 0) {
+		// has to be taken out of them before the first await
+		const files: Upload[] = []
+		const reads: Promise<void>[] = []
 		for (let i = 0; i < e.dataTransfer.items.length; i++) {
-				const entry: FileSystemEntry | null = e.dataTransfer.items[i].webkitGetAsEntry();
+			const entry = e.dataTransfer.items[i].webkitGetAsEntry()
 			if (entry !== null) {
-					read_dir_recursive(entry);
+				reads.push(read_entry(entry, files))
 				continue
 			}
-				const file: File | null = e.dataTransfer.items[i].getAsFile();
+
+			// Files which are not on the disk have no entry, like an image
+			// which is dragged out of another web page
+			const file = e.dataTransfer.items[i].getAsFile()
 			if (file !== null) {
-					args.upload([file]);
+				files.push({ file, path: file.name })
-					continue
 			}
 		}
-		} else if (e.dataTransfer && e.dataTransfer.files && e.dataTransfer.files.length > 0) {
+		args.upload(Promise.all(reads).then(() => files))
-			const files: File[] = []
-			for (let i = 0; i < e.dataTransfer.files.length; i++) {
-				files.push(e.dataTransfer.files[i])
 	}
-			args.upload(files)
+
+	// Adds a file, or all the files in a directory, to the list. The path comes
+	// from the entry. Firefox also puts it in the file's webkitRelativePath,
+	// but Chromium leaves that empty for dropped files
+	const read_entry = async (entry: FileSystemEntry, files: Upload[]) => {
+		if (entry.isFile) {
+			files.push({
+				path: entry.fullPath.slice(1),
+				file: await new Promise<File>((resolve, reject) =>
+					(entry as FileSystemFileEntry).file(resolve, reject)
+				),
+			})
+			return
 		}
+
+		// A reader returns the directory in parts, Chromium gives 100 entries
+		// at a time. An empty part marks the end
+		const reader = (entry as FileSystemDirectoryEntry).createReader()
+		while (true) {
+			const entries = await new Promise<FileSystemEntry[]>(
+				(resolve, reject) => reader.readEntries(resolve, reject)
+			)
+			if (entries.length === 0) {
+				return
+			}
+
+			// One at a time on purpose. Chromium asks another process for
+			// each file, and handles all the answers which have arrived in
+			// one go. Asking for a whole tree at once is a third faster, but
+			// freezes the page for seconds: 7 for 100k files
+			for (const e of entries) {
+				await read_entry(e, files)
 			}
-	const read_dir_recursive = (item: FileSystemEntry) => {
-		if (item.isDirectory) {
-			(item as FileSystemDirectoryEntry).createReader().readEntries(entries => {
-				entries.forEach(entry => {
-					read_dir_recursive(entry);
-				});
-			});
-		} else {
-			(item as FileSystemFileEntry).file(file => {
-				args.upload([file])
-			});
 		}
 	}
 

svelte/src/wrap/App.svelte

@@ -17,6 +17,7 @@ import { breadcrumbs_store, toolbar_store } from "./HeaderStore";
 import ProgressBar from "util/ProgressBar.svelte";
 import NovaLogo, { update_favicon } from "util/NovaLogo.svelte";
 import SearchBar from "filesystem/SearchBar.svelte";
+import FSUploadWidget from "filesystem/upload_widget/FSUploadWidget.svelte";
 
 // The menu swipe will be detected if it was less than this much pixels from the
 // screen edge
@@ -320,6 +321,7 @@ const set_offset = (off: number) => {
 </div>
 
 <SearchBar/>
+<FSUploadWidget/>
 
 <style>
 .wrap {