Add remote sync feature to run rclone on the server side

This commit is contained in:
Pixeldrain-Server committed 2026-11-12 15:50:06 +01:00
1 parent b423446412
commit ee06b91b92
6 files changed
+68 -68

No files matched your search

File renamed without changes.

Before

Width:  |  Height:  |  Size: 65 KiB

After

Width:  |  Height:  |  Size: 65 KiB

+3 -3
View File
@@ -26,10 +26,10 @@ const slides = [
text: `Follow the downloads and data transfer of every file, from the text: `Follow the downloads and data transfer of every file, from the
last day up to five years back.`, last day up to five years back.`,
}, { }, {
image: "transfers", image: "remote_sync",
title: "Transfers", title: "Remote sync",
text: `Import from or export to S3, Backblaze B2, Azure, SFTP, FTP, text: `Import from or export to S3, Backblaze B2, Azure, SFTP, FTP,
WebDAV or pixeldrain. Transfers run on our servers, so you can close the WebDAV or pixeldrain. Syncs run on our servers, so you can close the
page while they run.`, page while they run.`,
}, },
] ]
+4 -4
View File
@@ -12,7 +12,7 @@ import { get_user, type User } from "lib/NovaAPI";
import SharingSettings from "./SharingSettings.svelte"; import SharingSettings from "./SharingSettings.svelte";
import SubscriptionManage from "./SubscriptionManage.svelte"; import SubscriptionManage from "./SubscriptionManage.svelte";
import SubscriptionUsage from "./SubscriptionUsage.svelte"; import SubscriptionUsage from "./SubscriptionUsage.svelte";
import Transfers from "./transfers/Transfers.svelte"; import RemoteSync from "./remote_sync/RemoteSync.svelte";
let pages: Tab[] = [ let pages: Tab[] = [
{ {
@@ -54,10 +54,10 @@ let pages: Tab[] = [
icon: "share", icon: "share",
component: SharingSettings, component: SharingSettings,
}, { }, {
path: "/user/transfers", path: "/user/remote_sync",
title: "Transfers", title: "Remote sync",
icon: "sync_alt", icon: "sync_alt",
component: Transfers, component: RemoteSync,
}, { }, {
path: "/user/connect_app", path: "/user/connect_app",
title: "Apps", title: "Apps",
@@ -5,24 +5,24 @@ import { check_response, dict_to_form, get_endpoint } from "lib/NovaAPI";
import ProgressBar from "util/ProgressBar.svelte"; import ProgressBar from "util/ProgressBar.svelte";
import { formatDataVolume, formatDate, formatDuration } from "util/Formatting"; import { formatDataVolume, formatDate, formatDuration } from "util/Formatting";
import { user } from "lib/UserStore"; import { user } from "lib/UserStore";
import TransferForm from "./TransferForm.svelte"; import RemoteSyncForm from "./RemoteSyncForm.svelte";
import { edit_transfer, form, secret_fields, service_of, type Transfer } from "./TransferLib.svelte"; import { edit_remote_sync, form, secret_fields, service_of, type RemoteSync } from "./RemoteSyncLib.svelte";
let transfers: Transfer[] = $state([]) let syncs: RemoteSync[] = $state([])
// Polls every second while a transfer runs, the server saves the // Polls every second while a sync runs, the server saves the
// progress that often // progress that often
let mounted = true let mounted = true
let timeout: ReturnType<typeof setTimeout> let timeout: ReturnType<typeof setTimeout>
const load = async () => { const load = async () => {
clearTimeout(timeout) clearTimeout(timeout)
try { try {
transfers = await check_response(await fetch(get_endpoint() + "/user/transfer")) syncs = await check_response(await fetch(get_endpoint() + "/user/remote_sync"))
} catch (err) { } catch (err) {
console.error("Failed to load transfers:", err) console.error("Failed to load remote syncs:", err)
} }
if (mounted && transfers.some(t => t.status === "running")) { if (mounted && syncs.some(t => t.status === "running")) {
timeout = setTimeout(load, 1000) timeout = setTimeout(load, 1000)
} }
} }
@@ -35,33 +35,33 @@ onMount(() => {
} }
}) })
const edit = (t: Transfer) => { const edit = (t: RemoteSync) => {
edit_transfer(t) edit_remote_sync(t)
document.getElementById("transfer_form")?.scrollIntoView({ behavior: "smooth" }) document.getElementById("remote_sync_form")?.scrollIntoView({ behavior: "smooth" })
} }
const restart = async (t: Transfer, body = {}) => { const restart = async (t: RemoteSync, body = {}) => {
try { try {
await loading_run(async () => check_response( await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer/" + t.id + "/restart", { await fetch(get_endpoint() + "/user/remote_sync/" + t.id + "/restart", {
method: "POST", method: "POST",
body: dict_to_form(body), body: dict_to_form(body),
}) })
)) ))
} catch (err) { } catch (err) {
alert("Failed to restart the transfer: " + (err.message ?? err)) alert("Failed to restart the remote sync: " + (err.message ?? err))
return return
} }
asking = "" asking = ""
load() load()
} }
// A transfer without remembered credentials asks for them again. What the form // A sync without remembered credentials asks for them again. What the form
// still has for that service is filled in // still has for that service is filled in
let asking = $state("") let asking = $state("")
let secrets: { [name: string]: string } = $state({}) let secrets: { [name: string]: string } = $state({})
let remember_again = $state(false) let remember_again = $state(false)
const ask = (t: Transfer) => { const ask = (t: RemoteSync) => {
const service = service_of(t) const service = service_of(t)
secrets = Object.fromEntries(secret_fields(service).map(f => secrets = Object.fromEntries(secret_fields(service).map(f =>
[f.name, (form.options[service][f.name] as string) ?? ""] [f.name, (form.options[service][f.name] as string) ?? ""]
@@ -70,13 +70,13 @@ const ask = (t: Transfer) => {
asking = t.id asking = t.id
} }
const remove = async (t: Transfer) => { const remove = async (t: RemoteSync) => {
try { try {
await loading_run(async () => check_response( await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer/" + t.id, { method: "DELETE" }) await fetch(get_endpoint() + "/user/remote_sync/" + t.id, { method: "DELETE" })
)) ))
} catch (err) { } catch (err) {
alert("Failed to remove the transfer: " + (err.message ?? err)) alert("Failed to remove the remote sync: " + (err.message ?? err))
return return
} }
load() load()
@@ -84,29 +84,29 @@ const remove = async (t: Transfer) => {
</script> </script>
<section> <section>
<h2>Transfers</h2> <h2>Remote sync</h2>
<p> <p>
Copy files between Nova and another storage service. The transfer runs Copy files between Nova and another storage service. The sync runs
on our servers, so you can close this page while it runs. A transfer on our servers, so you can close this page while it runs. A sync
which stopped can be started again, files which are already there are which stopped can be started again, files which are already there are
skipped. skipped.
</p> </p>
{#if $user !== null && $user.subscription.id === ""} {#if $user !== null && $user.subscription.id === ""}
<p> <p>
Transfers are available on paid plans. Remote sync is available on paid plans.
<a class="button button_highlight" href="/user/subscription/manage"> <a class="button button_highlight" href="/user/subscription/manage">
<i class="icon i_shopping_cart"></i> Upgrade <i class="icon i_shopping_cart"></i> Upgrade
</a> </a>
</p> </p>
{:else} {:else}
<TransferForm onsaved={load}/> <RemoteSyncForm onsaved={load}/>
{/if} {/if}
{#each transfers as t (t.id)} {#each syncs as t (t.id)}
{@const name = t.directory_path || t.directory} {@const name = t.directory_path || t.directory}
{@const service = service_of(t)} {@const service = service_of(t)}
<div class="transfer" class:running={t.status === "running"} class:editing={form.editing === t.id}> <div class="remote_sync" class:running={t.status === "running"} class:editing={form.editing === t.id}>
<div class="row"> <div class="row">
<span class="title"> <span class="title">
<i class="icon {t.direction === 'import' ? 'i_download' : 'i_upload'}"></i> <i class="icon {t.direction === 'import' ? 'i_download' : 'i_upload'}"></i>
@@ -149,7 +149,7 @@ const remove = async (t: Transfer) => {
{t.stats.checks} of {t.stats.totalChecks} {t.stats.checks} of {t.stats.totalChecks}
</div> </div>
<div> <div>
<span>Transfer rate</span> <span>RemoteSync rate</span>
{formatDataVolume(t.stats.speed, 3)}/s {formatDataVolume(t.stats.speed, 3)}/s
</div> </div>
<div> <div>
@@ -166,8 +166,8 @@ const remove = async (t: Transfer) => {
{#if asking === t.id} {#if asking === t.id}
<form class="form" onsubmit={e => { e.preventDefault(); restart(t, { ...secrets, remember: remember_again }) }}> <form class="form" onsubmit={e => { e.preventDefault(); restart(t, { ...secrets, remember: remember_again }) }}>
<!-- Keeps Firefox from filling in the account's login, see TransferForm --> <!-- Keeps Firefox from filling in the account's login, see RemoteSyncForm -->
<input type="text" autocomplete="username" value={"Transfer " + t.remote} style="display: none;"/> <input type="text" autocomplete="username" value={"RemoteSync " + t.remote} style="display: none;"/>
{#each secret_fields(service) as f (f.name)} {#each secret_fields(service) as f (f.name)}
<label for={"restart_" + f.name}>{f.label}</label> <label for={"restart_" + f.name}>{f.label}</label>
{#if f.type === "textarea"} {#if f.type === "textarea"}
@@ -211,15 +211,15 @@ const remove = async (t: Transfer) => {
<style> <style>
/* The cards of the API keys page */ /* The cards of the API keys page */
.transfer { .remote_sync {
margin: 0.8em 0; margin: 0.8em 0;
padding: 0.4em 0.6em; padding: 0.4em 0.6em;
border-radius: 8px; border-radius: 8px;
border-left: 3px solid var(--separator); border-left: 3px solid var(--separator);
background: var(--card_color); background: var(--card_color);
} }
.transfer.running, .remote_sync.running,
.transfer.editing { .remote_sync.editing {
border-left-color: var(--accent_color); border-left-color: var(--accent_color);
} }
.row { .row {
@@ -2,9 +2,9 @@
import { loading_run } from "lib/Loading"; import { loading_run } from "lib/Loading";
import { check_response, dict_to_form, get_endpoint } from "lib/NovaAPI"; import { check_response, dict_to_form, get_endpoint } from "lib/NovaAPI";
import FilePicker from "filesystem/filemanager/FilePicker.svelte"; import FilePicker from "filesystem/filemanager/FilePicker.svelte";
import { backends, form, s3_presets, type Field } from "./TransferLib.svelte"; import { backends, form, s3_presets, type Field } from "./RemoteSyncLib.svelte";
// Called after a transfer was started or saved // Called after a sync was started or saved
let { onsaved }: { onsaved: () => void } = $props() let { onsaved }: { onsaved: () => void } = $props()
let picker: FilePicker = $state() let picker: FilePicker = $state()
@@ -33,7 +33,7 @@ const submit = async (e: SubmitEvent) => {
try { try {
await loading_run(async () => check_response( await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer" + (form.editing ? "/" + form.editing : ""), { await fetch(get_endpoint() + "/user/remote_sync" + (form.editing ? "/" + form.editing : ""), {
method: form.editing ? "PUT" : "POST", method: form.editing ? "PUT" : "POST",
body: dict_to_form({ body: dict_to_form({
direction: form.direction, sync: form.sync, remember: form.remember, directory: form.dir.id, direction: form.direction, sync: form.sync, remember: form.remember, directory: form.dir.id,
@@ -42,7 +42,7 @@ const submit = async (e: SubmitEvent) => {
}) })
)) ))
} catch (err) { } catch (err) {
alert("Failed to " + (form.editing ? "save" : "start") + " the transfer: " + (err.message ?? err)) alert("Failed to " + (form.editing ? "save" : "start") + " the remote sync: " + (err.message ?? err))
return return
} }
@@ -56,19 +56,19 @@ const submit = async (e: SubmitEvent) => {
{#if f.type === "checkbox"} {#if f.type === "checkbox"}
<label><input type="checkbox" bind:checked={form.options[form.backend][f.name]}/> {f.label}</label> <label><input type="checkbox" bind:checked={form.options[form.backend][f.name]}/> {f.label}</label>
{:else} {:else}
<label for={"transfer_" + f.name}>{f.label}</label> <label for={"remote_sync_" + f.name}>{f.label}</label>
{#if f.type === "textarea"} {#if f.type === "textarea"}
<textarea id={"transfer_" + f.name} bind:value={form.options[form.backend][f.name]}></textarea> <textarea id={"remote_sync_" + f.name} bind:value={form.options[form.backend][f.name]}></textarea>
{:else if f.type === "password"} {:else if f.type === "password"}
<input <input
id={"transfer_" + f.name} id={"remote_sync_" + f.name}
type="password" type="password"
autocomplete="off" autocomplete="off"
placeholder={placeholder(f)} placeholder={placeholder(f)}
bind:value={form.options[form.backend][f.name]} bind:value={form.options[form.backend][f.name]}
/> />
{:else} {:else}
<input id={"transfer_" + f.name} type="text" autocomplete="off" placeholder={placeholder(f)} bind:value={form.options[form.backend][f.name]}/> <input id={"remote_sync_" + f.name} type="text" autocomplete="off" placeholder={placeholder(f)} bind:value={form.options[form.backend][f.name]}/>
{/if} {/if}
{#if f.help} {#if f.help}
<small>{@html f.help}</small> <small>{@html f.help}</small>
@@ -78,13 +78,13 @@ const submit = async (e: SubmitEvent) => {
{/snippet} {/snippet}
{#snippet path()} {#snippet path()}
<label for="transfer_path">{backends[form.backend].path}</label> <label for="remote_sync_path">{backends[form.backend].path}</label>
<input id="transfer_path" type="text" bind:value={form.paths[form.backend]} required={backends[form.backend].path_required}/> <input id="remote_sync_path" type="text" bind:value={form.paths[form.backend]} required={backends[form.backend].path_required}/>
{/snippet} {/snippet}
<div id="transfer_form" class="highlight_border"> <div id="remote_sync_form" class="highlight_border">
<div class="navbar"> <div class="navbar">
<b>{form.editing ? "Edit transfer" : "New transfer"}</b> <b>{form.editing ? "Edit remote sync" : "New remote sync"}</b>
{#if form.step === 2} {#if form.step === 2}
<button onclick={() => form.step = 1}> <button onclick={() => form.step = 1}>
{form.direction === "import" ? "Import into" : "Export from"} {form.dir.path} {form.direction === "import" ? "Import into" : "Export from"} {form.dir.path}
@@ -123,14 +123,14 @@ const submit = async (e: SubmitEvent) => {
</label> </label>
<hr/> <hr/>
<label for="transfer_dir">Local directory</label> <label for="remote_sync_dir">Local directory</label>
<button id="transfer_dir" type="button" onclick={() => picker.open("/me")}> <button id="remote_sync_dir" type="button" onclick={() => picker.open("/me")}>
<i class="icon i_folder"></i> <i class="icon i_folder"></i>
{form.dir ? form.dir.path : "Choose a directory"} {form.dir ? form.dir.path : "Choose a directory"}
</button> </button>
<label for="transfer_backend">Remote service</label> <label for="remote_sync_backend">Remote service</label>
<select id="transfer_backend" bind:value={form.backend}> <select id="remote_sync_backend" bind:value={form.backend}>
{#each Object.entries(backends) as [type, b]} {#each Object.entries(backends) as [type, b]}
<option value={type}>{b.name}</option> <option value={type}>{b.name}</option>
{/each} {/each}
@@ -144,11 +144,11 @@ const submit = async (e: SubmitEvent) => {
<form class="form" onsubmit={submit}> <form class="form" onsubmit={submit}>
<!-- Firefox fills the account's login into a form with a password <!-- Firefox fills the account's login into a form with a password
field, unless the form has a username field of its own --> field, unless the form has a username field of its own -->
<input type="text" autocomplete="username" value={"Transfer " + backends[form.backend].name} style="display: none;"/> <input type="text" autocomplete="username" value={"RemoteSync " + backends[form.backend].name} style="display: none;"/>
{#if form.backend === "s3"} {#if form.backend === "s3"}
<label for="transfer_preset">Preset</label> <label for="remote_sync_preset">Preset</label>
<select id="transfer_preset" onchange={e => use_preset(e.currentTarget.value)}> <select id="remote_sync_preset" onchange={e => use_preset(e.currentTarget.value)}>
<option value="" disabled selected>Choose a service to fill in the fields below</option> <option value="" disabled selected>Choose a service to fill in the fields below</option>
{#each s3_presets as p, i} {#each s3_presets as p, i}
{#if p.regions} {#if p.regions}
@@ -193,7 +193,7 @@ const submit = async (e: SubmitEvent) => {
{#if form.editing} {#if form.editing}
<i class="icon i_save"></i> Save <i class="icon i_save"></i> Save
{:else} {:else}
<i class="icon i_sync_alt"></i> Start transfer <i class="icon i_sync_alt"></i> Start sync
{/if} {/if}
</button> </button>
</form> </form>
@@ -204,7 +204,7 @@ const submit = async (e: SubmitEvent) => {
<style> <style>
/* Edit scrolls here, clear of the page's sticky header */ /* Edit scrolls here, clear of the page's sticky header */
#transfer_form { #remote_sync_form {
scroll-margin-top: 3.5em; scroll-margin-top: 3.5em;
} }
/* The choices of the first step, like the checkout widget */ /* The choices of the first step, like the checkout widget */
@@ -168,12 +168,12 @@ export const s3_presets: {
export const secret_fields = (service: string) => backends[service].fields.filter(f => f.secret) export const secret_fields = (service: string) => backends[service].fields.filter(f => f.secret)
export type Transfer = { export type RemoteSync = {
id: string id: string
direction: string direction: string
sync: boolean sync: boolean
directory: string directory: string
// From the home directory, when the transfer started // From the home directory, when the sync started
directory_path: string directory_path: string
remote: string remote: string
// What a restart uses. The options never have the credentials // What a restart uses. The options never have the credentials
@@ -197,9 +197,9 @@ export type Transfer = {
} | null } | null
} }
// The service of a transfer, Nova and pixeldrain both use rclone's pixeldrain // The service of a sync, Nova and pixeldrain both use rclone's pixeldrain
// backend. Empty for a transfer from before the settings were saved // backend. Empty for a type which isn't in backends
export const service_of = (t: Transfer) => { export const service_of = (t: RemoteSync) => {
const service = t.type === "pixeldrain" && t.options.api_url === nova_api ? "nova" : t.type const service = t.type === "pixeldrain" && t.options.api_url === nova_api ? "nova" : t.type
return backends[service] === undefined ? "" : service return backends[service] === undefined ? "" : service
} }
@@ -209,9 +209,9 @@ export const service_of = (t: Transfer) => {
// the passwords in it // the passwords in it
export const form = $state({ export const form = $state({
step: 1, step: 1,
// The transfer which is edited, empty for a new one // The sync which is edited, empty for a new one
editing: "", editing: "",
// The edited transfer has its credentials saved on the server // The edited sync has its credentials saved on the server
saved_secrets: false, saved_secrets: false,
direction: "import", direction: "import",
dir: null as { id: string, path: string } | null, dir: null as { id: string, path: string } | null,
@@ -225,9 +225,9 @@ export const form = $state({
remember: false, remember: false,
}) })
// edit_transfer loads a transfer into the form. Its credentials stay empty, the // edit_remote_sync loads a sync into the form. Its credentials stay empty, the
// server never sends them, and leaving them empty keeps the saved ones // server never sends them, and leaving them empty keeps the saved ones
export const edit_transfer = (t: Transfer) => { export const edit_remote_sync = (t: RemoteSync) => {
const service = service_of(t) const service = service_of(t)
form.step = 1 form.step = 1
form.editing = t.id form.editing = t.id