Add remote sync feature to run rclone on the server side

This commit is contained in:
Pixeldrain-Server committed 2026-11-12 15:50:06 +01:00
1 parent b423446412
commit ee06b91b92
6 files changed
+68 -68

No files matched your search

File renamed without changes.

Before

Width:  |  Height:  |  Size: 65 KiB

After

Width:  |  Height:  |  Size: 65 KiB

+3 -3
View File
@@ -26,10 +26,10 @@ const slides = [
text: `Follow the downloads and data transfer of every file, from the
last day up to five years back.`,
}, {
image: "transfers",
title: "Transfers",
image: "remote_sync",
title: "Remote sync",
text: `Import from or export to S3, Backblaze B2, Azure, SFTP, FTP,
WebDAV or pixeldrain. Transfers run on our servers, so you can close the
WebDAV or pixeldrain. Syncs run on our servers, so you can close the
page while they run.`,
},
]
+4 -4
View File
@@ -12,7 +12,7 @@ import { get_user, type User } from "lib/NovaAPI";
import SharingSettings from "./SharingSettings.svelte";
import SubscriptionManage from "./SubscriptionManage.svelte";
import SubscriptionUsage from "./SubscriptionUsage.svelte";
import Transfers from "./transfers/Transfers.svelte";
import RemoteSync from "./remote_sync/RemoteSync.svelte";
let pages: Tab[] = [
{
@@ -54,10 +54,10 @@ let pages: Tab[] = [
icon: "share",
component: SharingSettings,
}, {
path: "/user/transfers",
title: "Transfers",
path: "/user/remote_sync",
title: "Remote sync",
icon: "sync_alt",
component: Transfers,
component: RemoteSync,
}, {
path: "/user/connect_app",
title: "Apps",
@@ -5,24 +5,24 @@ import { check_response, dict_to_form, get_endpoint } from "lib/NovaAPI";
import ProgressBar from "util/ProgressBar.svelte";
import { formatDataVolume, formatDate, formatDuration } from "util/Formatting";
import { user } from "lib/UserStore";
import TransferForm from "./TransferForm.svelte";
import { edit_transfer, form, secret_fields, service_of, type Transfer } from "./TransferLib.svelte";
import RemoteSyncForm from "./RemoteSyncForm.svelte";
import { edit_remote_sync, form, secret_fields, service_of, type RemoteSync } from "./RemoteSyncLib.svelte";
let transfers: Transfer[] = $state([])
let syncs: RemoteSync[] = $state([])
// Polls every second while a transfer runs, the server saves the
// Polls every second while a sync runs, the server saves the
// progress that often
let mounted = true
let timeout: ReturnType<typeof setTimeout>
const load = async () => {
clearTimeout(timeout)
try {
transfers = await check_response(await fetch(get_endpoint() + "/user/transfer"))
syncs = await check_response(await fetch(get_endpoint() + "/user/remote_sync"))
} catch (err) {
console.error("Failed to load transfers:", err)
console.error("Failed to load remote syncs:", err)
}
if (mounted && transfers.some(t => t.status === "running")) {
if (mounted && syncs.some(t => t.status === "running")) {
timeout = setTimeout(load, 1000)
}
}
@@ -35,33 +35,33 @@ onMount(() => {
}
})
const edit = (t: Transfer) => {
edit_transfer(t)
document.getElementById("transfer_form")?.scrollIntoView({ behavior: "smooth" })
const edit = (t: RemoteSync) => {
edit_remote_sync(t)
document.getElementById("remote_sync_form")?.scrollIntoView({ behavior: "smooth" })
}
const restart = async (t: Transfer, body = {}) => {
const restart = async (t: RemoteSync, body = {}) => {
try {
await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer/" + t.id + "/restart", {
await fetch(get_endpoint() + "/user/remote_sync/" + t.id + "/restart", {
method: "POST",
body: dict_to_form(body),
})
))
} catch (err) {
alert("Failed to restart the transfer: " + (err.message ?? err))
alert("Failed to restart the remote sync: " + (err.message ?? err))
return
}
asking = ""
load()
}
// A transfer without remembered credentials asks for them again. What the form
// A sync without remembered credentials asks for them again. What the form
// still has for that service is filled in
let asking = $state("")
let secrets: { [name: string]: string } = $state({})
let remember_again = $state(false)
const ask = (t: Transfer) => {
const ask = (t: RemoteSync) => {
const service = service_of(t)
secrets = Object.fromEntries(secret_fields(service).map(f =>
[f.name, (form.options[service][f.name] as string) ?? ""]
@@ -70,13 +70,13 @@ const ask = (t: Transfer) => {
asking = t.id
}
const remove = async (t: Transfer) => {
const remove = async (t: RemoteSync) => {
try {
await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer/" + t.id, { method: "DELETE" })
await fetch(get_endpoint() + "/user/remote_sync/" + t.id, { method: "DELETE" })
))
} catch (err) {
alert("Failed to remove the transfer: " + (err.message ?? err))
alert("Failed to remove the remote sync: " + (err.message ?? err))
return
}
load()
@@ -84,29 +84,29 @@ const remove = async (t: Transfer) => {
</script>
<section>
<h2>Transfers</h2>
<h2>Remote sync</h2>
<p>
Copy files between Nova and another storage service. The transfer runs
on our servers, so you can close this page while it runs. A transfer
Copy files between Nova and another storage service. The sync runs
on our servers, so you can close this page while it runs. A sync
which stopped can be started again, files which are already there are
skipped.
</p>
{#if $user !== null && $user.subscription.id === ""}
<p>
Transfers are available on paid plans.
Remote sync is available on paid plans.
<a class="button button_highlight" href="/user/subscription/manage">
<i class="icon i_shopping_cart"></i> Upgrade
</a>
</p>
{:else}
<TransferForm onsaved={load}/>
<RemoteSyncForm onsaved={load}/>
{/if}
{#each transfers as t (t.id)}
{#each syncs as t (t.id)}
{@const name = t.directory_path || t.directory}
{@const service = service_of(t)}
<div class="transfer" class:running={t.status === "running"} class:editing={form.editing === t.id}>
<div class="remote_sync" class:running={t.status === "running"} class:editing={form.editing === t.id}>
<div class="row">
<span class="title">
<i class="icon {t.direction === 'import' ? 'i_download' : 'i_upload'}"></i>
@@ -149,7 +149,7 @@ const remove = async (t: Transfer) => {
{t.stats.checks} of {t.stats.totalChecks}
</div>
<div>
<span>Transfer rate</span>
<span>RemoteSync rate</span>
{formatDataVolume(t.stats.speed, 3)}/s
</div>
<div>
@@ -166,8 +166,8 @@ const remove = async (t: Transfer) => {
{#if asking === t.id}
<form class="form" onsubmit={e => { e.preventDefault(); restart(t, { ...secrets, remember: remember_again }) }}>
<!-- Keeps Firefox from filling in the account's login, see TransferForm -->
<input type="text" autocomplete="username" value={"Transfer " + t.remote} style="display: none;"/>
<!-- Keeps Firefox from filling in the account's login, see RemoteSyncForm -->
<input type="text" autocomplete="username" value={"RemoteSync " + t.remote} style="display: none;"/>
{#each secret_fields(service) as f (f.name)}
<label for={"restart_" + f.name}>{f.label}</label>
{#if f.type === "textarea"}
@@ -211,15 +211,15 @@ const remove = async (t: Transfer) => {
<style>
/* The cards of the API keys page */
.transfer {
.remote_sync {
margin: 0.8em 0;
padding: 0.4em 0.6em;
border-radius: 8px;
border-left: 3px solid var(--separator);
background: var(--card_color);
}
.transfer.running,
.transfer.editing {
.remote_sync.running,
.remote_sync.editing {
border-left-color: var(--accent_color);
}
.row {
@@ -2,9 +2,9 @@
import { loading_run } from "lib/Loading";
import { check_response, dict_to_form, get_endpoint } from "lib/NovaAPI";
import FilePicker from "filesystem/filemanager/FilePicker.svelte";
import { backends, form, s3_presets, type Field } from "./TransferLib.svelte";
import { backends, form, s3_presets, type Field } from "./RemoteSyncLib.svelte";
// Called after a transfer was started or saved
// Called after a sync was started or saved
let { onsaved }: { onsaved: () => void } = $props()
let picker: FilePicker = $state()
@@ -33,7 +33,7 @@ const submit = async (e: SubmitEvent) => {
try {
await loading_run(async () => check_response(
await fetch(get_endpoint() + "/user/transfer" + (form.editing ? "/" + form.editing : ""), {
await fetch(get_endpoint() + "/user/remote_sync" + (form.editing ? "/" + form.editing : ""), {
method: form.editing ? "PUT" : "POST",
body: dict_to_form({
direction: form.direction, sync: form.sync, remember: form.remember, directory: form.dir.id,
@@ -42,7 +42,7 @@ const submit = async (e: SubmitEvent) => {
})
))
} catch (err) {
alert("Failed to " + (form.editing ? "save" : "start") + " the transfer: " + (err.message ?? err))
alert("Failed to " + (form.editing ? "save" : "start") + " the remote sync: " + (err.message ?? err))
return
}
@@ -56,19 +56,19 @@ const submit = async (e: SubmitEvent) => {
{#if f.type === "checkbox"}
<label><input type="checkbox" bind:checked={form.options[form.backend][f.name]}/> {f.label}</label>
{:else}
<label for={"transfer_" + f.name}>{f.label}</label>
<label for={"remote_sync_" + f.name}>{f.label}</label>
{#if f.type === "textarea"}
<textarea id={"transfer_" + f.name} bind:value={form.options[form.backend][f.name]}></textarea>
<textarea id={"remote_sync_" + f.name} bind:value={form.options[form.backend][f.name]}></textarea>
{:else if f.type === "password"}
<input
id={"transfer_" + f.name}
id={"remote_sync_" + f.name}
type="password"
autocomplete="off"
placeholder={placeholder(f)}
bind:value={form.options[form.backend][f.name]}
/>
{:else}
<input id={"transfer_" + f.name} type="text" autocomplete="off" placeholder={placeholder(f)} bind:value={form.options[form.backend][f.name]}/>
<input id={"remote_sync_" + f.name} type="text" autocomplete="off" placeholder={placeholder(f)} bind:value={form.options[form.backend][f.name]}/>
{/if}
{#if f.help}
<small>{@html f.help}</small>
@@ -78,13 +78,13 @@ const submit = async (e: SubmitEvent) => {
{/snippet}
{#snippet path()}
<label for="transfer_path">{backends[form.backend].path}</label>
<input id="transfer_path" type="text" bind:value={form.paths[form.backend]} required={backends[form.backend].path_required}/>
<label for="remote_sync_path">{backends[form.backend].path}</label>
<input id="remote_sync_path" type="text" bind:value={form.paths[form.backend]} required={backends[form.backend].path_required}/>
{/snippet}
<div id="transfer_form" class="highlight_border">
<div id="remote_sync_form" class="highlight_border">
<div class="navbar">
<b>{form.editing ? "Edit transfer" : "New transfer"}</b>
<b>{form.editing ? "Edit remote sync" : "New remote sync"}</b>
{#if form.step === 2}
<button onclick={() => form.step = 1}>
{form.direction === "import" ? "Import into" : "Export from"} {form.dir.path}
@@ -123,14 +123,14 @@ const submit = async (e: SubmitEvent) => {
</label>
<hr/>
<label for="transfer_dir">Local directory</label>
<button id="transfer_dir" type="button" onclick={() => picker.open("/me")}>
<label for="remote_sync_dir">Local directory</label>
<button id="remote_sync_dir" type="button" onclick={() => picker.open("/me")}>
<i class="icon i_folder"></i>
{form.dir ? form.dir.path : "Choose a directory"}
</button>
<label for="transfer_backend">Remote service</label>
<select id="transfer_backend" bind:value={form.backend}>
<label for="remote_sync_backend">Remote service</label>
<select id="remote_sync_backend" bind:value={form.backend}>
{#each Object.entries(backends) as [type, b]}
<option value={type}>{b.name}</option>
{/each}
@@ -144,11 +144,11 @@ const submit = async (e: SubmitEvent) => {
<form class="form" onsubmit={submit}>
<!-- Firefox fills the account's login into a form with a password
field, unless the form has a username field of its own -->
<input type="text" autocomplete="username" value={"Transfer " + backends[form.backend].name} style="display: none;"/>
<input type="text" autocomplete="username" value={"RemoteSync " + backends[form.backend].name} style="display: none;"/>
{#if form.backend === "s3"}
<label for="transfer_preset">Preset</label>
<select id="transfer_preset" onchange={e => use_preset(e.currentTarget.value)}>
<label for="remote_sync_preset">Preset</label>
<select id="remote_sync_preset" onchange={e => use_preset(e.currentTarget.value)}>
<option value="" disabled selected>Choose a service to fill in the fields below</option>
{#each s3_presets as p, i}
{#if p.regions}
@@ -193,7 +193,7 @@ const submit = async (e: SubmitEvent) => {
{#if form.editing}
<i class="icon i_save"></i> Save
{:else}
<i class="icon i_sync_alt"></i> Start transfer
<i class="icon i_sync_alt"></i> Start sync
{/if}
</button>
</form>
@@ -204,7 +204,7 @@ const submit = async (e: SubmitEvent) => {
<style>
/* Edit scrolls here, clear of the page's sticky header */
#transfer_form {
#remote_sync_form {
scroll-margin-top: 3.5em;
}
/* The choices of the first step, like the checkout widget */
@@ -168,12 +168,12 @@ export const s3_presets: {
export const secret_fields = (service: string) => backends[service].fields.filter(f => f.secret)
export type Transfer = {
export type RemoteSync = {
id: string
direction: string
sync: boolean
directory: string
// From the home directory, when the transfer started
// From the home directory, when the sync started
directory_path: string
remote: string
// What a restart uses. The options never have the credentials
@@ -197,9 +197,9 @@ export type Transfer = {
} | null
}
// The service of a transfer, Nova and pixeldrain both use rclone's pixeldrain
// backend. Empty for a transfer from before the settings were saved
export const service_of = (t: Transfer) => {
// The service of a sync, Nova and pixeldrain both use rclone's pixeldrain
// backend. Empty for a type which isn't in backends
export const service_of = (t: RemoteSync) => {
const service = t.type === "pixeldrain" && t.options.api_url === nova_api ? "nova" : t.type
return backends[service] === undefined ? "" : service
}
@@ -209,9 +209,9 @@ export const service_of = (t: Transfer) => {
// the passwords in it
export const form = $state({
step: 1,
// The transfer which is edited, empty for a new one
// The sync which is edited, empty for a new one
editing: "",
// The edited transfer has its credentials saved on the server
// The edited sync has its credentials saved on the server
saved_secrets: false,
direction: "import",
dir: null as { id: string, path: string } | null,
@@ -225,9 +225,9 @@ export const form = $state({
remember: false,
})
// edit_transfer loads a transfer into the form. Its credentials stay empty, the
// edit_remote_sync loads a sync into the form. Its credentials stay empty, the
// server never sends them, and leaving them empty keeps the saved ones
export const edit_transfer = (t: Transfer) => {
export const edit_remote_sync = (t: RemoteSync) => {
const service = service_of(t)
form.step = 1
form.editing = t.id