feat: expose adaptive journal options for S3

docs/settings.md

@@ -301,6 +301,18 @@ Setting key: bucketCustomHeaders
 
 Custom HTTP headers to include in every request sent to the Object Storage bucket. Specify them in the format `Header-Name: Value`, with each header on a new line.
 
+#### Journal data format
+
+Setting key: journalFormat
+
+Existing Object Storage profiles use `opaque-v1` unless Adaptive Journal is selected explicitly. `adaptive-v1` uses an authenticated manifest and immutable Commit Bundles under a separate namespace, with larger Packs stored separately. Changing between formats requires an explicit remote Rebuild; LiveSync does not migrate or read both representations.
+
+#### Pack retrieval
+
+Setting key: packReadPolicy
+
+Adaptive Journal can download a complete immutable Pack (`whole-pack`) or request only the required byte ranges (`range`). Complete Pack retrieval is the portable, throughput-oriented default. Selecting Range requires exact byte-range support from the configured S3-compatible endpoint. The connection test verifies that capability, and synchronisation refuses an unsupported selection before writing.
+
 #### Test Connection
 
 #### Apply Settings

src/common/messages/combinedMessages.prod.ts

@@ -288,6 +288,13 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         zh: "生效中的远程配置",
         "zh-tw": "目前啟用的遠端設定",
     },
+    "Adaptive Journal (experimental)": {
+        def: "Adaptive Journal (experimental)",
+    },
+    "Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats.":
+        {
+            def: "Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats.",
+        },
     "Add default patterns": {
         def: "Add default patterns",
         es: "Añadir patrones predeterminados",
@@ -784,6 +791,10 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         zh: "兼容性（问题修复）",
         "zh-tw": "相容性（問題修復）",
     },
+    "Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing.":
+        {
+            def: "Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing.",
+        },
     "Compute revisions for chunks": {
         def: "Compute revisions for chunks",
         es: "Calcular revisiones para los chunks",
@@ -1585,6 +1596,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         ko: "문서 기록",
         "zh-tw": "文件歷程",
     },
+    "Download complete Packs": {
+        def: "Download complete Packs",
+    },
     Duplicate: {
         def: "Duplicate",
         es: "Duplicar",
@@ -2605,6 +2619,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         ru: "Интервал (сек)",
         zh: "间隔(秒)",
     },
+    "Invalid Object Storage settings: ${reason}": {
+        def: "Invalid Object Storage settings: ${reason}",
+    },
     INVERTED: {
         def: "INVERTED",
         es: "INVERTIDO",
@@ -2617,6 +2634,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         def: "It is strongly advised to create a backup before proceeding. Continuing without a backup may lead to data loss.",
         es: "Se recomienda encarecidamente crear una copia de seguridad antes de continuar. Continuar sin copia de seguridad puede provocar pérdida de datos.",
     },
+    "Journal data format": {
+        def: "Journal data format",
+    },
     "Just for a minute, please!": {
         def: "Just for a minute, please!",
         es: "¡Solo un momento, por favor!",
@@ -6187,6 +6207,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         def: "On this device, switch to the camera app or use a QR code scanner to scan the displayed QR code.",
         es: "En este dispositivo, cambia a la aplicación de cámara o usa un lector de QR para escanear el código mostrado.",
     },
+    "Opaque Journal (current format)": {
+        def: "Opaque Journal (current format)",
+    },
     Open: {
         def: "Open",
         es: "Abrir",
@@ -6447,6 +6470,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         ru: "P2P Sync с name начат.",
         zh: "P2P Sync with ${name} have been started.",
     },
+    "Pack retrieval": {
+        def: "Pack retrieval",
+    },
     "paneMaintenance.markDeviceResolvedAfterBackup": {
         def: "paneMaintenance.markDeviceResolvedAfterBackup",
         es: "Marcar el dispositivo como resuelto después de hacer una copia de seguridad",
@@ -11069,6 +11095,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         def: "Use Random Number",
         es: "Usar número aleatorio",
     },
+    "Use S3 Range requests": {
+        def: "Use S3 Range requests",
+    },
     "Use Segmented-splitter": {
         def: "Use Segmented-splitter",
         es: "Usar divisor segmentado",

src/common/messagesJson/en.json

@@ -44,6 +44,8 @@
     "Action": "Action",
     "Activate": "Activate",
     "Active Remote Configuration": "Active Remote Configuration",
+    "Adaptive Journal (experimental)": "Adaptive Journal (experimental)",
+    "Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats.": "Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats.",
     "Add default patterns": "Add default patterns",
     "Add new connection": "Add new connection",
     "AddOn Module (ConfigSync) has not been loaded. This is very unexpected situation. Please report this issue.": "AddOn Module (ConfigSync) has not been loaded. This is very unexpected situation. Please report this issue.",
@@ -112,6 +114,7 @@
     "Compatibility (Metadata)": "Compatibility (Metadata)",
     "Compatibility (Remote Database)": "Compatibility (Remote Database)",
     "Compatibility (Trouble addressed)": "Compatibility (Trouble addressed)",
+    "Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing.": "Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing.",
     "Compute revisions for chunks": "Compute revisions for chunks",
     "Configuration": "Configuration",
     "Configuration Encryption": "Configuration Encryption",
@@ -212,6 +215,7 @@
     "Doctor.Message.SomeSkipped": "We left some issues as is. Shall I ask you again on next startup?",
     "Doctor.RULES.E2EE_V02500.REASON": "The End-to-End Encryption has got now more robust and faster. Also because, the previous E2EE was found to be compromised in a re-conducted code review. It should be applied as soon as possible. Really apologises for your inconvenience. And, this setting is not forward compatible. All synchronised devices must be updated to v0.25.0 or higher. Rebuilds are not required and will be converted from the new transfer to the new format, However, it is recommended to rebuild whenever possible.",
     "Document History": "Document History",
+    "Download complete Packs": "Download complete Packs",
     "Duplicate": "Duplicate",
     "Duplicate remote": "Duplicate remote",
     "E2EE Configuration": "E2EE Configuration",
@@ -355,9 +359,11 @@
     "Initialise journal received history. On the next sync, every item except this device sent will be downloaded again.": "Initialise journal received history. On the next sync, every item except this device sent will be downloaded again.",
     "Initialise journal sent history. On the next sync, every item except this device received will be sent again.": "Initialise journal sent history. On the next sync, every item except this device received will be sent again.",
     "Interval (sec)": "Interval (sec)",
+    "Invalid Object Storage settings: ${reason}": "Invalid Object Storage settings: ${reason}",
     "INVERTED": "INVERTED",
     "Issue detection log:": "Issue detection log:",
     "It is strongly advised to create a backup before proceeding. Continuing without a backup may lead to data loss.": "It is strongly advised to create a backup before proceeding. Continuing without a backup may lead to data loss.",
+    "Journal data format": "Journal data format",
     "Just for a minute, please!": "Just for a minute, please!",
     "JWT (JSON Web Token) authentication allows you to securely authenticate with the CouchDB server using tokens. Ensure that your CouchDB server is configured to accept JWTs and that the provided key and settings match the server's configuration. Incidentally, I have not verified it very thoroughly.": "JWT (JSON Web Token) authentication allows you to securely authenticate with the CouchDB server using tokens. Ensure that your CouchDB server is configured to accept JWTs and that the provided key and settings match the server's configuration. Incidentally, I have not verified it very thoroughly.",
     "JWT Algorithm": "JWT Algorithm",
@@ -738,6 +744,7 @@
     "On the source device, open Obsidian.": "On the source device, open Obsidian.",
     "On this device, please keep this Vault open.": "On this device, please keep this Vault open.",
     "On this device, switch to the camera app or use a QR code scanner to scan the displayed QR code.": "On this device, switch to the camera app or use a QR code scanner to scan the displayed QR code.",
+    "Opaque Journal (current format)": "Opaque Journal (current format)",
     "Open": "Open",
     "Open connection": "Open connection",
     "Open P2P Setup...": "Open P2P Setup...",
@@ -768,6 +775,7 @@
     "P2P.SyncAlreadyRunning": "P2P Sync is already running.",
     "P2P.SyncCompleted": "P2P Sync completed.",
     "P2P.SyncStartedWith": "P2P Sync with ${name} have been started.",
+    "Pack retrieval": "Pack retrieval",
     "paneMaintenance.markDeviceResolvedAfterBackup": "paneMaintenance.markDeviceResolvedAfterBackup",
     "paneMaintenance.remoteLockedAndDeviceNotAccepted": "paneMaintenance.remoteLockedAndDeviceNotAccepted",
     "paneMaintenance.remoteLockedResolvedDevice": "paneMaintenance.remoteLockedResolvedDevice",
@@ -1418,6 +1426,7 @@
     "Use JWT Authentication": "Use JWT Authentication",
     "Use Path-Style Access": "Use Path-Style Access",
     "Use Random Number": "Use Random Number",
+    "Use S3 Range requests": "Use S3 Range requests",
     "Use Segmented-splitter": "Use Segmented-splitter",
     "Use splitting-limit-capped chunk splitter": "Use splitting-limit-capped chunk splitter",
     "Use the trash bin": "Use the trash bin",

src/common/messagesYAML/en.yaml

@@ -66,6 +66,21 @@ AddOn Module (HiddenFileSync) has not been loaded. This is very unexpected situa
     situation. Please report this issue.
 Advanced: Advanced
 Advanced Settings: Advanced Settings
+Adaptive Journal (experimental): Adaptive Journal (experimental)
+Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats.:
+    Adaptive Journal uses immutable objects and a separate remote format.
+    Existing Opaque Journal data is not migrated or read. Rebuild the remote when
+    changing formats.
+Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing.:
+    Complete Pack reads favour throughput. Range reads can reduce transferred
+    bytes. The connection test verifies exact Range support on this endpoint, and
+    synchronisation refuses an unsupported selection before writing.
+Download complete Packs: Download complete Packs
+"Invalid Object Storage settings: ${reason}": "Invalid Object Storage settings: ${reason}"
+Journal data format: Journal data format
+Opaque Journal (current format): Opaque Journal (current format)
+Pack retrieval: Pack retrieval
+Use S3 Range requests: Use S3 Range requests
 After restarting, the data on this device will be uploaded to the server as the 'master copy'. Please be aware that any unintended data currently on the server will be completely overwritten.:
     After restarting, the data on this device will be uploaded to the server as
     the 'master copy'. Please be aware that any unintended data currently on the

src/modules/features/SettingDialogue/remoteConfigBuffer.unit.spec.ts

@@ -28,6 +28,9 @@ describe("syncActivatedRemoteSettings", () => {
             useCustomRequestHandler: false,
             forcePathStyle: true,
             bucketCustomHeaders: "",
+            expectedRepositoryId: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+            journalFormat: "adaptive-v1" as const,
+            packReadPolicy: "range" as const,
         };
 
         syncActivatedRemoteSettings(target, source);
@@ -40,6 +43,9 @@ describe("syncActivatedRemoteSettings", () => {
         expect(target.bucket).toBe("vault");
         expect(target.region).toBe("sz-hq");
         expect(target.bucketPrefix).toBe("folder/");
+        expect(target.expectedRepositoryId).toBe("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+        expect(target.journalFormat).toBe("adaptive-v1");
+        expect(target.packReadPolicy).toBe("range");
         expect(target.encrypt).toBe(true);
     });
 

src/modules/features/SetupManager.unit.spec.ts

@@ -407,6 +407,9 @@ describe("SetupManager", () => {
                 useCustomRequestHandler: false,
                 bucketCustomHeaders: "",
                 forcePathStyle: true,
+                expectedRepositoryId: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+                journalFormat: "adaptive-v1",
+                packReadPolicy: "range",
             })
             .mockResolvedValueOnce(true);
 
@@ -419,6 +422,9 @@ describe("SetupManager", () => {
         const activeProfile = current.remoteConfigurations[current.activeConfigurationId];
         expect(activeProfile?.name).toBe("S3 notes");
         expect(activeProfile?.uri).toContain("sls+s3://key:secret@storage.example");
+        expect(activeProfile?.uri).toContain("journalFormat=adaptive-v1");
+        expect(activeProfile?.uri).toContain("packReadPolicy=range");
+        expect(activeProfile?.uri).toContain("expectedRepositoryId=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
     });
 
     it("creates and selects a P2P profile during fresh manual onboarding", async () => {

src/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte

@@ -20,10 +20,16 @@
     import { copyTo, pickBucketSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/utils";
     import { TYPE_CANCELLED, type SetupRemoteBucketResultType } from "./setupDialogTypes";
     import { $msg as translateMessage } from "@/common/translation";
+    import { normaliseS3JournalSettings } from "./s3JournalSettings";
 
     const default_setting = pickBucketSyncSettings(DEFAULT_SETTINGS);
 
-    let syncSetting = $state<BucketSyncSetting>({ ...default_setting });
+    let syncSetting = $state<BucketSyncSetting>({
+        ...default_setting,
+        expectedRepositoryId: default_setting.expectedRepositoryId ?? "",
+        journalFormat: default_setting.journalFormat ?? "opaque-v1",
+        packReadPolicy: default_setting.packReadPolicy ?? "whole-pack",
+    });
 
     type Props = GuestDialogProps<SetupRemoteBucketResultType, BucketSyncSetting>;
 
@@ -58,11 +64,10 @@
             isEndpointSupplied
         );
     });
+    const isAdaptive = $derived(syncSetting.journalFormat === "adaptive-v1");
 
     function generateSetting() {
-        const connSetting: BucketSyncSetting = {
-            ...syncSetting,
-        };
+        const connSetting = normaliseS3JournalSettings(syncSetting);
         const trialSettings: BucketSyncSetting = {
             ...connSetting,
         };
@@ -115,8 +120,13 @@
         }
     }
     function commit() {
-        const setting = pickBucketSyncSettings(generateSetting());
-        setResult(setting);
+        error = "";
+        try {
+            const setting = pickBucketSyncSettings(generateSetting());
+            setResult(setting);
+        } catch (e) {
+            error = translateMessage("Invalid Object Storage settings: ${reason}", { reason: `${e}` });
+        }
     }
     function cancel() {
         setResult(TYPE_CANCELLED);
@@ -220,6 +230,30 @@
 </InfoNote>
 
 <ExtraItems title={translateMessage("Advanced Settings")}>
+    <InputRow label={translateMessage("Journal data format")}>
+        <select name="s3-journal-format" bind:value={syncSetting.journalFormat}>
+            <option value="opaque-v1">{translateMessage("Opaque Journal (current format)")}</option>
+            <option value="adaptive-v1">{translateMessage("Adaptive Journal (experimental)")}</option>
+        </select>
+    </InputRow>
+    <InfoNote warning visible={isAdaptive}>
+        {translateMessage(
+            "Adaptive Journal uses immutable objects and a separate remote format. Existing Opaque Journal data is not migrated or read. Rebuild the remote when changing formats."
+        )}
+    </InfoNote>
+    {#if isAdaptive}
+        <InputRow label={translateMessage("Pack retrieval")}>
+            <select name="s3-pack-read-policy" bind:value={syncSetting.packReadPolicy}>
+                <option value="whole-pack">{translateMessage("Download complete Packs")}</option>
+                <option value="range">{translateMessage("Use S3 Range requests")}</option>
+            </select>
+        </InputRow>
+        <InfoNote>
+            {translateMessage(
+                "Complete Pack reads favour throughput. Range reads can reduce transferred bytes. The connection test verifies exact Range support on this endpoint, and synchronisation refuses an unsupported selection before writing."
+            )}
+        </InfoNote>
+    {/if}
     <InputRow label={translateMessage("Custom Headers")}>
         <textarea
             name="bucket-custom-headers"

src/modules/features/SetupWizard/dialogs/s3JournalSettings.ts

@@ -0,0 +1,25 @@
+import { DEFAULT_SETTINGS, REMOTE_MINIO, type BucketSyncSetting } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { journalProtocolConfigurationForSettings } from "@vrtmrz/livesync-commonlib/journal-storage";
+
+export function normaliseS3JournalSettings(settings: BucketSyncSetting): BucketSyncSetting {
+    const journalFormat = settings.journalFormat ?? "opaque-v1";
+    const candidate: BucketSyncSetting = {
+        ...settings,
+        bucket: settings.bucket.trim(),
+        bucketPrefix: settings.bucketPrefix.trim(),
+        endpoint: settings.endpoint.trim(),
+        expectedRepositoryId: journalFormat === "adaptive-v1" ? (settings.expectedRepositoryId ?? "").trim() : "",
+        journalFormat,
+        packReadPolicy: journalFormat === "adaptive-v1" ? (settings.packReadPolicy ?? "whole-pack") : "whole-pack",
+        region: settings.region.trim(),
+    };
+    const protocol = journalProtocolConfigurationForSettings({
+        ...DEFAULT_SETTINGS,
+        ...candidate,
+        remoteType: REMOTE_MINIO,
+    });
+    return {
+        ...candidate,
+        ...protocol,
+    };
+}

src/modules/features/SetupWizard/dialogs/s3JournalSettings.unit.spec.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import { DEFAULT_SETTINGS, type BucketSyncSetting } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { pickBucketSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/utils";
+import { normaliseS3JournalSettings } from "./s3JournalSettings";
+
+function settings(overrides: Partial<BucketSyncSetting> = {}): BucketSyncSetting {
+    return {
+        ...pickBucketSyncSettings(DEFAULT_SETTINGS),
+        ...overrides,
+    };
+}
+
+describe("normaliseS3JournalSettings", () => {
+    it("retains and validates Adaptive repository options", () => {
+        const repositoryId = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+
+        const result = normaliseS3JournalSettings(
+            settings({
+                bucket: " vault ",
+                bucketPrefix: " journals/ ",
+                endpoint: " https://storage.example ",
+                expectedRepositoryId: ` ${repositoryId} `,
+                journalFormat: "adaptive-v1",
+                packReadPolicy: "range",
+                region: " auto ",
+            })
+        );
+
+        expect(result).toMatchObject({
+            bucket: "vault",
+            bucketPrefix: "journals/",
+            endpoint: "https://storage.example",
+            expectedRepositoryId: repositoryId,
+            journalFormat: "adaptive-v1",
+            packReadPolicy: "range",
+            region: "auto",
+        });
+    });
+
+    it("uses conservative Opaque defaults for older profiles", () => {
+        const legacy = settings();
+        delete legacy.expectedRepositoryId;
+        delete legacy.journalFormat;
+        delete legacy.packReadPolicy;
+
+        expect(normaliseS3JournalSettings(legacy)).toMatchObject({
+            expectedRepositoryId: "",
+            journalFormat: "opaque-v1",
+            packReadPolicy: "whole-pack",
+        });
+    });
+
+    it("clears Adaptive-only options when Opaque Journal is selected", () => {
+        expect(
+            normaliseS3JournalSettings(
+                settings({
+                    expectedRepositoryId: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+                    journalFormat: "opaque-v1",
+                    packReadPolicy: "range",
+                })
+            )
+        ).toMatchObject({
+            expectedRepositoryId: "",
+            journalFormat: "opaque-v1",
+            packReadPolicy: "whole-pack",
+        });
+    });
+
+    it("rejects a non-canonical expected repository ID", () => {
+        expect(() =>
+            normaliseS3JournalSettings(
+                settings({
+                    expectedRepositoryId: "not-a-repository-id",
+                    journalFormat: "adaptive-v1",
+                })
+            )
+        ).toThrow("expectedRepositoryId must be a canonical base64url-encoded 32-byte value");
+    });
+});

updates.md

@@ -12,6 +12,12 @@ Earlier releases remain available in the 0.25 release history and the legacy rel
 
 ## Unreleased
 
+### Synchronisation and storage
+
+#### Improved
+
+- Object Storage setup can select the experimental Adaptive Journal format and choose complete Pack or verified Range retrieval. Existing Opaque Journal repositories remain the default and require an explicit remote Rebuild before changing formats.
+
 ### P2P and experimental browser applications
 
 #### Improved