From 5acad15eba1044e8996af132327445a87fd8d913 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 31 Jul 2026 03:21:36 +0000 Subject: [PATCH] feat: add WebDAV and PostgREST journal remotes --- README.md | 12 +- docs/settings.md | 78 +++++- docs/troubleshooting.md | 25 +- src/apps/cli/commands/runCommand.ts | 4 +- src/apps/cli/commands/runCommand.unit.spec.ts | 151 ++++++++---- src/common/messages/combinedMessages.prod.ts | 18 ++ src/common/messagesJson/en.json | 6 + src/common/messagesYAML/en.yaml | 6 + src/common/reportTool.ts | 9 +- src/common/reportTool.unit.spec.ts | 64 +++++ src/modules/core/ModuleReplicatorCouchDB.ts | 7 +- .../core/ModuleReplicatorCouchDB.unit.spec.ts | 15 ++ src/modules/core/ModuleReplicatorMinIO.ts | 5 +- .../core/ModuleReplicatorMinIO.unit.spec.ts | 46 ++++ .../ObsidianLiveSyncSettingTab.ts | 33 ++- .../SettingDialogue/PaneMaintenance.ts | 182 +++++++------- .../SettingDialogue/PaneRemoteConfig.ts | 98 +++++--- .../SettingDialogue/remoteConfigBuffer.ts | 10 +- .../remoteConfigBuffer.unit.spec.ts | 34 +++ src/modules/features/SetupManager.ts | 76 ++++++ .../features/SetupManager.unit.spec.ts | 52 ++++ .../SetupWizard/dialogs/SetupRemote.svelte | 28 ++- .../dialogs/SetupRemotePostgREST.svelte | 223 ++++++++++++++++++ .../dialogs/SetupRemoteWebDAV.svelte | 203 ++++++++++++++++ .../SetupWizard/dialogs/setupDialogTypes.ts | 16 +- src/modules/services/ObsidianAPIService.ts | 12 +- .../services/ObsidianAPIService.unit.spec.ts | 26 +- src/serviceFeatures/redFlag.ts | 14 +- src/serviceFeatures/redFlag.unit.spec.ts | 86 ++++++- updates.md | 15 ++ 30 files changed, 1317 insertions(+), 237 deletions(-) create mode 100644 src/common/reportTool.unit.spec.ts create mode 100644 src/modules/core/ModuleReplicatorMinIO.unit.spec.ts create mode 100644 src/modules/features/SetupWizard/dialogs/SetupRemotePostgREST.svelte create mode 100644 src/modules/features/SetupWizard/dialogs/SetupRemoteWebDAV.svelte diff --git a/README.md b/README.md index b293b2b4..f95e05e8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [Japanese docs](./README_ja.md) - [Chinese docs](./README_cn.md). -Self-hosted LiveSync is a community-developed synchronisation plug-in available on all Obsidian-compatible platforms. It leverages robust server solutions such as CouchDB or object storage systems (e.g., MinIO, S3, R2, etc.) to ensure reliable data synchronisation. +Self-hosted LiveSync is a community-developed synchronisation plug-in available on all Obsidian-compatible platforms. It supports CouchDB and S3-compatible Object Storage for established server-backed synchronisation, together with experimental WebDAV and PostgREST Journal Storage transports. Additionally, it supports peer-to-peer synchronisation using WebRTC, enabling devices to exchange notes without a central data-storage server. A signalling relay is still required for peer discovery. See [How peer-to-peer synchronisation works](./docs/p2p.md). @@ -17,6 +17,8 @@ Additionally, it supports peer-to-peer synchronisation using WebRTC, enabling de - Automatically merge simple conflicts. - Use open-source solutions for the server. - Compatible solutions are supported. +- Experiment with WebDAV or PostgREST backed by PostgreSQL as alternative Journal Storage transports. + - Review their server, security, and performance constraints in [Remote Configuration](docs/settings.md#3-remote-configuration). - Support end-to-end encryption. - Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Beta)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md). - Enable supported, opt-in WebRTC peer-to-peer synchronisation. @@ -54,7 +56,13 @@ Choose a synchronisation method, prepare its server where required, then follow 2. Object Storage 1. Prepare the server. A maintained MinIO server installation guide is not currently available here, so set up an S3-compatible service or server of your choice. 2. Configure the clients by following [Object Storage Setup](docs/setup_object_storage.md). -3. Peer-to-Peer +3. WebDAV Journal Storage (experimental) + 1. Prepare a dedicated WebDAV collection whose account can use `MKCOL`, `PUT`, `GET`, `PROPFIND`, and `DELETE`. + 2. Configure the clients by following [WebDAV Journal Storage](docs/settings.md#6-webdav-journal-storage). +4. PostgREST Journal Storage (experimental) + 1. Apply the packaged PostgreSQL contract and configure PostgREST and a Vault-scoped JWT as described in [PostgREST Journal Storage](docs/settings.md#7-postgrest-journal-storage). + 2. Configure each client with the same stable Vault ID. +5. Peer-to-Peer 1. No central data-storage server is required. The project's public signalling relay requires no server provisioning; controlled deployments can provide another compatible relay. 2. Configure the clients by following [Peer-to-Peer Setup](docs/setup_p2p.md). diff --git a/docs/settings.md b/docs/settings.md index 1ac07517..17aa615f 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -12,7 +12,7 @@ The following status applies to optional and compatibility features in the 1.0 l | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Supported, opt-in | Peer-to-Peer Synchronisation, Hidden File Sync, and Customisation Sync | Maintained and covered by focused real-runtime tests. Enable them only where their separate setup and operational constraints are acceptable. | | Maintained, advanced | Data Compression | Available as an explicit storage and bandwidth trade-off. It remains disabled by default because the measured processing and memory costs outweigh the mixed-dataset saving. | -| Beta or experimental | JWT authentication, ignore files, automatic newer-file conflict resolution, and Garbage Collection V3 for CouchDB | Retained for explicit testing and specialised use. They remain disabled by default and are not part of the minimum supported setup. | +| Beta or experimental | JWT authentication, WebDAV and PostgREST Journal Storage, ignore files, automatic newer-file conflict resolution, and Garbage Collection V3 for CouchDB | Retained for explicit testing and specialised use. They remain disabled by default and are not part of the minimum supported setup. | | Compatibility only | V1 dynamic iteration counts, the old IndexedDB adapter, non-current hash algorithms, Eden chunks, and the stored `doNotUseFixedRevisionForChunks` key | Existing settings and data remain readable. New Vaults use the current defaults, and compatibility controls are shown only where a migration or recovery path still needs them. | | Icon | Description | @@ -58,7 +58,7 @@ A current Setup URI retains its remote profiles, display names, and separate mai Step-by-step setup for Self-hosted LiveSync. You can setup Self-hosted LiveSync manually with Minimal setting items. -Completing manual CouchDB, Object Storage, or P2P setup creates the corresponding remote profile without replacing profiles which are already saved. CouchDB and Object Storage setup select the new profile as the main remote. P2P setup selects it for P2P use and, when the wizard is enabling LiveSync, also selects it as the main remote. A descriptive display name is generated and can be changed later. +Completing manual CouchDB, Object Storage, WebDAV, PostgREST, or P2P setup creates the corresponding remote profile without replacing profiles which are already saved. CouchDB and each Journal Storage setup select the new profile as the main remote. P2P setup selects it for P2P use and, when the wizard is enabling LiveSync, also selects it as the main remote. A descriptive display name is generated and can be changed later. #### Enable LiveSync @@ -171,12 +171,12 @@ Show verbose log. Please enable when you report the logs ### 1. Connection settings -Self-hosted LiveSync stores multiple remote connection profiles under **Connection settings** → **Saved connections**. Each profile represents a CouchDB database, an Object Storage connection, or a P2P configuration, and several profiles can be kept in one Vault. +Self-hosted LiveSync stores multiple remote connection profiles under **Connection settings** → **Saved connections**. Each profile represents a CouchDB database, an S3-compatible Object Storage connection, a WebDAV collection, a PostgREST Journal endpoint, or a P2P configuration, and several profiles can be kept in one Vault. Each profile has an opaque identifier and a presentation name. The name does not need to be unique and is not used to select the profile. The main remote and the P2P remote are selected independently, so code and settings imports must preserve both selections rather than relying on a special identifier such as `default`. - **➕ Add new connection**: Create a new connection profile by launching the setup dialogue. -- **📥 Import connection**: Paste a connection string (e.g., `sls+https://...`, `sls+s3://...`, `sls+p2p://...`) to import a remote configuration profile. +- **📥 Import connection**: Paste a connection string (e.g., `sls+https://...`, `sls+s3://...`, `sls+webdav://...`, `sls+postgrest://...`, or `sls+p2p://...`) to import a remote configuration profile. - **🔧 Configure**: Open the setup dialogue to edit settings for the selected connection profile. - **✅ Activate**: Select and activate this profile as the current active remote. - **🗑️ Delete**: Remove this connection profile from the list. @@ -305,7 +305,73 @@ Custom HTTP headers to include in every request sent to the Object Storage bucke #### Apply Settings -### 6. CouchDB +### 6. WebDAV Journal Storage + +These settings are configured within the WebDAV Journal Configuration dialogue when adding (`➕`) or editing (`🔧`) a WebDAV connection profile. WebDAV is an experimental Journal Storage transport. + +#### Connection URI + +Setting key: webDAVactiveConnectionURI + +This opaque value stores the fields from the dialogue, including credentials and custom headers. Use the dialogue or an encrypted Setup URI instead of editing or sharing it as plain text. + +#### Endpoint URL + +The HTTP or HTTPS URL of the dedicated WebDAV collection. Only secure HTTPS connections can be used on Obsidian Mobile. The server must support `MKCOL`, `PUT`, `GET`, `PROPFIND`, and `DELETE`, and must preserve binary request bodies. + +#### Username and password + +Optional Basic authentication credentials. Whether anonymous access is allowed is determined by the WebDAV server. + +#### Collection Prefix + +A prefix under the configured collection. Use a dedicated prefix so unrelated files are not included in Journal listing. + +#### Use internal API + +Enable this compatibility option when standard browser requests are blocked by CORS. It sends the configured credentials through Obsidian's internal request API, so use it only with a server you trust. + +#### Custom Headers + +Custom HTTP headers to include in every WebDAV request. Specify them in the format `Header-Name: Value`, with each header on a new line. + +WebDAV normally performs a depth-one listing of the complete collection before filtering Journal keys. Its listing work therefore grows with retained Journal history. Changing only credentials or custom headers retains the stable Journal checkpoint. The first run after the earlier experimental WebDAV implementation also migrates its credential-dependent checkpoint when the endpoint and prefix are unchanged. + +### 7. PostgREST Journal Storage + +PostgREST is an experimental Journal Storage transport backed by PostgreSQL. It stores one opaque binary Journal object per row; it does not translate LiveSync metadata and chunks into a relational file model. + +Before connecting a client, install the packaged [`sql/postgrest/001_journal_storage.sql`](https://github.com/vrtmrz/livesync-commonlib/blob/main/sql/postgrest/001_journal_storage.sql) contract and follow its [deployment notes](https://github.com/vrtmrz/livesync-commonlib/blob/main/sql/postgrest/README.md). Expose the `livesync_api` schema through PostgREST, and issue a JWT whose `role` is `livesync_journal_client` and whose non-empty `vault_id` claim identifies exactly one Vault. + +#### Connection URI + +Setting key: postgrestActiveConnectionURI + +This opaque value stores the endpoint, bearer token, Vault ID, schema, internal-API choice, and custom headers. Treat it as a credential and use an encrypted Setup URI when transferring it to another device. + +#### PostgREST Endpoint URL + +The HTTP or HTTPS base URL of the PostgREST service. Only secure HTTPS connections can be used on Obsidian Mobile. + +#### Bearer Token + +A JWT authorised for `livesync_journal_client`. A long-lived token grants access to every Journal object for its signed Vault, so protect it like the Vault encryption passphrase. Prefer a short-lived token only where token refresh can be managed safely. + +#### Vault ID + +The stable identifier for this Vault. It must exactly match the signed `vault_id` claim. Changing the token while retaining the claim preserves the Journal checkpoint; changing the Vault ID selects a different row namespace and checkpoint. + +#### API Schema + +The PostgreSQL identifier exposed by PostgREST. The packaged contract uses `livesync_api`. + +#### Use internal API and Custom Headers + +The internal API can bypass browser CORS restrictions for a trusted endpoint. Custom headers use one `Header-Name: Value` entry per line. Both are stored in the opaque connection URI. + +PostgREST uses an indexed, bytewise keyset query over `(vault_id, object_key)`. It can therefore reduce listing work compared with a full WebDAV collection scan when many Journal objects are retained. It does not inherently make compression, encryption, upload bandwidth, or download bandwidth faster, and network latency, PostgreSQL resources, and reverse proxies can remove the listing advantage. + +### 8. CouchDB These settings are configured within the CouchDB Setup dialogue when adding (`➕`) or editing (`🔧`) a CouchDB connection profile. @@ -399,7 +465,7 @@ This optional check reads the CouchDB server configuration through Obsidian's in #### Apply Settings -### 7. Peer-to-Peer (P2P) Synchronisation +### 9. Peer-to-Peer (P2P) Synchronisation #### Enable P2P Synchronisation diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 0b7c283e..91e99728 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -10,7 +10,7 @@ Before changing settings: 1. Back up the affected Vaults and, where possible, the remote database or bucket. 2. Stop editing on other devices. 3. Confirm that every participating device uses the intended plug-in version. -4. Identify whether the active main remote is CouchDB, Object Storage, or P2P. +4. Identify whether the active main remote is CouchDB, S3-compatible Object Storage, WebDAV, PostgREST, or P2P. 5. Open `Show log` and note the first error, rather than only the final summary. For a report, run `Generate full report for opening the issue with debug info`, remove credentials and private server details, and include the steps which caused the symptom. @@ -34,6 +34,29 @@ A Cloudflare `524` response means that Cloudflare timed out while waiting for th For JWT-specific setup and key-format errors, see [JWT Authentication on CouchDB](tips/jwt-on-couchdb.md). +## WebDAV Journal Storage does not connect + +Check that the endpoint identifies the intended dedicated collection and that the account can use `MKCOL`, `PUT`, `GET`, `PROPFIND`, and `DELETE`. Confirm that a depth-one `PROPFIND` response includes resource paths and content lengths, and that a binary `PUT` body is returned unchanged by `GET`. + +On mobile, use HTTPS with a certificate trusted by the operating system. For a browser CORS error, correct the WebDAV or reverse-proxy CORS policy where possible. `Use internal API` is a compatibility option for a trusted endpoint, not evidence that the standard server configuration is correct. + +A large or slow `PROPFIND` response is different from a failed connection. Use a dedicated collection prefix and remove unrelated files. Listing work normally grows with retained Journal history. + +## PostgREST Journal Storage does not connect + +Confirm all of the following: + +1. the packaged Commonlib PostgREST SQL has been applied to the target database; +2. PostgREST exposes the configured schema, normally `livesync_api`; +3. the JWT contains `role: livesync_journal_client` and a non-empty signed `vault_id`; +4. the dialogue's Vault ID exactly matches that signed claim; +5. the authenticator can switch to `livesync_journal_client`; and +6. the reverse proxy preserves `Authorization`, content type, binary request bodies, and the custom object headers. + +An HTTP `401` normally means that PostgREST could not authenticate the token. An HTTP `403` commonly indicates a role, claim, row-level-security, or Vault-ID mismatch. An HTTP `404`, `406`, or media-type error can indicate that the configured schema or packaged binary RPC contract is not exposed. Check the PostgREST and PostgreSQL logs before rotating credentials or resetting a Vault. + +PostgREST may make large Journal listings faster through its `(vault_id, object_key)` index. A slow upload or download is a separate network, proxy, or server-resource problem and does not demonstrate an indexing fault. + ## CouchDB was working but synchronisation stopped Do not switch to P2P or reset the database as the first response. Check: diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index c1961500..106f2d98 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -6,10 +6,10 @@ import { type FilePathWithPrefix, type ObsidianLiveSyncSettings, REMOTE_COUCHDB, - REMOTE_MINIO, type EntryMilestoneInfo, type EntryDoc, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { isJournalRemoteType } from "@vrtmrz/livesync-commonlib/journal-storage"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import { activateRemoteConfiguration, @@ -59,7 +59,7 @@ async function verifyRemoteState( return false; } milestone = await dbRet.db.get(MILESTONE_DOCID); - } else if (settings.remoteType === REMOTE_MINIO) { + } else if (isJournalRemoteType(settings.remoteType)) { milestone = await (replicator as LiveSyncJournalReplicator).client.downloadJson("_00000000-milestone.json"); } diff --git a/src/apps/cli/commands/runCommand.unit.spec.ts b/src/apps/cli/commands/runCommand.unit.spec.ts index f3301575..84b7d7fe 100644 --- a/src/apps/cli/commands/runCommand.unit.spec.ts +++ b/src/apps/cli/commands/runCommand.unit.spec.ts @@ -2,7 +2,18 @@ import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node"; import * as processSetting from "@vrtmrz/livesync-commonlib/compat/API/processSetting"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const"; -import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + DEFAULT_SETTINGS, + REMOTE_COUCHDB, + REMOTE_MINIO, + REMOTE_P2P, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + REMOTE_POSTGREST, + REMOTE_WEBDAV, + serialisePostgRESTConnectionURI, + serialiseWebDAVConnectionURI, +} from "@vrtmrz/livesync-commonlib/journal-storage"; import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { runCommand } from "./runCommand"; import type { CLIOptions } from "./types"; @@ -190,6 +201,36 @@ const protocolFixtures: ProtocolFixture[] = [ expect(settings.region).toBe("ap-northeast-1"); }, }, + { + protocol: "webdav", + connectionString: serialiseWebDAVConnectionURI({ + endpoint: "https://dav.example/vault", + username: "webdav-user", + password: "webdav-pass", + prefix: "journal/", + useCustomRequestHandler: true, + customHeaders: "x-test: 1", + }), + assertProjectedFields: (settings) => { + expect(settings.remoteType).toBe(REMOTE_WEBDAV); + expect(settings.webDAVactiveConnectionURI).toContain("sls+webdav://"); + }, + }, + { + protocol: "postgrest", + connectionString: serialisePostgRESTConnectionURI({ + endpoint: "https://journal.example", + bearerToken: "signed-token", + vaultId: "vault-1", + schema: "livesync_api", + useCustomRequestHandler: true, + customHeaders: "x-test: 1", + }), + assertProjectedFields: (settings) => { + expect(settings.remoteType).toBe(REMOTE_POSTGREST); + expect(settings.postgrestActiveConnectionURI).toContain("sls+postgrest://"); + }, + }, { protocol: "p2p", connectionString: ConnectionStringParser.serialize({ @@ -616,55 +657,48 @@ describe("runCommand abnormal cases", () => { } ); - it.each([ - ["couchdb", "sls+https://user:pass@example.com:5984/?db=vault"] as const, - [ - "s3", - "sls+s3://ak:sk@example.com/?endpoint=https%3A%2F%2Fs3.example.com&bucket=my-bucket®ion=ap-northeast-1", - ] as const, - [ - "p2p", - "sls+p2p://room-abc?passphrase=pass-123&relays=wss%3A%2F%2Frelay.example&appId=self-hosted-livesync", - ] as const, - ])("remote command round-trip works for %s", async (_protocol, initialConnStr) => { - const core = createCoreMock(); + it.each(protocolFixtures.map(({ protocol, connectionString }) => [protocol, connectionString] as const))( + "remote command round-trip works for %s", + async (_protocol, initialConnStr) => { + const core = createCoreMock(); - const addOut = captureStdout(core); - const addResult = await runCommand(makeOptions("remote-add", ["rt", initialConnStr]), { - ...context, - core, - }); - expect(addResult).toBe(true); - const remoteId = parseAddedRemoteIdFromLines(addOut.lines()); - expect(remoteId).not.toBe(""); + const addOut = captureStdout(core); + const addResult = await runCommand(makeOptions("remote-add", ["rt", initialConnStr]), { + ...context, + core, + }); + expect(addResult).toBe(true); + const remoteId = parseAddedRemoteIdFromLines(addOut.lines()); + expect(remoteId).not.toBe(""); - const export1Out = captureStdout(core); - const export1Result = await runCommand(makeOptions("remote-export", [remoteId]), { - ...context, - core, - }); - expect(export1Result).toBe(true); - const export1Lines = export1Out.lines(); - const exported1 = export1Lines.length > 0 ? export1Lines[export1Lines.length - 1] : ""; - expect(exported1).toBe(ConnectionStringParser.serialize(ConnectionStringParser.parse(initialConnStr))); + const export1Out = captureStdout(core); + const export1Result = await runCommand(makeOptions("remote-export", [remoteId]), { + ...context, + core, + }); + expect(export1Result).toBe(true); + const export1Lines = export1Out.lines(); + const exported1 = export1Lines.length > 0 ? export1Lines[export1Lines.length - 1] : ""; + expect(exported1).toBe(ConnectionStringParser.serialize(ConnectionStringParser.parse(initialConnStr))); - const roundTripInput = ConnectionStringParser.serialize(ConnectionStringParser.parse(exported1)); - const setResult = await runCommand(makeOptions("remote-set", [remoteId, roundTripInput]), { - ...context, - core, - }); - expect(setResult).toBe(true); + const roundTripInput = ConnectionStringParser.serialize(ConnectionStringParser.parse(exported1)); + const setResult = await runCommand(makeOptions("remote-set", [remoteId, roundTripInput]), { + ...context, + core, + }); + expect(setResult).toBe(true); - const export2Out = captureStdout(core); - const export2Result = await runCommand(makeOptions("remote-export", [remoteId]), { - ...context, - core, - }); - expect(export2Result).toBe(true); - const export2Lines = export2Out.lines(); - const exported2 = export2Lines.length > 0 ? export2Lines[export2Lines.length - 1] : ""; - expect(exported2).toBe(roundTripInput); - }); + const export2Out = captureStdout(core); + const export2Result = await runCommand(makeOptions("remote-export", [remoteId]), { + ...context, + core, + }); + expect(export2Result).toBe(true); + const export2Lines = export2Out.lines(); + const exported2 = export2Lines.length > 0 ? export2Lines[export2Lines.length - 1] : ""; + expect(exported2).toBe(roundTripInput); + } + ); describe("runCommand with decoupled vault path", () => { it("push resolves target path relative to vaultPath, not databasePath", async () => { @@ -706,6 +740,31 @@ describe("runCommand abnormal cases", () => { }); describe("mark-resolved and unlock-remote commands", () => { + it.each([REMOTE_WEBDAV, REMOTE_POSTGREST])( + "verifies the Journal milestone after mark-resolved for %s", + async (remoteType) => { + const core = createCoreMock(); + const downloadJson = vi.fn(async () => ({ + locked: false, + accepted_nodes: ["test-node-id"], + })); + core.services.setting.currentSettings().remoteType = remoteType; + core.services.replicator.getActiveReplicator.mockReturnValue({ + nodeid: "test-node-id", + initializeDatabaseForReplication: vi.fn(async () => {}), + client: { downloadJson }, + }); + + const result = await runCommand(makeOptions("mark-resolved", []), { + ...context, + core, + }); + + expect(result).toBe(true); + expect(downloadJson).toHaveBeenCalledWith("_00000000-milestone.json"); + } + ); + it("mark-resolved without args runs on active database", async () => { const core = createCoreMock(); const result = await runCommand(makeOptions("mark-resolved", []), { diff --git a/src/common/messages/combinedMessages.prod.ts b/src/common/messages/combinedMessages.prod.ts index 00cfba16..0b6c65cb 100644 --- a/src/common/messages/combinedMessages.prod.ts +++ b/src/common/messages/combinedMessages.prod.ts @@ -8913,6 +8913,12 @@ export const allMessages: Readonly `${REDACTED}(${source.length} letters)`; const toSchemeOnly = (uri: string) => { try { diff --git a/src/common/reportTool.unit.spec.ts b/src/common/reportTool.unit.spec.ts new file mode 100644 index 00000000..21b873d5 --- /dev/null +++ b/src/common/reportTool.unit.spec.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/settings"; +import { + REMOTE_POSTGREST, + REMOTE_WEBDAV, + serialisePostgRESTConnectionURI, + serialiseWebDAVConnectionURI, +} from "@vrtmrz/livesync-commonlib/journal-storage"; +import { generateReport } from "./reportTool"; + +vi.mock("./utils", () => ({ + requestToCouchDBWithCredentials: vi.fn(), +})); + +describe("generateReport Journal connection redaction", () => { + it.each([ + { + remoteType: REMOTE_WEBDAV, + settingKey: "webDAVactiveConnectionURI", + secret: "webdav-secret", + uri: serialiseWebDAVConnectionURI({ + endpoint: "https://dav.example/vault", + username: "alice", + password: "webdav-secret", + prefix: "journal/", + useCustomRequestHandler: false, + customHeaders: "x-private-header: private-value", + }), + }, + { + remoteType: REMOTE_POSTGREST, + settingKey: "postgrestActiveConnectionURI", + secret: "signed-jwt-secret", + uri: serialisePostgRESTConnectionURI({ + endpoint: "https://journal.example", + bearerToken: "signed-jwt-secret", + vaultId: "private-vault-id", + schema: "livesync_api", + useCustomRequestHandler: false, + customHeaders: "x-private-header: private-value", + }), + }, + ] as const)("redacts the flat $remoteType connection URI", async (provider) => { + const settings = { + ...DEFAULT_SETTINGS, + remoteType: provider.remoteType, + [provider.settingKey]: provider.uri, + }; + const core = { + services: { + vault: { + isStorageInsensitive: () => false, + }, + }, + } as any; + + const report = await generateReport(settings, core); + const serialised = JSON.stringify(report); + + expect(serialised).not.toContain(provider.secret); + expect(serialised).not.toContain("private-value"); + expect(report.pluginConfig[provider.settingKey]).toBe("𝑅𝐸𝐷𝐴𝐶𝑇𝐸𝐷"); + }); +}); diff --git a/src/modules/core/ModuleReplicatorCouchDB.ts b/src/modules/core/ModuleReplicatorCouchDB.ts index d96fb5c0..405b11b0 100644 --- a/src/modules/core/ModuleReplicatorCouchDB.ts +++ b/src/modules/core/ModuleReplicatorCouchDB.ts @@ -1,5 +1,6 @@ import { fireAndForget } from "octagonal-wheels/promises"; -import { REMOTE_MINIO, REMOTE_P2P, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { REMOTE_P2P, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { isJournalRemoteType } from "@vrtmrz/livesync-commonlib/journal-storage"; import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import type { LiveSyncAbstractReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/LiveSyncAbstractReplicator"; import { AbstractModule } from "@/modules/AbstractModule"; @@ -9,7 +10,7 @@ export class ModuleReplicatorCouchDB extends AbstractModule { _anyNewReplicator(settingOverride: Partial = {}): Promise { const settings = { ...this.settings, ...settingOverride }; // If new remote types were added, add them here. Do not use `REMOTE_COUCHDB` directly for the safety valve. - if (settings.remoteType == REMOTE_MINIO || settings.remoteType == REMOTE_P2P) { + if (isJournalRemoteType(settings.remoteType) || settings.remoteType == REMOTE_P2P) { return Promise.resolve(false); } return Promise.resolve(new LiveSyncCouchDBReplicator(this.core)); @@ -17,7 +18,7 @@ export class ModuleReplicatorCouchDB extends AbstractModule { _everyAfterResumeProcess(): Promise { if (this.services.appLifecycle.isSuspended()) return Promise.resolve(true); if (!this.services.appLifecycle.isReady()) return Promise.resolve(true); - if (this.settings.remoteType != REMOTE_MINIO && this.settings.remoteType != REMOTE_P2P) { + if (!isJournalRemoteType(this.settings.remoteType) && this.settings.remoteType != REMOTE_P2P) { const LiveSyncEnabled = this.settings.liveSync; const continuous = LiveSyncEnabled; const eventualOnStart = !LiveSyncEnabled && this.settings.syncOnStart; diff --git a/src/modules/core/ModuleReplicatorCouchDB.unit.spec.ts b/src/modules/core/ModuleReplicatorCouchDB.unit.spec.ts index c4296e19..623e5733 100644 --- a/src/modules/core/ModuleReplicatorCouchDB.unit.spec.ts +++ b/src/modules/core/ModuleReplicatorCouchDB.unit.spec.ts @@ -1,4 +1,10 @@ import { describe, expect, it, vi } from "vitest"; +import { REMOTE_POSTGREST, REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/journal-storage"; + +vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator", () => ({ + LiveSyncCouchDBReplicator: class {}, +})); + import { ModuleReplicatorCouchDB } from "./ModuleReplicatorCouchDB.ts"; function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isReplicationReady = true) { @@ -43,6 +49,15 @@ function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isR } describe("ModuleReplicatorCouchDB resume replication activity", () => { + it.each([REMOTE_WEBDAV, REMOTE_POSTGREST])("does not claim the %s Journal provider", async (remoteType) => { + const { module } = createModule({ + liveSync: false, + syncOnStart: false, + }); + + expect(await module._anyNewReplicator({ remoteType })).toBe(false); + }); + it("exposes start-up one-shot replication as finite replication activity", async () => { const { module, openReplication, runFiniteReplicationActivity } = createModule({ liveSync: false, diff --git a/src/modules/core/ModuleReplicatorMinIO.ts b/src/modules/core/ModuleReplicatorMinIO.ts index 68dcc22e..abb5987d 100644 --- a/src/modules/core/ModuleReplicatorMinIO.ts +++ b/src/modules/core/ModuleReplicatorMinIO.ts @@ -1,4 +1,5 @@ -import { REMOTE_MINIO, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { isJournalRemoteType } from "@vrtmrz/livesync-commonlib/journal-storage"; import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; import type { LiveSyncAbstractReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/LiveSyncAbstractReplicator"; import type { LiveSyncCore } from "@/main"; @@ -7,7 +8,7 @@ import { AbstractModule } from "@/modules/AbstractModule"; export class ModuleReplicatorMinIO extends AbstractModule { _anyNewReplicator(settingOverride: Partial = {}): Promise { const settings = { ...this.settings, ...settingOverride }; - if (settings.remoteType == REMOTE_MINIO) { + if (isJournalRemoteType(settings.remoteType)) { return Promise.resolve(new LiveSyncJournalReplicator(this.core)); } return Promise.resolve(false); diff --git a/src/modules/core/ModuleReplicatorMinIO.unit.spec.ts b/src/modules/core/ModuleReplicatorMinIO.unit.spec.ts new file mode 100644 index 00000000..27c96252 --- /dev/null +++ b/src/modules/core/ModuleReplicatorMinIO.unit.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import { REMOTE_MINIO, REMOTE_POSTGREST, REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/journal-storage"; +import { REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; + +vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator", () => ({ + LiveSyncJournalReplicator: class {}, +})); + +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; +import { ModuleReplicatorMinIO } from "./ModuleReplicatorMinIO.ts"; + +function createModule(remoteType: string): ModuleReplicatorMinIO { + const services = { + API: { + addCommand: vi.fn(), + addLog: vi.fn(), + addRibbonIcon: vi.fn(), + registerProtocolHandler: vi.fn(), + registerWindow: vi.fn(), + }, + setting: { + saveSettingData: vi.fn(async () => undefined), + }, + }; + const core = { + _services: services, + services, + settings: { remoteType }, + } as any; + return new ModuleReplicatorMinIO(core); +} + +describe("ModuleReplicatorMinIO Journal provider routing", () => { + it.each([REMOTE_MINIO, REMOTE_WEBDAV, REMOTE_POSTGREST])( + "creates the Journal replicator for %s", + async (remoteType) => { + const replicator = await createModule(remoteType)._anyNewReplicator(); + + expect(replicator).toBeInstanceOf(LiveSyncJournalReplicator); + } + ); + + it("does not claim CouchDB", async () => { + expect(await createModule(REMOTE_COUCHDB)._anyNewReplicator()).toBe(false); + }); +}); diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 8e6e04c0..5b3b7e00 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -6,13 +6,13 @@ import { FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_COUCHDB, - REMOTE_MINIO, type ConfigLevel, LEVEL_POWER_USER, LEVEL_ADVANCED, LEVEL_EDGE_CASE, REMOTE_P2P, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { isJournalRemoteType } from "@vrtmrz/livesync-commonlib/journal-storage"; import { delay, isObjectDifferent, sizeToHumanReadable } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger"; import { checkSyncInfo } from "@vrtmrz/livesync-commonlib/compat/pouchdb/negotiation"; @@ -61,8 +61,7 @@ import { panePowerUsers } from "./PanePowerUsers.ts"; import { panePatches } from "./PanePatches.ts"; import { paneMaintenance } from "./PaneMaintenance.ts"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; -import { JournalSyncCore } from "@vrtmrz/livesync-commonlib/compat/replication/journal/JournalSyncCore"; -import { MinioStorageAdapter } from "@vrtmrz/livesync-commonlib/compat/replication/journal/objectstore/MinioStorageAdapter"; +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; import { closeObsidianSettings } from "@/common/obsidianSettings.ts"; // For creating a document @@ -516,22 +515,23 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { ({ visibility: this.isConfiguredAs("remoteType", REMOTE_COUCHDB), }) as OnUpdateResult; - onlyOnMinIO = () => + onlyOnJournal = () => ({ - visibility: this.isConfiguredAs("remoteType", REMOTE_MINIO), + visibility: isJournalRemoteType(this.editingSettings.remoteType), }) as OnUpdateResult; onlyOnOnlyP2P = () => ({ visibility: this.isConfiguredAs("remoteType", REMOTE_P2P), }) as OnUpdateResult; - onlyOnCouchDBOrMinIO = () => + onlyOnCouchDBOrJournal = () => ({ visibility: - this.isConfiguredAs("remoteType", REMOTE_COUCHDB) || this.isConfiguredAs("remoteType", REMOTE_MINIO), + this.isConfiguredAs("remoteType", REMOTE_COUCHDB) || + isJournalRemoteType(this.editingSettings.remoteType), }) as OnUpdateResult; // E2EE Function checkWorkingPassphrase = async (): Promise => { - if (this.editingSettings.remoteType == REMOTE_MINIO) return true; + if (isJournalRemoteType(this.editingSettings.remoteType)) return true; const settingForCheck: RemoteDBSettings = { ...this.editingSettings, @@ -831,18 +831,13 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }); } - getMinioJournalSyncClient() { - // return new JournalSyncMinio(this.core.settings, this.core.simpleStore, this.core); - // const settings = this.editingSettings as ObsidianLiveSyncSettings; - return new JournalSyncCore( - this.core.settings, - this.core.simpleStore, - this.core, - new MinioStorageAdapter(this.core.settings, this.core) - ); + getJournalSyncClient() { + if (!(this.core.replicator instanceof LiveSyncJournalReplicator)) { + throw new Error("The active remote is not a Journal remote"); + } + return this.core.replicator.client; } async resetRemoteBucket() { - const minioJournal = this.getMinioJournalSyncClient(); - await minioJournal.resetBucket(); + await this.getJournalSyncClient().resetBucket(); } } diff --git a/src/modules/features/SettingDialogue/PaneMaintenance.ts b/src/modules/features/SettingDialogue/PaneMaintenance.ts index 47fd1424..3f59f2c3 100644 --- a/src/modules/features/SettingDialogue/PaneMaintenance.ts +++ b/src/modules/features/SettingDialogue/PaneMaintenance.ts @@ -81,7 +81,7 @@ export function paneMaintenance( await this.services.replication.markLocked(); }) ) - .addOnUpdate(this.onlyOnCouchDBOrMinIO); + .addOnUpdate(this.onlyOnCouchDBOrJournal); new Setting(paneEl) .setName("Emergency restart") @@ -127,7 +127,7 @@ export function paneMaintenance( ); }); - void addPanel(paneEl, "Syncing", () => {}, this.onlyOnCouchDBOrMinIO).then((paneEl) => { + void addPanel(paneEl, "Syncing", () => {}, this.onlyOnCouchDBOrJournal).then((paneEl) => { new Setting(paneEl) .setName("Resend") .setDesc("Resend all chunks to the remote.") @@ -155,7 +155,7 @@ export function paneMaintenance( .setWarning() .setDisabled(false) .onClick(async () => { - await this.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ + await this.getJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: new Set(), knownIDs: new Set(), @@ -163,7 +163,7 @@ export function paneMaintenance( Logger(`Journal received history has been cleared.`, LOG_LEVEL_NOTICE); }) ) - .addOnUpdate(this.onlyOnMinIO); + .addOnUpdate(this.onlyOnJournal); new Setting(paneEl) .setName("Reset journal sent history") @@ -176,7 +176,7 @@ export function paneMaintenance( .setWarning() .setDisabled(false) .onClick(async () => { - await this.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ + await this.getJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, lastLocalSeq: 0, sentIDs: new Set(), @@ -185,7 +185,7 @@ export function paneMaintenance( Logger(`Journal sent history has been cleared.`, LOG_LEVEL_NOTICE); }) ) - .addOnUpdate(this.onlyOnMinIO); + .addOnUpdate(this.onlyOnJournal); }); void addPanel(paneEl, "Garbage Collection V3 (Beta)", (e) => e, this.onlyOnCouchDB).then((paneEl) => { new Setting(paneEl) @@ -288,94 +288,96 @@ export function paneMaintenance( // } // ); - void addPanel(paneEl, "Rebuilding Operations (Remote Only)", () => {}, this.onlyOnCouchDBOrMinIO).then((paneEl) => { - new Setting(paneEl) - .setName("Perform cleanup") - .setDesc( - "Reduces storage space by discarding all non-latest revisions. This requires the same amount of free space on the remote server and the local client." - ) - .addButton((button) => - button - .setButtonText("Perform") - .setDisabled(false) - .onClick(async () => { - const replicator = this.core.replicator as LiveSyncCouchDBReplicator; - Logger(`Cleanup has been began`, LOG_LEVEL_NOTICE, "compaction"); - if (await replicator.compactRemote(this.editingSettings)) { - Logger(`Cleanup has been completed!`, LOG_LEVEL_NOTICE, "compaction"); - } else { - Logger(`Cleanup has been failed!`, LOG_LEVEL_NOTICE, "compaction"); - } - }) - ) - .addOnUpdate(this.onlyOnCouchDB); + void addPanel(paneEl, "Rebuilding Operations (Remote Only)", () => {}, this.onlyOnCouchDBOrJournal).then( + (paneEl) => { + new Setting(paneEl) + .setName("Perform cleanup") + .setDesc( + "Reduces storage space by discarding all non-latest revisions. This requires the same amount of free space on the remote server and the local client." + ) + .addButton((button) => + button + .setButtonText("Perform") + .setDisabled(false) + .onClick(async () => { + const replicator = this.core.replicator as LiveSyncCouchDBReplicator; + Logger(`Cleanup has been began`, LOG_LEVEL_NOTICE, "compaction"); + if (await replicator.compactRemote(this.editingSettings)) { + Logger(`Cleanup has been completed!`, LOG_LEVEL_NOTICE, "compaction"); + } else { + Logger(`Cleanup has been failed!`, LOG_LEVEL_NOTICE, "compaction"); + } + }) + ) + .addOnUpdate(this.onlyOnCouchDB); - new Setting(paneEl) - .setName("Overwrite remote") - .setDesc("Overwrite remote with local DB and passphrase.") - .addButton((button) => - button - .setButtonText("Send") - .setWarning() - .setDisabled(false) - .onClick(async () => { - await this.rebuildDB("remoteOnly"); - }) - ); + new Setting(paneEl) + .setName("Overwrite remote") + .setDesc("Overwrite remote with local DB and passphrase.") + .addButton((button) => + button + .setButtonText("Send") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.rebuildDB("remoteOnly"); + }) + ); - new Setting(paneEl) - .setName("Reset all journal counter") - .setDesc("Initialise all journal history, On the next sync, every item will be received and sent.") - .addButton((button) => - button - .setButtonText("Reset all") - .setWarning() - .setDisabled(false) - .onClick(async () => { - await this.getMinioJournalSyncClient().resetCheckpointInfo(); - Logger(`Journal exchange history has been cleared.`, LOG_LEVEL_NOTICE); - }) - ) - .addOnUpdate(this.onlyOnMinIO); + new Setting(paneEl) + .setName("Reset all journal counter") + .setDesc("Initialise all journal history, On the next sync, every item will be received and sent.") + .addButton((button) => + button + .setButtonText("Reset all") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.getJournalSyncClient().resetCheckpointInfo(); + Logger(`Journal exchange history has been cleared.`, LOG_LEVEL_NOTICE); + }) + ) + .addOnUpdate(this.onlyOnJournal); - new Setting(paneEl) - .setName("Purge all journal counter") - .setDesc("Purge all download/upload cache.") - .addButton((button) => - button - .setButtonText("Reset all") - .setWarning() - .setDisabled(false) - .onClick(() => { - this.getMinioJournalSyncClient().resetAllCaches(); - Logger(`Journal download/upload cache has been cleared.`, LOG_LEVEL_NOTICE); - }) - ) - .addOnUpdate(this.onlyOnMinIO); + new Setting(paneEl) + .setName("Purge all journal counter") + .setDesc("Purge all download/upload cache.") + .addButton((button) => + button + .setButtonText("Reset all") + .setWarning() + .setDisabled(false) + .onClick(() => { + this.getJournalSyncClient().resetAllCaches(); + Logger(`Journal download/upload cache has been cleared.`, LOG_LEVEL_NOTICE); + }) + ) + .addOnUpdate(this.onlyOnJournal); - new Setting(paneEl) - .setName("Fresh Start Wipe") - .setDesc("Delete all data on the remote server.") - .addButton((button) => - button - .setButtonText("Delete") - .setWarning() - .setDisabled(false) - .onClick(async () => { - await this.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ - ...info, - receivedFiles: new Set(), - knownIDs: new Set(), - lastLocalSeq: 0, - sentIDs: new Set(), - sentFiles: new Set(), - })); - await this.resetRemoteBucket(); - Logger(`Deleted all data on remote server`, LOG_LEVEL_NOTICE); - }) - ) - .addOnUpdate(this.onlyOnMinIO); - }); + new Setting(paneEl) + .setName("Fresh Start Wipe") + .setDesc("Delete all data on the remote server.") + .addButton((button) => + button + .setButtonText("Delete") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.getJournalSyncClient().updateCheckPointInfo((info) => ({ + ...info, + receivedFiles: new Set(), + knownIDs: new Set(), + lastLocalSeq: 0, + sentIDs: new Set(), + sentFiles: new Set(), + })); + await this.resetRemoteBucket(); + Logger(`Deleted all data on remote server`, LOG_LEVEL_NOTICE); + }) + ) + .addOnUpdate(this.onlyOnJournal); + } + ); void addPanel(paneEl, "Reset").then((paneEl) => { new Setting(paneEl) diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index e3ae96f5..98e48b1a 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -6,7 +6,9 @@ import { LOG_LEVEL_NOTICE, type ObsidianLiveSyncSettings, LOG_LEVEL_VERBOSE, + type RemoteType, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { REMOTE_POSTGREST, REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/journal-storage"; import { Menu, type ButtonComponent } from "@/deps.ts"; import { $msg } from "@/common/translation"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; @@ -21,6 +23,7 @@ import { SetupManager, UserMode } from "@/modules/features/SetupManager.ts"; import { OnDialogSettingsDefault, type AllSettings } from "./settingConstants.ts"; import { activateRemoteConfiguration, + suggestRemoteConfigurationName, type RemoteConfiguration, } from "@vrtmrz/livesync-commonlib/remote-configurations"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; @@ -29,6 +32,8 @@ import SetupRemote from "@/modules/features/SetupWizard/dialogs/SetupRemote.svel import SetupRemoteCouchDB from "@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte"; import SetupRemoteBucket from "@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte"; import SetupRemoteP2P from "@/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte"; +import SetupRemotePostgREST from "@/modules/features/SetupWizard/dialogs/SetupRemotePostgREST.svelte"; +import SetupRemoteWebDAV from "@/modules/features/SetupWizard/dialogs/SetupRemoteWebDAV.svelte"; import type { SetupRemoteCouchDBInitialData, SetupRemoteCouchDBResultType, @@ -60,9 +65,38 @@ function serializeRemoteConfiguration(settings: ObsidianLiveSyncSettings): strin if (settings.remoteType === REMOTE_P2P) { return ConnectionStringParser.serialize({ type: "p2p", settings }); } + if (settings.remoteType === REMOTE_WEBDAV) { + return ConnectionStringParser.serialize({ type: "webdav", settings }); + } + if (settings.remoteType === REMOTE_POSTGREST) { + return ConnectionStringParser.serialize({ type: "postgrest", settings }); + } return ConnectionStringParser.serialize({ type: "couchdb", settings }); } +function remoteTypeFromConfiguration(parsed: RemoteConfigurationResult): RemoteType { + switch (parsed.type) { + case "s3": + return REMOTE_MINIO; + case "p2p": + return REMOTE_P2P; + case "webdav": + return REMOTE_WEBDAV; + case "postgrest": + return REMOTE_POSTGREST; + case "couchdb": + return REMOTE_COUCHDB; + } +} + +function describeRemoteConfiguration(uri: string): string { + try { + return suggestRemoteConfigurationName(ConnectionStringParser.parse(uri)); + } catch { + return "Connection details unavailable"; + } +} + function setEmojiButton(button: ButtonComponent, emoji: string, tooltip: string) { button.setButtonText(emoji); button.setTooltip(tooltip, { delay: 10, placement: "top" }); @@ -71,21 +105,6 @@ function setEmojiButton(button: ButtonComponent, emoji: string, tooltip: string) return button; } -function suggestRemoteConfigurationName(parsed: RemoteConfigurationResult): string { - if (parsed.type === "couchdb") { - try { - const url = new URL(parsed.settings.couchDB_URI); - return `CouchDB ${url.host}`; - } catch { - return "Imported CouchDB"; - } - } - if (parsed.type === "s3") { - return `S3 ${parsed.settings.bucket || parsed.settings.endpoint}`; - } - return `P2P ${parsed.settings.P2P_roomID || "Remote"}`; -} - export function paneRemoteConfig( this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, @@ -189,7 +208,7 @@ export function paneRemoteConfig( }; const runRemoteSetup = async ( baseSettings: ObsidianLiveSyncSettings, - remoteType?: typeof REMOTE_COUCHDB | typeof REMOTE_MINIO | typeof REMOTE_P2P + remoteType?: RemoteType ): Promise => { const setupManager = this.core.getModule(SetupManager); const dialogManager = setupManager.dialogManager; @@ -201,7 +220,15 @@ export function paneRemoteConfig( return false; } targetRemoteType = - method === "bucket" ? REMOTE_MINIO : method === "p2p" ? REMOTE_P2P : REMOTE_COUCHDB; + method === "bucket" + ? REMOTE_MINIO + : method === "p2p" + ? REMOTE_P2P + : method === "webdav" + ? REMOTE_WEBDAV + : method === "postgrest" + ? REMOTE_POSTGREST + : REMOTE_COUCHDB; } if (targetRemoteType === REMOTE_MINIO) { @@ -220,6 +247,25 @@ export function paneRemoteConfig( return { ...baseSettings, ...p2pConf, remoteType: REMOTE_P2P }; } + if (targetRemoteType === REMOTE_WEBDAV) { + const webDAVConf = await dialogManager.openWithExplicitCancel(SetupRemoteWebDAV, baseSettings); + if (webDAVConf === "cancelled" || typeof webDAVConf !== "object") { + return false; + } + return { ...baseSettings, ...webDAVConf, remoteType: REMOTE_WEBDAV }; + } + + if (targetRemoteType === REMOTE_POSTGREST) { + const postgrestConf = await dialogManager.openWithExplicitCancel( + SetupRemotePostgREST, + baseSettings + ); + if (postgrestConf === "cancelled" || typeof postgrestConf !== "object") { + return false; + } + return { ...baseSettings, ...postgrestConf, remoteType: REMOTE_POSTGREST }; + } + const couchConf = await dialogManager.openWithExplicitCancel< SetupRemoteCouchDBResultType, SetupRemoteCouchDBInitialData @@ -328,7 +374,7 @@ export function paneRemoteConfig( for (const config of Object.values(configs)) { const row = new Setting(listContainer) .setName(config.name) - .setDesc(config.uri.split("@").pop() || ""); // Show host part for privacy + .setDesc(describeRemoteConfiguration(config.uri)); if (config.id === this.editingSettings.activeConfigurationId) { row.nameEl.addClass("sls-active-remote-name"); @@ -349,13 +395,7 @@ export function paneRemoteConfig( return; } const workSettings = createBaseRemoteSettings(); - if (parsed.type === "couchdb") { - workSettings.remoteType = REMOTE_COUCHDB; - } else if (parsed.type === "s3") { - workSettings.remoteType = REMOTE_MINIO; - } else { - workSettings.remoteType = REMOTE_P2P; - } + workSettings.remoteType = remoteTypeFromConfiguration(parsed); Object.assign(workSettings, parsed.settings); const nextSettings = await runRemoteSetup(workSettings, workSettings.remoteType); @@ -459,13 +499,7 @@ export function paneRemoteConfig( return; } const workSettings = createBaseRemoteSettings(); - if (parsed.type === "couchdb") { - workSettings.remoteType = REMOTE_COUCHDB; - } else if (parsed.type === "s3") { - workSettings.remoteType = REMOTE_MINIO; - } else { - workSettings.remoteType = REMOTE_P2P; - } + workSettings.remoteType = remoteTypeFromConfiguration(parsed); Object.assign(workSettings, parsed.settings); const newTweaks = await this.services.tweakValue.checkAndAskUseRemoteConfiguration( diff --git a/src/modules/features/SettingDialogue/remoteConfigBuffer.ts b/src/modules/features/SettingDialogue/remoteConfigBuffer.ts index 3d6c23be..7c5b5e00 100644 --- a/src/modules/features/SettingDialogue/remoteConfigBuffer.ts +++ b/src/modules/features/SettingDialogue/remoteConfigBuffer.ts @@ -1,4 +1,10 @@ -import { pickBucketSyncSettings, pickCouchDBSyncSettings, pickP2PSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { + pickBucketSyncSettings, + pickCouchDBSyncSettings, + pickP2PSyncSettings, + pickPostgRESTSyncSettings, + pickWebDAVSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/utils"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; // Keep the setting dialogue buffer aligned with the current core settings before persisting other dirty keys. @@ -13,5 +19,7 @@ export function syncActivatedRemoteSettings( ...pickBucketSyncSettings(source), ...pickCouchDBSyncSettings(source), ...pickP2PSyncSettings(source), + ...pickWebDAVSyncSettings(source), + ...pickPostgRESTSyncSettings(source), }); } diff --git a/src/modules/features/SettingDialogue/remoteConfigBuffer.unit.spec.ts b/src/modules/features/SettingDialogue/remoteConfigBuffer.unit.spec.ts index 66355e65..b964075e 100644 --- a/src/modules/features/SettingDialogue/remoteConfigBuffer.unit.spec.ts +++ b/src/modules/features/SettingDialogue/remoteConfigBuffer.unit.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { REMOTE_POSTGREST, REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/journal-storage"; import { syncActivatedRemoteSettings } from "./remoteConfigBuffer"; describe("syncActivatedRemoteSettings", () => { @@ -80,4 +81,37 @@ describe("syncActivatedRemoteSettings", () => { expect(target.couchDB_PASSWORD).toBe("current-pass"); expect(target.couchDB_DBNAME).toBe("current-db"); }); + + it.each([ + { + activeConfigurationId: "remote-webdav", + remoteType: REMOTE_WEBDAV, + settingKey: "webDAVactiveConnectionURI", + uri: "sls+webdav://alice:secret@dav.example/vault?prefix=journal%2F", + }, + { + activeConfigurationId: "remote-postgrest", + remoteType: REMOTE_POSTGREST, + settingKey: "postgrestActiveConnectionURI", + uri: "sls+postgrest://:token@journal.example?vaultId=vault-a&schema=livesync_api", + }, + ] as const)("should copy the active $remoteType URI into the editing buffer", (provider) => { + const target = { + ...DEFAULT_SETTINGS, + remoteType: REMOTE_COUCHDB, + activeConfigurationId: "old-remote", + }; + const source = { + ...DEFAULT_SETTINGS, + remoteType: provider.remoteType, + activeConfigurationId: provider.activeConfigurationId, + [provider.settingKey]: provider.uri, + }; + + syncActivatedRemoteSettings(target, source); + + expect(target.remoteType).toBe(provider.remoteType); + expect(target.activeConfigurationId).toBe(provider.activeConfigurationId); + expect(target[provider.settingKey]).toBe(provider.uri); + }); }); diff --git a/src/modules/features/SetupManager.ts b/src/modules/features/SetupManager.ts index b37f3ec0..e2c44640 100644 --- a/src/modules/features/SetupManager.ts +++ b/src/modules/features/SetupManager.ts @@ -8,6 +8,12 @@ import { REMOTE_COUCHDB, REMOTE_MINIO, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + REMOTE_POSTGREST, + REMOTE_WEBDAV, + type PostgRESTSyncSetting, + type WebDAVSyncSetting, +} from "@vrtmrz/livesync-commonlib/journal-storage"; import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; import { upsertRemoteConfigurationInPlace } from "@vrtmrz/livesync-commonlib/remote-configurations"; import { isObjectDifferent } from "@vrtmrz/livesync-commonlib/compat/common/utils"; @@ -23,6 +29,8 @@ import SetupRemote from "./SetupWizard/dialogs/SetupRemote.svelte"; import SetupRemoteCouchDB from "./SetupWizard/dialogs/SetupRemoteCouchDB.svelte"; import SetupRemoteBucket from "./SetupWizard/dialogs/SetupRemoteBucket.svelte"; import SetupRemoteP2P from "./SetupWizard/dialogs/SetupRemoteP2P.svelte"; +import SetupRemotePostgREST from "./SetupWizard/dialogs/SetupRemotePostgREST.svelte"; +import SetupRemoteWebDAV from "./SetupWizard/dialogs/SetupRemoteWebDAV.svelte"; import SetupRemoteE2EE from "./SetupWizard/dialogs/SetupRemoteE2EE.svelte"; import { decodeSettingsFromQRCodeData } from "@vrtmrz/livesync-commonlib/compat/API/processSetting"; import { AbstractModule } from "@/modules/AbstractModule.ts"; @@ -36,7 +44,9 @@ import type { SetupRemoteCouchDBInitialData, SetupRemoteE2EEResultType, SetupRemoteP2PResultType, + SetupRemotePostgRESTResultType, SetupRemoteResultType, + SetupRemoteWebDAVResultType, UseSetupURIResultType, } from "./SetupWizard/dialogs/setupDialogTypes.ts"; import { @@ -227,6 +237,68 @@ export class SetupManager extends AbstractModule { return await this.onConfirmApplySettingsFromWizard(newSetting, userMode, activate); } + /** + * Handles manual setup for WebDAV Journal storage. + * @param userMode + * @param currentSetting + * @param activate Whether to activate WebDAV as the main remote + * @returns Promise that resolves to true if setup completed successfully, false otherwise + */ + async onWebDAVManualSetup( + userMode: UserMode, + currentSetting: ObsidianLiveSyncSettings, + activate = true + ): Promise { + const webDAVConf = await this.dialogManager.openWithExplicitCancel< + SetupRemoteWebDAVResultType, + WebDAVSyncSetting + >(SetupRemoteWebDAV, currentSetting); + if (webDAVConf === "cancelled") { + this._log("Manual configuration cancelled.", LOG_LEVEL_NOTICE); + return await this.onOnboard(userMode); + } + const newSetting = { + ...copySettingsForRemoteProfileUpdate(currentSetting), + ...webDAVConf, + } as ObsidianLiveSyncSettings; + if (activate) { + newSetting.remoteType = REMOTE_WEBDAV; + } + upsertRemoteConfigurationInPlace(newSetting, "webdav", { activate }); + return await this.onConfirmApplySettingsFromWizard(newSetting, userMode, activate); + } + + /** + * Handles manual setup for PostgREST Journal storage. + * @param userMode + * @param currentSetting + * @param activate Whether to activate PostgREST as the main remote + * @returns Promise that resolves to true if setup completed successfully, false otherwise + */ + async onPostgRESTManualSetup( + userMode: UserMode, + currentSetting: ObsidianLiveSyncSettings, + activate = true + ): Promise { + const postgrestConf = await this.dialogManager.openWithExplicitCancel< + SetupRemotePostgRESTResultType, + PostgRESTSyncSetting + >(SetupRemotePostgREST, currentSetting); + if (postgrestConf === "cancelled") { + this._log("Manual configuration cancelled.", LOG_LEVEL_NOTICE); + return await this.onOnboard(userMode); + } + const newSetting = { + ...copySettingsForRemoteProfileUpdate(currentSetting), + ...postgrestConf, + } as ObsidianLiveSyncSettings; + if (activate) { + newSetting.remoteType = REMOTE_POSTGREST; + } + upsertRemoteConfigurationInPlace(newSetting, "postgrest", { activate }); + return await this.onConfirmApplySettingsFromWizard(newSetting, userMode, activate); + } + /** * Handles manual setup for P2P * @param userMode @@ -317,6 +389,10 @@ export class SetupManager extends AbstractModule { return await this.onBucketManualSetup(userMode, currentSetting, true); } else if (method === "p2p") { return await this.onP2PManualSetup(userMode, currentSetting, true); + } else if (method === "webdav") { + return await this.onWebDAVManualSetup(userMode, currentSetting, true); + } else if (method === "postgrest") { + return await this.onPostgRESTManualSetup(userMode, currentSetting, true); } else if (method === "cancelled") { this._log("Manual configuration cancelled.", LOG_LEVEL_NOTICE); if (userMode !== UserMode.Unknown) { diff --git a/src/modules/features/SetupManager.unit.spec.ts b/src/modules/features/SetupManager.unit.spec.ts index b9eb45b6..18edbdf4 100644 --- a/src/modules/features/SetupManager.unit.spec.ts +++ b/src/modules/features/SetupManager.unit.spec.ts @@ -8,6 +8,12 @@ import { import { SettingService } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService"; import { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; +import { + REMOTE_POSTGREST, + REMOTE_WEBDAV, + serialisePostgRESTConnectionURI, + serialiseWebDAVConnectionURI, +} from "@vrtmrz/livesync-commonlib/journal-storage"; vi.mock("./SetupWizard/dialogs/Intro.svelte", () => ({ default: {} })); vi.mock("./SetupWizard/dialogs/SelectMethodNewUser.svelte", () => ({ default: {} })); @@ -21,6 +27,8 @@ vi.mock("./SetupWizard/dialogs/SetupRemote.svelte", () => ({ default: {} })); vi.mock("./SetupWizard/dialogs/SetupRemoteCouchDB.svelte", () => ({ default: {} })); vi.mock("./SetupWizard/dialogs/SetupRemoteBucket.svelte", () => ({ default: {} })); vi.mock("./SetupWizard/dialogs/SetupRemoteP2P.svelte", () => ({ default: {} })); +vi.mock("./SetupWizard/dialogs/SetupRemotePostgREST.svelte", () => ({ default: {} })); +vi.mock("./SetupWizard/dialogs/SetupRemoteWebDAV.svelte", () => ({ default: {} })); vi.mock("./SetupWizard/dialogs/SetupRemoteE2EE.svelte", () => ({ default: {} })); vi.mock("@vrtmrz/livesync-commonlib/compat/API/processSetting", () => ({ @@ -147,6 +155,50 @@ describe("SetupManager", () => { expect(configureManually).toHaveBeenCalledWith(createNewVaultSettings(), UserMode.NewUser); }); + it("registers WebDAV and PostgREST manual settings as named remote profiles", async () => { + const { manager, setting, dialogManager } = createSetupManager(); + const confirmApply = vi.spyOn(manager, "onConfirmApplySettingsFromWizard").mockResolvedValue(true); + const webDAVactiveConnectionURI = serialiseWebDAVConnectionURI({ + endpoint: "https://dav.example/vault", + username: "alice", + password: "secret", + prefix: "journal/", + useCustomRequestHandler: false, + customHeaders: "", + }); + dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ webDAVactiveConnectionURI }); + + await manager.onWebDAVManualSetup(UserMode.Update, setting.currentSettings(), true); + + const webDAVSettings = confirmApply.mock.calls[0][0]; + expect(webDAVSettings.remoteType).toBe(REMOTE_WEBDAV); + expect(Object.values(webDAVSettings.remoteConfigurations)).toEqual([ + expect.objectContaining({ + uri: expect.stringMatching(/^sls\+webdav:/u), + }), + ]); + + const postgrestActiveConnectionURI = serialisePostgRESTConnectionURI({ + endpoint: "https://journal.example", + bearerToken: "token", + vaultId: "vault-a", + schema: "livesync_api", + useCustomRequestHandler: false, + customHeaders: "", + }); + dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ postgrestActiveConnectionURI }); + + await manager.onPostgRESTManualSetup(UserMode.Update, setting.currentSettings(), true); + + const postgrestSettings = confirmApply.mock.calls[1][0]; + expect(postgrestSettings.remoteType).toBe(REMOTE_POSTGREST); + expect(Object.values(postgrestSettings.remoteConfigurations)).toEqual([ + expect.objectContaining({ + uri: expect.stringMatching(/^sls\+postgrest:/u), + }), + ]); + }); + it("compatibility: normalises imported flat remote settings from a Setup URI before applying", async () => { const { manager, setting, dialogManager } = createSetupManager(); dialogManager.openWithExplicitCancel diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemote.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemote.svelte index bf15de65..01ac5153 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemote.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemote.svelte @@ -11,6 +11,8 @@ TYPE_COUCHDB, TYPE_BUCKET, TYPE_P2P, + TYPE_POSTGREST, + TYPE_WEBDAV, TYPE_CANCELLED, type SetupRemoteResultType, } from "./setupDialogTypes"; @@ -27,12 +29,22 @@ return translateMessage("Ui.SetupWizard.SetupRemote.ProceedBucket"); } else if (userType === TYPE_P2P) { return translateMessage("Ui.SetupWizard.SetupRemote.ProceedP2P"); + } else if (userType === TYPE_WEBDAV) { + return translateMessage("Ui.SetupWizard.SetupRemote.ProceedWebDAV"); + } else if (userType === TYPE_POSTGREST) { + return translateMessage("Ui.SetupWizard.SetupRemote.ProceedPostgREST"); } else { return "Please select an option to proceed"; } }); const canProceed = $derived.by(() => { - return userType === TYPE_COUCHDB || userType === TYPE_BUCKET || userType === TYPE_P2P; + return ( + userType === TYPE_COUCHDB || + userType === TYPE_BUCKET || + userType === TYPE_P2P || + userType === TYPE_WEBDAV || + userType === TYPE_POSTGREST + ); }); @@ -59,6 +71,20 @@ "No central data-storage server is required, but a signalling relay is required for peer discovery. Both devices must be online at the same time. Vault data travels through the encrypted P2P connection, not through the signalling relay. Some features may be limited." )} + + diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemotePostgREST.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemotePostgREST.svelte new file mode 100644 index 00000000..a01b5ce0 --- /dev/null +++ b/src/modules/features/SetupWizard/dialogs/SetupRemotePostgREST.svelte @@ -0,0 +1,223 @@ + + + + + Configure the LiveSync Journal RPC schema exposed by PostgREST. This is a Journal object transport, not a CouchDB + replacement or direct table editor. + + + + + +Secure HTTPS connections are required on Obsidian Mobile. + + + + + + + + + The Vault ID must exactly match the signed `vault_id` claim in the token. Keep it stable when rotating a token so + that the Journal checkpoint and PostgreSQL row ownership remain unchanged. + + + + + + + Supply an HTTP(S) endpoint without a query or fragment, a bearer token, a Vault ID, and a PostgreSQL identifier for + the schema. + + + + + + + Enable this when browser-compatible requests are blocked by CORS. It uses Obsidian's internal request API and may + behave differently from standard browser fetch. + + + + + + + + + + The server must have the packaged LiveSync PostgREST SQL installed. The saved connection contains a bearer token + and custom headers; protect exported connection strings as credentials. + +{error} + +{#if processing} + Checking connection... Please wait. +{:else} + + checkAndCommit()} /> + commit()} /> + setResult(TYPE_CANCELLED)} /> + +{/if} diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteWebDAV.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteWebDAV.svelte new file mode 100644 index 00000000..3e482f04 --- /dev/null +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteWebDAV.svelte @@ -0,0 +1,203 @@ + + + + + Configure a dedicated WebDAV collection for Journal synchronisation. The server must support MKCOL, PUT, GET, + PROPFIND, and DELETE. + + + + + +Secure HTTPS connections are required on Obsidian Mobile. + + Enter a complete HTTP or HTTPS endpoint without a query string or fragment. + + + + + + + + + + + + + Use a dedicated prefix. WebDAV listing scans the collection, so unrelated files and a very long Journal history + increase synchronisation work. + + + + + + + Enable this when browser-compatible requests are blocked by CORS. It uses Obsidian's internal request API and may + behave differently from standard browser fetch. + + + + + + + + + + The saved connection contains credentials and custom headers. LiveSync encrypts it when configuration encryption is + enabled; do not share an exported connection string as ordinary text. + +{error} + +{#if processing} + Checking connection... Please wait. +{:else} + + checkAndCommit()} /> + commit()} /> + setResult(TYPE_CANCELLED)} /> + +{/if} diff --git a/src/modules/features/SetupWizard/dialogs/setupDialogTypes.ts b/src/modules/features/SetupWizard/dialogs/setupDialogTypes.ts index 8ad4a97b..aaf4d783 100644 --- a/src/modules/features/SetupWizard/dialogs/setupDialogTypes.ts +++ b/src/modules/features/SetupWizard/dialogs/setupDialogTypes.ts @@ -4,6 +4,8 @@ import type { EncryptionSettings, ObsidianLiveSyncSettings, P2PConnectionInfo, + PostgRESTSyncSetting, + WebDAVSyncSetting, } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.type"; export const TYPE_IDENTICAL = "identical"; @@ -40,6 +42,8 @@ export const TYPE_CLOSE = "close"; export const TYPE_COUCHDB = "couchdb"; export const TYPE_BUCKET = "bucket"; export const TYPE_P2P = "p2p"; +export const TYPE_WEBDAV = "webdav"; +export const TYPE_POSTGREST = "postgrest"; export type ResultTypeVault = | typeof TYPE_IDENTICAL @@ -93,7 +97,13 @@ export type SelectMethodExistingResultType = | typeof TYPE_CONFIGURE_MANUALLY | typeof TYPE_CANCELLED; -export type SetupRemoteResultType = typeof TYPE_COUCHDB | typeof TYPE_BUCKET | typeof TYPE_P2P | typeof TYPE_CANCELLED; +export type SetupRemoteResultType = + | typeof TYPE_COUCHDB + | typeof TYPE_BUCKET + | typeof TYPE_P2P + | typeof TYPE_WEBDAV + | typeof TYPE_POSTGREST + | typeof TYPE_CANCELLED; export type UseSetupURIResultType = typeof TYPE_CANCELLED | ObsidianLiveSyncSettings; @@ -110,4 +120,8 @@ export type SetupRemoteCouchDBInitialData = { export type SetupRemoteP2PResultType = typeof TYPE_CANCELLED | P2PConnectionInfo; +export type SetupRemoteWebDAVResultType = typeof TYPE_CANCELLED | WebDAVSyncSetting; + +export type SetupRemotePostgRESTResultType = typeof TYPE_CANCELLED | PostgRESTSyncSetting; + export type ScanQRCodeResultType = typeof TYPE_CLOSE; diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index c455d172..26543f64 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -147,14 +147,10 @@ export class ObsidianAPIService extends InjectableAPIService ({ platform: { isMobile: false, }, + requestUrl: vi.fn(), })); vi.mock("@/deps.ts", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/deps", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/modules/essentialObsidian/APILib/ObsHttpHandler", () => ({ @@ -65,3 +66,24 @@ describe("ObsidianAPIService.showWindowOnRight", () => { expect(workspace.revealLeaf).toHaveBeenCalledWith(rightLeaf); }); }); + +describe("ObsidianAPIService.nativeFetch", () => { + it("converts a binary body supplied with a URL string to an exact ArrayBuffer", async () => { + mocks.requestUrl.mockResolvedValue({ + arrayBuffer: new ArrayBuffer(0), + headers: {}, + status: 200, + }); + const service = createService({}); + const body = new Uint8Array([0, 1, 2, 255]); + + await service.nativeFetch("https://journal.example/object", { + body: body as BodyInit, + method: "PUT", + }); + + const transmittedBody = mocks.requestUrl.mock.calls[0][0].body; + expect(transmittedBody).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(transmittedBody)).toEqual(body); + }); +}); diff --git a/src/serviceFeatures/redFlag.ts b/src/serviceFeatures/redFlag.ts index 67399057..a71c4172 100644 --- a/src/serviceFeatures/redFlag.ts +++ b/src/serviceFeatures/redFlag.ts @@ -8,7 +8,7 @@ import { import FetchEverything from "@/modules/features/SetupWizard/dialogs/FetchEverything.svelte"; import RebuildEverything from "@/modules/features/SetupWizard/dialogs/RebuildEverything.svelte"; import { extractObject } from "octagonal-wheels/object"; -import { REMOTE_MINIO, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; +import { REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/settings"; import { TweakValuesShouldMatchedTemplate } from "@vrtmrz/livesync-commonlib/compat/common/models/tweak.definition"; import type { @@ -17,7 +17,11 @@ import type { } from "@/modules/features/SetupWizard/dialogs/setupDialogTypes"; import { askAndPerformFastSetupOnScheduledFetchAll } from "./redFlag.simpleFetch"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; -import { activateRemoteConfiguration } from "@vrtmrz/livesync-commonlib/remote-configurations"; +import { + activateRemoteConfiguration, + suggestRemoteConfigurationName, +} from "@vrtmrz/livesync-commonlib/remote-configurations"; +import { isJournalRemoteType } from "@vrtmrz/livesync-commonlib/journal-storage"; import { isP2PMainRemote } from "@/common/remoteConfiguration"; /** @@ -61,7 +65,7 @@ async function askAndActivateRemoteDatabase(host: NecessaryServices<"UI" | "sett "Multiple remote configurations detected. Please select the remote configuration you want to fetch from."; const options = Object.entries(settings.remoteConfigurations).map(([id, config]) => { const parsed = ConnectionStringParser.parse(config.uri); - const displayURI = (config.uri.split("@").pop() || "").substring(0, 20) + "..."; // Show only the last part of URI for better readability and privacy. + const displayURI = suggestRemoteConfigurationName(parsed); return { name: `${config.name} - ${parsed.type} (${displayURI})`, id: id, @@ -164,8 +168,8 @@ export function createFetchAllFlagHandler( } const { vault, extra } = method; const settings = await Promise.resolve(host.services.setting.currentSettings()); - // If remote is MinIO, makeLocalChunkBeforeSync is not available. (because no-deduplication on sending). - const makeLocalChunkBeforeSyncAvailable = settings.remoteType !== REMOTE_MINIO; + // Journal Storage remotes do not deduplicate chunks while sending. + const makeLocalChunkBeforeSyncAvailable = !isJournalRemoteType(settings.remoteType); const mapVaultStateToAction = { identical: { makeLocalChunkBeforeSync: makeLocalChunkBeforeSyncAvailable, diff --git a/src/serviceFeatures/redFlag.unit.spec.ts b/src/serviceFeatures/redFlag.unit.spec.ts index de8d998c..6b802b9a 100644 --- a/src/serviceFeatures/redFlag.unit.spec.ts +++ b/src/serviceFeatures/redFlag.unit.spec.ts @@ -41,6 +41,11 @@ import { askSimpleFetchMode, } from "./redFlag.simpleFetch"; import { activateRemoteConfiguration } from "@vrtmrz/livesync-commonlib/remote-configurations"; +import { + REMOTE_POSTGREST, + REMOTE_WEBDAV, + serialiseWebDAVConnectionURI, +} from "@vrtmrz/livesync-commonlib/journal-storage"; //Mock synchroniseAllFilesBetweenDBandStorage vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", async (importOriginal) => { const originalModule = (await importOriginal()) as any; @@ -50,8 +55,10 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", asyn }; }); -vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig", () => { +vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig", async (importOriginal) => { + const originalModule = (await importOriginal()) as any; return { + ...originalModule, activateRemoteConfiguration: vi.fn((settings: any, configurationId: string) => { if (!settings?.remoteConfigurations?.[configurationId]) return false; return { @@ -579,6 +586,40 @@ describe("Red Flag Feature", () => { expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); }); + it("does not expose unauthenticated WebDAV headers in the remote selection", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const privateHeader = "x-private-header: private-value"; + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { + name: "Alpha", + uri: serialiseWebDAVConnectionURI({ + endpoint: "https://dav.example/alpha", + username: "", + password: "", + prefix: "", + useCustomRequestHandler: false, + customHeaders: privateHeader, + }), + }, + beta: { + name: "Beta", + uri: "sls+https://user:pass@example.com/db2", + }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce("Cancel"); + + await createFetchAllFlagHandler(host as any, log).handle(); + + const selections = host.mocks.ui.confirm.askSelectStringDialogue.mock.calls[0]?.[1] as string[]; + expect(selections.join("\n")).not.toContain("private-value"); + expect(selections).toContain("Alpha - webdav (WebDAV dav.example)"); + }); + it("should activate selected remote configuration", async () => { const host = createHostMock(); const log = createLoggerMock(); @@ -1140,25 +1181,44 @@ describe("Red Flag Feature", () => { }); }); - describe("MinIO configuration handling", () => { - it("should not enable makeLocalChunkBeforeSync when remote is MinIO", () => { - const host = createHostMock(); - host.mocks.setting.settings.remoteType = REMOTE_MINIO; + describe("Journal Storage configuration handling", () => { + it.each([REMOTE_MINIO, REMOTE_WEBDAV, REMOTE_POSTGREST])( + "does not prepare deduplicated local chunks for %s", + async (remoteType) => { + const host = createHostMock(); + const log = createLoggerMock(); + host.mocks.setting.settings.remoteType = remoteType; + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_DETAILED); + host.mocks.ui.dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ + vault: "identical", + backup: "backup_skipped", + extra: { preventFetchingConfig: true }, + }); - const settings = host.mocks.setting.currentSettings(); - const isMinIO = settings.remoteType === REMOTE_MINIO; + const result = await createFetchAllFlagHandler(host as any, log).handle(); - expect(isMinIO).toBe(true); - }); + expect(result).toBe(true); + expect(host.mocks.rebuilder.$fetchLocal).toHaveBeenCalledWith(false, true); + } + ); - it("should enable makeLocalChunkBeforeSync for non-MinIO remotes", () => { + it("prepares deduplicated local chunks for CouchDB", async () => { const host = createHostMock(); + const log = createLoggerMock(); host.mocks.setting.settings.remoteType = "CouchDB"; + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_DETAILED); + host.mocks.ui.dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ + vault: "identical", + backup: "backup_skipped", + extra: { preventFetchingConfig: true }, + }); - const settings = host.mocks.setting.currentSettings(); - const isMinIO = settings.remoteType === REMOTE_MINIO; + const result = await createFetchAllFlagHandler(host as any, log).handle(); - expect(isMinIO).toBe(false); + expect(result).toBe(true); + expect(host.mocks.rebuilder.$fetchLocal).toHaveBeenCalledWith(true, true); }); }); diff --git a/updates.md b/updates.md index f6605723..0e6b9821 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,21 @@ Earlier releases remain available in the 0.25 release history and the legacy rel ## Unreleased +### Experimental Journal Storage + +#### Added + +- Added WebDAV Journal Storage to the 1.0 remote-profile, setup, maintenance, CLI, and diagnostic flows, including migration of the earlier experimental checkpoint identity when the endpoint and prefix are unchanged. +- Added a PostgREST Journal Storage option backed by the packaged PostgreSQL schema, Vault-scoped JWT access, row-level security, indexed keyset listing, binary object RPCs, and setup guidance. + +#### Security + +- WebDAV passwords, PostgREST bearer tokens, and custom headers are redacted from generated diagnostic reports and remote display names. + +#### Testing + +- Added disposable WebDAV and PostgREST integration coverage for binary object operations and two-device Journal synchronisation, together with plug-in routing, setup, CLI, native-request, and secret-redaction tests. + ## 1.0.1 29th July, 2026