Compare commits

..

3 Commits

Author SHA1 Message Date
vorotamoroz a722244a92 fix(cli): replace the headless log handler 2026-08-01 06:20:47 +00:00
vorotamoroz 4f14d7f160 feat(cli): support adaptive WebDAV remotes 2026-08-01 06:09:32 +00:00
vorotamoroz 7955ab54d3 refactor: compose Journal remotes generically 2026-08-01 06:09:02 +00:00
17 changed files with 546 additions and 41 deletions
+1
View File
@@ -45,6 +45,7 @@
"test:e2e:cli": "npm run test:e2e:ci --workspace self-hosted-livesync-cli",
"test:e2e:cli:p2p": "npm run test:e2e:p2p --workspace self-hosted-livesync-cli",
"test:e2e:cli:adaptive-s3": "npm run test:e2e:adaptive-s3 --workspace self-hosted-livesync-cli",
"test:e2e:cli:adaptive-webdav": "npm run test:e2e:adaptive-webdav --workspace self-hosted-livesync-cli",
"test:e2e:cli:all": "npm run test:e2e:all --workspace self-hosted-livesync-cli",
"test:integration": "npx dotenv-cli -e .env -e .test.env -- vitest run --config vitest.config.integration.ts",
"test:unit:coverage": "vitest run --config vitest.config.unit.ts --coverage",
+2 -2
View File
@@ -21,7 +21,7 @@ import { AbstractModule } from "./modules/AbstractModule";
import { ModulePeriodicProcess } from "./modules/core/ModulePeriodicProcess";
import { ModuleReplicator } from "./modules/core/ModuleReplicator";
import { ModuleReplicatorCouchDB } from "./modules/core/ModuleReplicatorCouchDB";
import { ModuleReplicatorMinIO } from "./modules/core/ModuleReplicatorMinIO";
import { ModuleReplicatorJournal } from "./modules/core/ModuleReplicatorJournal";
import { ModuleConflictChecker } from "./modules/coreFeatures/ModuleConflictChecker";
import { ModuleConflictResolver } from "./modules/coreFeatures/ModuleConflictResolver";
import { ModuleResolvingMismatchedTweaks } from "./modules/coreFeatures/ModuleResolveMismatchedTweaks";
@@ -139,7 +139,7 @@ export class LiveSyncBaseCore<
public registerModules(extraModules: AbstractModule[] = []) {
this._registerModule(new ModuleLiveSyncMain(this));
this._registerModule(new ModuleConflictChecker(this));
this._registerModule(new ModuleReplicatorMinIO(this));
this._registerModule(new ModuleReplicatorJournal(this));
this._registerModule(new ModuleReplicatorCouchDB(this));
this._registerModule(new ModuleReplicator(this));
this._registerModule(new ModuleConflictResolver(this));
+2 -2
View File
@@ -6,7 +6,7 @@ import {
type FilePathWithPrefix,
type ObsidianLiveSyncSettings,
REMOTE_COUCHDB,
REMOTE_MINIO,
isJournalRemoteType,
type EntryMilestoneInfo,
type EntryDoc,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
@@ -61,7 +61,7 @@ async function verifyRemoteState(
return false;
}
milestone = await dbRet.db.get(MILESTONE_DOCID);
} else if (settings.remoteType === REMOTE_MINIO) {
} else if (isJournalRemoteType(settings.remoteType)) {
const journalReplicator = replicator as LiveSyncJournalReplicator;
if (journalProtocolConfigurationForSettings(settings).journalFormat === "adaptive-v1") {
try {
@@ -7,6 +7,7 @@ import {
REMOTE_COUCHDB,
REMOTE_MINIO,
REMOTE_P2P,
REMOTE_WEBDAV,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { runCommand } from "./runCommand";
@@ -750,6 +751,32 @@ describe("runCommand abnormal cases", () => {
);
});
it("uses the Adaptive verification path for a WebDAV remote", async () => {
const core = createCoreMock();
const settings = core.services.setting.currentSettings();
settings.remoteType = REMOTE_WEBDAV;
settings.webDAVactiveConnectionURI = "sls+webdav://dav.example/dav";
settings.journalFormat = "adaptive-v1";
settings.packReadPolicy = "whole-pack";
const ensureCheckpointCachesAreFresh = vi.fn(async () => {});
core.services.replicator.getActiveReplicator.mockReturnValue({
nodeid: "test-node-id",
initializeDatabaseForReplication: vi.fn(async () => {}),
client: {
ensureCheckpointCachesAreFresh,
},
});
const result = await runCommand(makeOptions("mark-resolved", []), {
...context,
core,
});
expect(result).toBe(true);
expect(ensureCheckpointCachesAreFresh).toHaveBeenCalledTimes(1);
});
it("mark-resolved with remote-id temporarily activates it and runs markResolved", async () => {
const core = createCoreMock();
const settings = core.services.setting.currentSettings();
+27 -4
View File
@@ -37,6 +37,14 @@ defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
/** Injectable command boundary used by CLI integration probes. */
export type CliCommandRunner = (options: CLIOptions, context: CLICommandContext) => Promise<boolean>;
const SETTINGS_MANAGEMENT_COMMANDS: ReadonlySet<CLICommand> = new Set([
"setup",
"remote-add",
"remote-rm",
"remote-set",
"remote-activate",
]);
function printHelp(standardIo: StandardIo): void {
writeStdoutLine(
standardIo,
@@ -274,7 +282,10 @@ export async function main(
) {
const options = parseArgs(standardIo);
if (options.interval && options.command !== "daemon") {
writeStderrLine(standardIo, `Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
writeStderrLine(
standardIo,
`Warning: --interval is only used in daemon mode, ignored for '${options.command}'`
);
}
const avoidStdoutNoise =
options.command === "cat" ||
@@ -399,12 +410,15 @@ export async function main(
if (!options.verbose) return;
}
writeStderrLine(standardIo, prefix, message);
});
}, true);
// Prevent replication result from being processed automatically in non-daemon commands.
// In daemon mode the default handler must run so changes are applied to the filesystem.
if (options.command !== "daemon") {
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
writeStderrLine(standardIo, `[Info] Replication result received, but not processed automatically in CLI mode.`);
writeStderrLine(
standardIo,
`[Info] Replication result received, but not processed automatically in CLI mode.`
);
return await Promise.resolve(true);
}, -100);
}
@@ -506,7 +520,7 @@ export async function main(
// Save the settings file before any lifecycle events can mutate and persist them.
// suspendAllSync and other lifecycle hooks clobber sync settings in memory, and
// various code paths persist the clobbered state to disk. We restore on shutdown.
const settingsBackup = await fs.readFile(settingsPath, "utf-8").catch(() => null!);
let settingsBackup: string | null = await fs.readFile(settingsPath, "utf-8").catch(() => null);
// Restore settings file on any exit to undo lifecycle mutations.
// Write to a temp path first so a crash mid-write doesn't leave a truncated file.
@@ -576,6 +590,15 @@ export async function main(
settingsPath,
originalSyncSettings,
});
if (result && SETTINGS_MANAGEMENT_COMMANDS.has(options.command)) {
// Settings management is intentional, unlike the temporary changes made by suspendAllSync().
// setup replaces the complete configuration, while remote profile commands must retain the
// synchronisation mode which was active before the command lifecycle suspended it.
if (options.command !== "setup") {
await core.services.setting.applyPartial(originalSyncSettings, true);
}
settingsBackup = await fs.readFile(settingsPath, "utf-8");
}
if (!result) {
writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`);
process.exitCode = 1;
+2
View File
@@ -24,6 +24,8 @@
"test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose",
"pretest:e2e:adaptive-s3": "npm run build",
"test:e2e:adaptive-s3": "deno task --cwd testdeno test:adaptive-journal-s3",
"pretest:e2e:adaptive-webdav": "npm run build",
"test:e2e:adaptive-webdav": "deno task --cwd testdeno test:adaptive-journal-webdav",
"test:e2e:mirror": "bash test/test-mirror-linux.sh",
"test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh",
"pretest:e2e:all": "npm run build",
+2 -1
View File
@@ -35,7 +35,8 @@
"test:e2e-matrix:couchdb-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc1' test-e2e-two-vaults-matrix.ts",
"test:e2e-matrix:minio-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc0' test-e2e-two-vaults-matrix.ts",
"test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts",
"test:adaptive-journal-s3": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-s3.ts"
"test:adaptive-journal-s3": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-s3.ts",
"test:adaptive-journal-webdav": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-webdav.ts"
},
"imports": {
"@std/assert": "jsr:@std/assert@^1.0.13",
+8 -1
View File
@@ -23,6 +23,7 @@ export interface CliResult {
export const TEE_ENABLED = Deno.env.get("LIVESYNC_TEST_TEE") === "1";
const VERBOSE_ENABLED = Deno.env.get("LIVESYNC_CLI_VERBOSE") === "1";
const DEBUG_ENABLED = Deno.env.get("LIVESYNC_CLI_DEBUG") === "1";
const SETUP_URI_PREFIX = "obsidian://setuplivesync?settings=";
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
@@ -40,7 +41,13 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array {
}
export function formatTeeCommand(args: string[]): string {
return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" ");
const redactArgument = (argument: string): string => {
if (argument.startsWith(SETUP_URI_PREFIX)) {
return `${SETUP_URI_PREFIX}<redacted>`;
}
return argument.replace(/^(sls\+[^:]+:\/\/)[^/?#@]*@/u, "$1<redacted>@");
};
return ["node", CLI_DIST, ...args.map(redactArgument)].map((part) => JSON.stringify(part)).join(" ");
}
export function createLineTeeWriter(
+140
View File
@@ -330,6 +330,39 @@ const MINIO_CONTAINER = "minio-test";
const MINIO_IMAGE = "minio/minio";
const MINIO_MC_IMAGE = "minio/mc";
const WEBDAV_CONTAINER = "webdav-test";
const WEBDAV_IMAGE = "httpd:2.4.68";
const WEBDAV_HTTPD_CONFIG = `ServerRoot "/usr/local/apache2"
Listen 80
LoadModule mpm_event_module modules/mod_mpm_event.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule unixd_module modules/mod_unixd.so
User www-data
Group www-data
ServerName localhost
DocumentRoot "/usr/local/apache2/htdocs"
PidFile "/tmp/httpd.pid"
ErrorLog "/proc/self/fd/2"
LogLevel warn
DavLockDB "/usr/local/apache2/var/DavLock"
DavLockDiscovery Off
<Directory "/usr/local/apache2/htdocs">
AllowOverride None
Require all granted
</Directory>
<Directory "/usr/local/apache2/htdocs/dav">
Dav On
</Directory>
`;
export async function stopCouchdb(): Promise<void> {
await stopAndRemoveContainer(COUCHDB_CONTAINER);
untrackContainer(COUCHDB_CONTAINER);
@@ -625,6 +658,113 @@ export async function startMinio(
await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
}
// ---------------------------------------------------------------------------
// WebDAV
// ---------------------------------------------------------------------------
export async function stopWebDAV(): Promise<void> {
await stopAndRemoveContainer(WEBDAV_CONTAINER);
untrackContainer(WEBDAV_CONTAINER);
}
async function waitForWebDAV(endpoint: string): Promise<void> {
for (let attempt = 0; attempt < 30; attempt += 1) {
try {
const response = await fetch(endpoint, {
method: "PROPFIND",
headers: { Depth: "0" },
signal: AbortSignal.timeout(3000),
});
await response.body?.cancel().catch(() => {});
if (response.status === 207) return;
} catch {
// The container is still starting.
}
await sleep(500);
}
throw new Error(`WebDAV collection did not become ready: ${endpoint}`);
}
export async function startWebDAV(endpoint: string): Promise<void> {
const url = new URL(endpoint);
if (url.protocol !== "http:" || (url.hostname !== "127.0.0.1" && url.hostname !== "localhost")) {
throw new Error(`Managed WebDAV requires a local HTTP endpoint, received: ${endpoint}`);
}
if (url.pathname.replace(/\/+$/u, "") !== "/dav") {
throw new Error(`Managed WebDAV requires the /dav collection, received: ${endpoint}`);
}
const hostPort = url.port || "80";
const encodedConfig = btoa(WEBDAV_HTTPD_CONFIG);
const startCommand = `printf '%s' '${encodedConfig}' | base64 -d > /tmp/httpd.conf && exec httpd -DFOREGROUND -f /tmp/httpd.conf`;
console.log("[INFO] stopping leftover WebDAV container if present");
await stopWebDAV().catch(() => {});
console.log("[INFO] starting Apache WebDAV test container");
await dockerOrFail(
"run",
"-d",
"--name",
WEBDAV_CONTAINER,
"-p",
`${hostPort}:80`,
"--tmpfs",
"/usr/local/apache2/htdocs/dav:mode=0777",
"--tmpfs",
"/usr/local/apache2/var:mode=0777",
"--entrypoint",
"/bin/sh",
WEBDAV_IMAGE,
"-c",
startCommand
);
trackContainer(WEBDAV_CONTAINER);
await waitForWebDAV(endpoint);
}
function directWebDAVObjectUrl(collectionEndpoint: string, key: string): string {
return `${collectionEndpoint.replace(/\/+$/u, "")}/${encodeURIComponent(key)}`;
}
function webDAVRequestHeaders(): HeadersInit {
const username = Deno.env.get("WEBDAV_USERNAME") ?? "";
const password = Deno.env.get("WEBDAV_PASSWORD") ?? "";
if (!username && !password) return {};
return { Authorization: `Basic ${btoa(`${username}:${password}`)}` };
}
export async function listWebDAVObjectKeys(collectionEndpoint: string): Promise<string[]> {
const collectionUrl = new URL(`${collectionEndpoint.replace(/\/+$/u, "")}/`);
const response = await fetch(collectionUrl, {
method: "PROPFIND",
headers: { ...webDAVRequestHeaders(), Depth: "1" },
});
if (response.status !== 207) {
throw new Error(`Could not list WebDAV objects: HTTP ${response.status}`);
}
const xml = await response.text();
const hrefs = [
...xml.matchAll(/<(?:[A-Za-z_][\w.-]*:)?href\b[^>]*>([\s\S]*?)<\/(?:[A-Za-z_][\w.-]*:)?href>/giu),
].map((match) => match[1].trim());
const basePath = decodeURIComponent(collectionUrl.pathname);
const keys = new Set<string>();
for (const href of hrefs) {
const path = decodeURIComponent(new URL(href, collectionUrl).pathname);
if (!path.startsWith(basePath)) continue;
const key = path.slice(basePath.length).replace(/\/$/u, "");
if (key && !key.includes("/")) keys.add(key);
}
return [...keys].sort();
}
export async function readWebDAVObjectText(collectionEndpoint: string, key: string): Promise<string> {
const response = await fetch(directWebDAVObjectUrl(collectionEndpoint, key), { headers: webDAVRequestHeaders() });
if (!response.ok) {
throw new Error(`Could not read WebDAV object ${key}: HTTP ${response.status}`);
}
return await response.text();
}
// ---------------------------------------------------------------------------
// P2P relay (strfry)
// ---------------------------------------------------------------------------
+42 -23
View File
@@ -13,7 +13,12 @@ export async function initSettingsFile(settingsFile: string): Promise<void> {
* Generate a full setup URI from a settings file via the Commonlib package API.
* Mirrors the bash flow in test-setup-put-cat-linux.sh.
*/
export async function generateSetupUriFromSettings(settingsFile: string, setupPassphrase: string): Promise<string> {
export async function generateSetupUriFromSettings(
settingsFile: string,
setupPassphrase: string,
preserveSettings = false,
runtimeVaultPassphrase?: string
): Promise<string> {
const script = [
"import { fs } from '@vrtmrz/livesync-commonlib/node';",
"import { encodeSettingsToSetupURI } from '@vrtmrz/livesync-commonlib/compat/API/processSetting';",
@@ -21,13 +26,18 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
" const settingsPath = process.env.SETTINGS_FILE;",
" const passphrase = process.env.SETUP_PASSPHRASE;",
" const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));",
" settings.couchDB_DBNAME = 'setup-put-cat-db';",
" settings.couchDB_URI = 'http://127.0.0.1:5999';",
" settings.couchDB_USER = 'dummy';",
" settings.couchDB_PASSWORD = 'dummy';",
" settings.liveSync = false;",
" settings.syncOnStart = false;",
" settings.syncOnSave = false;",
" if (process.env.RUNTIME_VAULT_PASSPHRASE !== undefined) {",
" settings.passphrase = process.env.RUNTIME_VAULT_PASSPHRASE;",
" }",
" if (process.env.PRESERVE_SETTINGS !== 'true') {",
" settings.couchDB_DBNAME = 'setup-put-cat-db';",
" settings.couchDB_URI = 'http://127.0.0.1:5999';",
" settings.couchDB_USER = 'dummy';",
" settings.couchDB_PASSWORD = 'dummy';",
" settings.liveSync = false;",
" settings.syncOnStart = false;",
" settings.syncOnSave = false;",
" }",
" const uri = await encodeSettingsToSetupURI(settings, passphrase);",
" process.stdout.write(uri.trim());",
"})();",
@@ -41,13 +51,18 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
await Deno.writeTextFile(scriptPath, script);
try {
const env: Record<string, string> = {
SETTINGS_FILE: settingsFile,
SETUP_PASSPHRASE: setupPassphrase,
PRESERVE_SETTINGS: preserveSettings ? "true" : "false",
};
if (runtimeVaultPassphrase !== undefined) {
env.RUNTIME_VAULT_PASSPHRASE = runtimeVaultPassphrase;
}
const cmd = new Deno.Command("npx", {
args: ["tsx", scriptPath],
cwd: CLI_DIR,
env: {
SETTINGS_FILE: settingsFile,
SETUP_PASSPHRASE: setupPassphrase,
},
env,
stdin: "null",
stdout: "piped",
stderr: "piped",
@@ -112,7 +127,7 @@ export async function applyCouchdbSettings(
export async function applyRemoteSyncSettings(
settingsFile: string,
options: {
remoteType: "COUCHDB" | "MINIO";
remoteType: "COUCHDB" | "MINIO" | "WEBDAV";
couchdbUri?: string;
couchdbUser?: string;
couchdbPassword?: string;
@@ -121,6 +136,7 @@ export async function applyRemoteSyncSettings(
minioEndpoint?: string;
minioAccessKey?: string;
minioSecretKey?: string;
webDAVConnectionURI?: string;
encrypt?: boolean;
passphrase?: string;
enableCompression?: boolean;
@@ -138,7 +154,7 @@ export async function applyRemoteSyncSettings(
data.couchDB_USER = options.couchdbUser;
data.couchDB_PASSWORD = options.couchdbPassword;
data.couchDB_DBNAME = options.couchdbDbname;
} else {
} else if (options.remoteType === "MINIO") {
data.remoteType = "MINIO";
data.bucket = options.minioBucket;
data.endpoint = options.minioEndpoint;
@@ -146,15 +162,18 @@ export async function applyRemoteSyncSettings(
data.secretKey = options.minioSecretKey;
data.region = "auto";
data.forcePathStyle = true;
if (options.journalFormat !== undefined) {
data.journalFormat = options.journalFormat;
}
if (options.expectedRepositoryId !== undefined) {
data.expectedRepositoryId = options.expectedRepositoryId;
}
if (options.packReadPolicy !== undefined) {
data.packReadPolicy = options.packReadPolicy;
}
} else {
data.remoteType = "WEBDAV";
data.webDAVactiveConnectionURI = options.webDAVConnectionURI;
}
if (options.journalFormat !== undefined) {
data.journalFormat = options.journalFormat;
}
if (options.expectedRepositoryId !== undefined) {
data.expectedRepositoryId = options.expectedRepositoryId;
}
if (options.packReadPolicy !== undefined) {
data.packReadPolicy = options.packReadPolicy;
}
data.liveSync = true;
+1
View File
@@ -12,6 +12,7 @@ const TASKS = [
"test:e2e-matrix:minio-enc0",
"test:e2e-matrix:minio-enc1",
"test:adaptive-journal-s3",
"test:adaptive-journal-webdav",
] as const;
for (const [index, task] of TASKS.entries()) {
@@ -0,0 +1,215 @@
import { assert, assertEquals } from "@std/assert";
import { TempDir } from "./helpers/temp.ts";
import { assertFilesEqual, runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
import { applyRemoteSyncSettings, generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts";
import { listWebDAVObjectKeys, readWebDAVObjectText, startWebDAV, stopWebDAV } from "./helpers/docker.ts";
const EXTERNAL_PACK_TEST_BYTES = 9 * 1024 * 1024;
function deterministicBytes(length: number, seed: number): Uint8Array {
const bytes = new Uint8Array(length);
let state = seed;
for (let index = 0; index < bytes.byteLength; index += 1) {
state ^= state << 13;
state ^= state >>> 17;
state ^= state << 5;
bytes[index] = state & 0xff;
}
return bytes;
}
function webDAVConnectionURI(endpoint: string, prefix: string): string {
const endpointUrl = new URL(endpoint);
const proxyUrl = new URL(`https://${endpointUrl.host}${endpointUrl.pathname}`);
const username = Deno.env.get("WEBDAV_USERNAME") ?? "";
const password = Deno.env.get("WEBDAV_PASSWORD") ?? "";
proxyUrl.username = username;
proxyUrl.password = password;
if (endpointUrl.protocol === "http:") proxyUrl.searchParams.set("insecure", "true");
proxyUrl.searchParams.set("prefix", prefix);
return `sls+webdav:${proxyUrl.toString().slice("https:".length)}`;
}
function setPackReadPolicy(connectionURI: string, policy: "range" | "whole-pack"): string {
const url = new URL(connectionURI);
url.searchParams.set("packReadPolicy", policy);
return url.toString();
}
function selectAdaptiveJournal(connectionURI: string): string {
const url = new URL(connectionURI);
url.searchParams.set("journalFormat", "adaptive-v1");
return url.toString();
}
function remoteIdFromListing(listing: string): string {
const line = listing
.split(/\r?\n/u)
.find((candidate) => candidate.includes("\tWebDAV Remote\t") || candidate.includes("\tWebDAV "));
const id = line?.split("\t", 1)[0];
if (!id) throw new Error(`WebDAV remote profile was not listed:\n${listing}`);
return id;
}
Deno.test("e2e: two CLI vaults synchronise through Adaptive Journal WebDAV", async () => {
const suffix = `${Date.now()}-${Math.floor(Math.random() * 100000)}`;
const endpoint = (Deno.env.get("WEBDAV_ENDPOINT") ?? "http://127.0.0.1:8088/dav").replace(/\/+$/u, "");
const prefix = `adaptive-cli-${suffix}/`;
const collectionEndpoint = `${endpoint}/${prefix}`;
const connectionURI = webDAVConnectionURI(endpoint, prefix);
const adaptiveConnectionURI = selectAdaptiveJournal(connectionURI);
const vaultPassphrase = "adaptive-journal-webdav-cli-e2ee";
const setupPassphrase = "adaptive-journal-webdav-cli-setup";
await using workDir = await TempDir.create("livesync-cli-adaptive-journal-webdav");
const vaultA = workDir.join("vault-a");
const vaultB = workDir.join("vault-b");
const settingsA = workDir.join("settings-a.json");
const settingsB = workDir.join("settings-b.json");
const binarySourceA = workDir.join("source-a.bin");
const binarySourceB = workDir.join("source-b.bin");
const binaryDestinationA = workDir.join("destination-a.bin");
const binaryDestinationB = workDir.join("destination-b.bin");
await Deno.mkdir(vaultA, { recursive: true });
await Deno.mkdir(vaultB, { recursive: true });
const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0";
const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1";
if (shouldStartDocker) await startWebDAV(endpoint);
try {
await initSettingsFile(settingsA);
await applyRemoteSyncSettings(settingsA, {
remoteType: "WEBDAV",
webDAVConnectionURI: connectionURI,
encrypt: true,
passphrase: vaultPassphrase,
enableCompression: false,
journalFormat: "adaptive-v1",
packReadPolicy: "whole-pack",
});
const addedRemote = await runCliOrFail(
vaultA,
"--settings",
settingsA,
"remote-add",
"WebDAV E2E",
adaptiveConnectionURI
);
const remoteId = addedRemote.trim().split("\t", 1)[0];
assert(remoteId, `remote-add did not return a profile ID:\n${addedRemote}`);
const settingsAfterRemoteAdd = JSON.parse(await Deno.readTextFile(settingsA)) as {
remoteConfigurations?: Record<string, unknown>;
liveSync?: boolean;
};
assert(
settingsAfterRemoteAdd.remoteConfigurations?.[remoteId],
`remote-add did not persist profile ${remoteId}: ${Object.keys(settingsAfterRemoteAdd.remoteConfigurations ?? {}).join(", ")}`
);
assertEquals(settingsAfterRemoteAdd.liveSync, true, "remote-add changed the persisted synchronisation mode");
await runCliOrFail(vaultA, "--settings", settingsA, "remote-activate", remoteId);
const textPath = "adaptive/text.md";
const binaryPath = "adaptive/data.bin";
await runCliWithInputOrFail(`created-by-a-${suffix}\n`, vaultA, "--settings", settingsA, "put", textPath);
await Deno.writeFile(binarySourceA, deterministicBytes(EXTERNAL_PACK_TEST_BYTES, 0x1a2b3c4d));
await runCliOrFail(vaultA, "--settings", settingsA, "push", binarySourceA, binaryPath);
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
const remoteListing = await runCliOrFail(vaultA, "--settings", settingsA, "remote-ls");
assertEquals(remoteIdFromListing(remoteListing), remoteId);
assert(
remoteListing
.split(/\r?\n/u)
.some((line) => line.startsWith(`${remoteId}\t`) && line.includes("\tactive\t")),
`Activated WebDAV profile was not listed as active:\n${remoteListing}`
);
const exportedConnection = (
await runCliOrFail(vaultA, "--settings", settingsA, "remote-export", remoteId)
).trim();
assert(exportedConnection.startsWith("sls+webdav://"));
assert(exportedConnection.includes("journalFormat=adaptive-v1"));
assert(!exportedConnection.includes("packReadPolicy="));
const setupURI = await generateSetupUriFromSettings(settingsA, setupPassphrase, true, vaultPassphrase);
await initSettingsFile(settingsB);
await runCliWithInputOrFail(`${setupPassphrase}\n`, vaultB, "--settings", settingsB, "setup", setupURI);
const settingsAfterSetup = JSON.parse(await Deno.readTextFile(settingsB)) as {
encryptedPassphrase?: string;
};
assert(
typeof settingsAfterSetup.encryptedPassphrase === "string" &&
settingsAfterSetup.encryptedPassphrase.length > 0,
"setup did not persist the encrypted Vault passphrase"
);
await runCliOrFail(
vaultB,
"--settings",
settingsB,
"remote-set",
remoteId,
setPackReadPolicy(exportedConnection, "range")
);
const rangeConnection = (await runCliOrFail(vaultB, "--settings", settingsB, "remote-export", remoteId)).trim();
assert(rangeConnection.includes("packReadPolicy=range"));
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
assertEquals(
sanitiseCatStdout(await runCliOrFail(vaultB, "--settings", settingsB, "cat", textPath)).trimEnd(),
`created-by-a-${suffix}`
);
await runCliOrFail(vaultB, "--settings", settingsB, "pull", binaryPath, binaryDestinationB);
await assertFilesEqual(binarySourceA, binaryDestinationB, "Adaptive Journal Range transfer differs");
await runCliWithInputOrFail(`updated-by-b-${suffix}\n`, vaultB, "--settings", settingsB, "put", textPath);
await Deno.writeFile(binarySourceB, deterministicBytes(EXTERNAL_PACK_TEST_BYTES, 0x5e6f7788));
await runCliOrFail(vaultB, "--settings", settingsB, "push", binarySourceB, binaryPath);
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
assertEquals(
sanitiseCatStdout(await runCliOrFail(vaultA, "--settings", settingsA, "cat", textPath)).trimEnd(),
`updated-by-b-${suffix}`
);
await runCliOrFail(vaultA, "--settings", settingsA, "pull", binaryPath, binaryDestinationA);
await assertFilesEqual(binarySourceB, binaryDestinationA, "Adaptive Journal whole-Pack transfer differs");
await runCliOrFail(vaultA, "--settings", settingsA, "rm", binaryPath);
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
const deleted = await runCli(vaultB, "--settings", settingsB, "cat", binaryPath);
assert(deleted.code !== 0, `Deleted binary remained readable:\n${deleted.combined}`);
const objectKeys = await listWebDAVObjectKeys(collectionEndpoint);
assert(objectKeys.includes("a1~manifest.json"), `Adaptive manifest is missing:\n${objectKeys.join("\n")}`);
for (const objectPrefix of ["a1~writer~", "a1~pack~", "a1~commit~"]) {
assert(
objectKeys.some((key) => key.startsWith(objectPrefix)),
`Adaptive object with prefix ${objectPrefix} is missing:\n${objectKeys.join("\n")}`
);
}
const packKeys = objectKeys.filter((key) => key.startsWith("a1~pack~"));
assert(packKeys.length >= 2, `Expected external Packs from both CLI writers:\n${objectKeys.join("\n")}`);
for (const legacyPrefix of ["a1~index~", "a1~delta~", "a1~metadata~"]) {
assert(
!objectKeys.some((key) => key.startsWith(legacyPrefix)),
`Legacy Adaptive object with prefix ${legacyPrefix} was written:\n${objectKeys.join("\n")}`
);
}
const manifest = JSON.parse(await readWebDAVObjectText(collectionEndpoint, "a1~manifest.json")) as {
objectLayout?: unknown;
};
assertEquals(manifest.objectLayout, "commit-bundle-v1");
assert(
!objectKeys.some((key) => key.startsWith("a1~probe~")),
`Adaptive capability probe objects were not removed:\n${objectKeys.join("\n")}`
);
assert(
!objectKeys.includes("_00000000-milestone.json"),
`Legacy Journal milestone was written into the Adaptive repository:\n${objectKeys.join("\n")}`
);
} finally {
if (shouldStartDocker && !keepDocker) {
await stopWebDAV().catch(() => {});
}
}
});
+3 -3
View File
@@ -1,5 +1,5 @@
import { fireAndForget } from "octagonal-wheels/promises";
import { REMOTE_MINIO, REMOTE_P2P, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { isJournalRemoteType, REMOTE_P2P, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import type { LiveSyncAbstractReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/LiveSyncAbstractReplicator";
import { AbstractModule } from "@/modules/AbstractModule";
@@ -9,7 +9,7 @@ export class ModuleReplicatorCouchDB extends AbstractModule {
_anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator | false> {
const settings = { ...this.settings, ...settingOverride };
// If new remote types were added, add them here. Do not use `REMOTE_COUCHDB` directly for the safety valve.
if (settings.remoteType == REMOTE_MINIO || settings.remoteType == REMOTE_P2P) {
if (isJournalRemoteType(settings.remoteType) || settings.remoteType == REMOTE_P2P) {
return Promise.resolve(false);
}
return Promise.resolve(new LiveSyncCouchDBReplicator(this.core));
@@ -17,7 +17,7 @@ export class ModuleReplicatorCouchDB extends AbstractModule {
_everyAfterResumeProcess(): Promise<boolean> {
if (this.services.appLifecycle.isSuspended()) return Promise.resolve(true);
if (!this.services.appLifecycle.isReady()) return Promise.resolve(true);
if (this.settings.remoteType != REMOTE_MINIO && this.settings.remoteType != REMOTE_P2P) {
if (!isJournalRemoteType(this.settings.remoteType) && this.settings.remoteType != REMOTE_P2P) {
const LiveSyncEnabled = this.settings.liveSync;
const continuous = LiveSyncEnabled;
const eventualOnStart = !LiveSyncEnabled && this.settings.syncOnStart;
@@ -1,7 +1,11 @@
import { describe, expect, it, vi } from "vitest";
import { REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { ModuleReplicatorCouchDB } from "./ModuleReplicatorCouchDB.ts";
function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isReplicationReady = true) {
function createModule(
settings: { liveSync: boolean; remoteType?: string; syncOnStart: boolean },
isReplicationReady = true
) {
const openReplication = vi.fn(async () => true);
const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
const services = {
@@ -86,4 +90,19 @@ describe("ModuleReplicatorCouchDB resume replication activity", () => {
expect(runFiniteReplicationActivity).not.toHaveBeenCalled();
expect(openReplication).not.toHaveBeenCalled();
});
it("does not claim or resume WebDAV as CouchDB", async () => {
const { module, openReplication, runFiniteReplicationActivity } = createModule({
liveSync: true,
remoteType: REMOTE_WEBDAV,
syncOnStart: true,
});
await expect(module._anyNewReplicator()).resolves.toBe(false);
await module._everyAfterResumeProcess();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(runFiniteReplicationActivity).not.toHaveBeenCalled();
expect(openReplication).not.toHaveBeenCalled();
});
});
@@ -1,13 +1,13 @@
import { REMOTE_MINIO, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { isJournalRemoteType, type RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
import type { LiveSyncAbstractReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/LiveSyncAbstractReplicator";
import type { LiveSyncCore } from "@/main";
import { AbstractModule } from "@/modules/AbstractModule";
export class ModuleReplicatorMinIO extends AbstractModule {
export class ModuleReplicatorJournal extends AbstractModule {
_anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator | false> {
const settings = { ...this.settings, ...settingOverride };
if (settings.remoteType == REMOTE_MINIO) {
if (isJournalRemoteType(settings.remoteType)) {
return Promise.resolve(new LiveSyncJournalReplicator(this.core));
}
return Promise.resolve(false);
@@ -0,0 +1,47 @@
import { describe, expect, it, vi } from "vitest";
import { REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_WEBDAV } from "@vrtmrz/livesync-commonlib/compat/common/types";
const journalReplicatorConstructor = vi.hoisted(() => vi.fn());
vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator", () => ({
LiveSyncJournalReplicator: class {
constructor(env: unknown) {
journalReplicatorConstructor(env);
}
},
}));
import { ModuleReplicatorJournal } from "./ModuleReplicatorJournal.ts";
function createModule(remoteType: string) {
const services = {
API: {
addCommand: vi.fn(),
addLog: vi.fn(),
addRibbonIcon: vi.fn(),
registerProtocolHandler: vi.fn(),
registerWindow: vi.fn(),
},
};
const core = {
_services: services,
services,
settings: { remoteType },
} as any;
return { core, module: new ModuleReplicatorJournal(core) };
}
describe("ModuleReplicatorJournal", () => {
it.each([REMOTE_MINIO, REMOTE_WEBDAV])("creates a Journal replicator for %s", async (remoteType) => {
const { core, module } = createModule(remoteType);
await expect(module._anyNewReplicator()).resolves.not.toBe(false);
expect(journalReplicatorConstructor).toHaveBeenLastCalledWith(core);
});
it("does not claim a CouchDB remote", async () => {
const { module } = createModule(REMOTE_COUCHDB);
await expect(module._anyNewReplicator()).resolves.toBe(false);
});
});
@@ -83,7 +83,10 @@ function suggestRemoteConfigurationName(parsed: RemoteConfigurationResult): stri
if (parsed.type === "s3") {
return `S3 ${parsed.settings.bucket || parsed.settings.endpoint}`;
}
return `P2P ${parsed.settings.P2P_roomID || "Remote"}`;
if (parsed.type === "p2p") {
return `P2P ${parsed.settings.P2P_roomID || "Remote"}`;
}
return "WebDAV Remote";
}
export function paneRemoteConfig(