mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-08-11 22:25:46 +00:00
feat(cli): support adaptive WebDAV remotes
This commit is contained in:
vorotamoroz
@@ -45,6 +45,7 @@
|
||||
"test:e2e:cli": "npm run test:e2e:ci --workspace self-hosted-livesync-cli",
|
||||
"test:e2e:cli:p2p": "npm run test:e2e:p2p --workspace self-hosted-livesync-cli",
|
||||
"test:e2e:cli:adaptive-s3": "npm run test:e2e:adaptive-s3 --workspace self-hosted-livesync-cli",
|
||||
"test:e2e:cli:adaptive-webdav": "npm run test:e2e:adaptive-webdav --workspace self-hosted-livesync-cli",
|
||||
"test:e2e:cli:all": "npm run test:e2e:all --workspace self-hosted-livesync-cli",
|
||||
"test:integration": "npx dotenv-cli -e .env -e .test.env -- vitest run --config vitest.config.integration.ts",
|
||||
"test:unit:coverage": "vitest run --config vitest.config.unit.ts --coverage",
|
||||
|
||||
+26
-3
@@ -37,6 +37,14 @@ defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
|
||||
/** Injectable command boundary used by CLI integration probes. */
|
||||
export type CliCommandRunner = (options: CLIOptions, context: CLICommandContext) => Promise<boolean>;
|
||||
|
||||
const SETTINGS_MANAGEMENT_COMMANDS: ReadonlySet<CLICommand> = new Set([
|
||||
"setup",
|
||||
"remote-add",
|
||||
"remote-rm",
|
||||
"remote-set",
|
||||
"remote-activate",
|
||||
]);
|
||||
|
||||
function printHelp(standardIo: StandardIo): void {
|
||||
writeStdoutLine(
|
||||
standardIo,
|
||||
@@ -274,7 +282,10 @@ export async function main(
|
||||
) {
|
||||
const options = parseArgs(standardIo);
|
||||
if (options.interval && options.command !== "daemon") {
|
||||
writeStderrLine(standardIo, `Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
`Warning: --interval is only used in daemon mode, ignored for '${options.command}'`
|
||||
);
|
||||
}
|
||||
const avoidStdoutNoise =
|
||||
options.command === "cat" ||
|
||||
@@ -404,7 +415,10 @@ export async function main(
|
||||
// In daemon mode the default handler must run so changes are applied to the filesystem.
|
||||
if (options.command !== "daemon") {
|
||||
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
|
||||
writeStderrLine(standardIo, `[Info] Replication result received, but not processed automatically in CLI mode.`);
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
`[Info] Replication result received, but not processed automatically in CLI mode.`
|
||||
);
|
||||
return await Promise.resolve(true);
|
||||
}, -100);
|
||||
}
|
||||
@@ -506,7 +520,7 @@ export async function main(
|
||||
// Save the settings file before any lifecycle events can mutate and persist them.
|
||||
// suspendAllSync and other lifecycle hooks clobber sync settings in memory, and
|
||||
// various code paths persist the clobbered state to disk. We restore on shutdown.
|
||||
const settingsBackup = await fs.readFile(settingsPath, "utf-8").catch(() => null!);
|
||||
let settingsBackup: string | null = await fs.readFile(settingsPath, "utf-8").catch(() => null);
|
||||
|
||||
// Restore settings file on any exit to undo lifecycle mutations.
|
||||
// Write to a temp path first so a crash mid-write doesn't leave a truncated file.
|
||||
@@ -576,6 +590,15 @@ export async function main(
|
||||
settingsPath,
|
||||
originalSyncSettings,
|
||||
});
|
||||
if (result && SETTINGS_MANAGEMENT_COMMANDS.has(options.command)) {
|
||||
// Settings management is intentional, unlike the temporary changes made by suspendAllSync().
|
||||
// setup replaces the complete configuration, while remote profile commands must retain the
|
||||
// synchronisation mode which was active before the command lifecycle suspended it.
|
||||
if (options.command !== "setup") {
|
||||
await core.services.setting.applyPartial(originalSyncSettings, true);
|
||||
}
|
||||
settingsBackup = await fs.readFile(settingsPath, "utf-8");
|
||||
}
|
||||
if (!result) {
|
||||
writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`);
|
||||
process.exitCode = 1;
|
||||
|
||||
@@ -24,6 +24,8 @@
|
||||
"test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose",
|
||||
"pretest:e2e:adaptive-s3": "npm run build",
|
||||
"test:e2e:adaptive-s3": "deno task --cwd testdeno test:adaptive-journal-s3",
|
||||
"pretest:e2e:adaptive-webdav": "npm run build",
|
||||
"test:e2e:adaptive-webdav": "deno task --cwd testdeno test:adaptive-journal-webdav",
|
||||
"test:e2e:mirror": "bash test/test-mirror-linux.sh",
|
||||
"test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh",
|
||||
"pretest:e2e:all": "npm run build",
|
||||
|
||||
@@ -35,7 +35,8 @@
|
||||
"test:e2e-matrix:couchdb-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc1' test-e2e-two-vaults-matrix.ts",
|
||||
"test:e2e-matrix:minio-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc0' test-e2e-two-vaults-matrix.ts",
|
||||
"test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts",
|
||||
"test:adaptive-journal-s3": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-s3.ts"
|
||||
"test:adaptive-journal-s3": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-s3.ts",
|
||||
"test:adaptive-journal-webdav": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-webdav.ts"
|
||||
},
|
||||
"imports": {
|
||||
"@std/assert": "jsr:@std/assert@^1.0.13",
|
||||
|
||||
@@ -23,6 +23,7 @@ export interface CliResult {
|
||||
export const TEE_ENABLED = Deno.env.get("LIVESYNC_TEST_TEE") === "1";
|
||||
const VERBOSE_ENABLED = Deno.env.get("LIVESYNC_CLI_VERBOSE") === "1";
|
||||
const DEBUG_ENABLED = Deno.env.get("LIVESYNC_CLI_DEBUG") === "1";
|
||||
const SETUP_URI_PREFIX = "obsidian://setuplivesync?settings=";
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -40,7 +41,13 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array {
|
||||
}
|
||||
|
||||
export function formatTeeCommand(args: string[]): string {
|
||||
return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" ");
|
||||
const redactArgument = (argument: string): string => {
|
||||
if (argument.startsWith(SETUP_URI_PREFIX)) {
|
||||
return `${SETUP_URI_PREFIX}<redacted>`;
|
||||
}
|
||||
return argument.replace(/^(sls\+[^:]+:\/\/)[^/?#@]*@/u, "$1<redacted>@");
|
||||
};
|
||||
return ["node", CLI_DIST, ...args.map(redactArgument)].map((part) => JSON.stringify(part)).join(" ");
|
||||
}
|
||||
|
||||
export function createLineTeeWriter(
|
||||
|
||||
@@ -330,6 +330,39 @@ const MINIO_CONTAINER = "minio-test";
|
||||
const MINIO_IMAGE = "minio/minio:RELEASE.2025-04-22T22-12-26Z";
|
||||
const MINIO_MC_IMAGE = "minio/mc:RELEASE.2025-04-16T18-13-26Z";
|
||||
|
||||
const WEBDAV_CONTAINER = "webdav-test";
|
||||
const WEBDAV_IMAGE = "httpd:2.4.68";
|
||||
const WEBDAV_HTTPD_CONFIG = `ServerRoot "/usr/local/apache2"
|
||||
Listen 80
|
||||
|
||||
LoadModule mpm_event_module modules/mod_mpm_event.so
|
||||
LoadModule authn_core_module modules/mod_authn_core.so
|
||||
LoadModule authz_core_module modules/mod_authz_core.so
|
||||
LoadModule dav_module modules/mod_dav.so
|
||||
LoadModule dav_fs_module modules/mod_dav_fs.so
|
||||
LoadModule unixd_module modules/mod_unixd.so
|
||||
|
||||
User www-data
|
||||
Group www-data
|
||||
ServerName localhost
|
||||
DocumentRoot "/usr/local/apache2/htdocs"
|
||||
PidFile "/tmp/httpd.pid"
|
||||
ErrorLog "/proc/self/fd/2"
|
||||
LogLevel warn
|
||||
|
||||
DavLockDB "/usr/local/apache2/var/DavLock"
|
||||
DavLockDiscovery Off
|
||||
|
||||
<Directory "/usr/local/apache2/htdocs">
|
||||
AllowOverride None
|
||||
Require all granted
|
||||
</Directory>
|
||||
|
||||
<Directory "/usr/local/apache2/htdocs/dav">
|
||||
Dav On
|
||||
</Directory>
|
||||
`;
|
||||
|
||||
export async function stopCouchdb(): Promise<void> {
|
||||
await stopAndRemoveContainer(COUCHDB_CONTAINER);
|
||||
untrackContainer(COUCHDB_CONTAINER);
|
||||
@@ -625,6 +658,113 @@ export async function startMinio(
|
||||
await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WebDAV
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function stopWebDAV(): Promise<void> {
|
||||
await stopAndRemoveContainer(WEBDAV_CONTAINER);
|
||||
untrackContainer(WEBDAV_CONTAINER);
|
||||
}
|
||||
|
||||
async function waitForWebDAV(endpoint: string): Promise<void> {
|
||||
for (let attempt = 0; attempt < 30; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: "PROPFIND",
|
||||
headers: { Depth: "0" },
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (response.status === 207) return;
|
||||
} catch {
|
||||
// The container is still starting.
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
throw new Error(`WebDAV collection did not become ready: ${endpoint}`);
|
||||
}
|
||||
|
||||
export async function startWebDAV(endpoint: string): Promise<void> {
|
||||
const url = new URL(endpoint);
|
||||
if (url.protocol !== "http:" || (url.hostname !== "127.0.0.1" && url.hostname !== "localhost")) {
|
||||
throw new Error(`Managed WebDAV requires a local HTTP endpoint, received: ${endpoint}`);
|
||||
}
|
||||
if (url.pathname.replace(/\/+$/u, "") !== "/dav") {
|
||||
throw new Error(`Managed WebDAV requires the /dav collection, received: ${endpoint}`);
|
||||
}
|
||||
const hostPort = url.port || "80";
|
||||
const encodedConfig = btoa(WEBDAV_HTTPD_CONFIG);
|
||||
const startCommand = `printf '%s' '${encodedConfig}' | base64 -d > /tmp/httpd.conf && exec httpd -DFOREGROUND -f /tmp/httpd.conf`;
|
||||
|
||||
console.log("[INFO] stopping leftover WebDAV container if present");
|
||||
await stopWebDAV().catch(() => {});
|
||||
|
||||
console.log("[INFO] starting Apache WebDAV test container");
|
||||
await dockerOrFail(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
WEBDAV_CONTAINER,
|
||||
"-p",
|
||||
`${hostPort}:80`,
|
||||
"--tmpfs",
|
||||
"/usr/local/apache2/htdocs/dav:mode=0777",
|
||||
"--tmpfs",
|
||||
"/usr/local/apache2/var:mode=0777",
|
||||
"--entrypoint",
|
||||
"/bin/sh",
|
||||
WEBDAV_IMAGE,
|
||||
"-c",
|
||||
startCommand
|
||||
);
|
||||
trackContainer(WEBDAV_CONTAINER);
|
||||
await waitForWebDAV(endpoint);
|
||||
}
|
||||
|
||||
function directWebDAVObjectUrl(collectionEndpoint: string, key: string): string {
|
||||
return `${collectionEndpoint.replace(/\/+$/u, "")}/${encodeURIComponent(key)}`;
|
||||
}
|
||||
|
||||
function webDAVRequestHeaders(): HeadersInit {
|
||||
const username = Deno.env.get("WEBDAV_USERNAME") ?? "";
|
||||
const password = Deno.env.get("WEBDAV_PASSWORD") ?? "";
|
||||
if (!username && !password) return {};
|
||||
return { Authorization: `Basic ${btoa(`${username}:${password}`)}` };
|
||||
}
|
||||
|
||||
export async function listWebDAVObjectKeys(collectionEndpoint: string): Promise<string[]> {
|
||||
const collectionUrl = new URL(`${collectionEndpoint.replace(/\/+$/u, "")}/`);
|
||||
const response = await fetch(collectionUrl, {
|
||||
method: "PROPFIND",
|
||||
headers: { ...webDAVRequestHeaders(), Depth: "1" },
|
||||
});
|
||||
if (response.status !== 207) {
|
||||
throw new Error(`Could not list WebDAV objects: HTTP ${response.status}`);
|
||||
}
|
||||
const xml = await response.text();
|
||||
const hrefs = [
|
||||
...xml.matchAll(/<(?:[A-Za-z_][\w.-]*:)?href\b[^>]*>([\s\S]*?)<\/(?:[A-Za-z_][\w.-]*:)?href>/giu),
|
||||
].map((match) => match[1].trim());
|
||||
const basePath = decodeURIComponent(collectionUrl.pathname);
|
||||
const keys = new Set<string>();
|
||||
for (const href of hrefs) {
|
||||
const path = decodeURIComponent(new URL(href, collectionUrl).pathname);
|
||||
if (!path.startsWith(basePath)) continue;
|
||||
const key = path.slice(basePath.length).replace(/\/$/u, "");
|
||||
if (key && !key.includes("/")) keys.add(key);
|
||||
}
|
||||
return [...keys].sort();
|
||||
}
|
||||
|
||||
export async function readWebDAVObjectText(collectionEndpoint: string, key: string): Promise<string> {
|
||||
const response = await fetch(directWebDAVObjectUrl(collectionEndpoint, key), { headers: webDAVRequestHeaders() });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Could not read WebDAV object ${key}: HTTP ${response.status}`);
|
||||
}
|
||||
return await response.text();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P2P relay (strfry)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -13,7 +13,12 @@ export async function initSettingsFile(settingsFile: string): Promise<void> {
|
||||
* Generate a full setup URI from a settings file via the Commonlib package API.
|
||||
* Mirrors the bash flow in test-setup-put-cat-linux.sh.
|
||||
*/
|
||||
export async function generateSetupUriFromSettings(settingsFile: string, setupPassphrase: string): Promise<string> {
|
||||
export async function generateSetupUriFromSettings(
|
||||
settingsFile: string,
|
||||
setupPassphrase: string,
|
||||
preserveSettings = false,
|
||||
runtimeVaultPassphrase?: string
|
||||
): Promise<string> {
|
||||
const script = [
|
||||
"import { fs } from '@vrtmrz/livesync-commonlib/node';",
|
||||
"import { encodeSettingsToSetupURI } from '@vrtmrz/livesync-commonlib/compat/API/processSetting';",
|
||||
@@ -21,13 +26,18 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
|
||||
" const settingsPath = process.env.SETTINGS_FILE;",
|
||||
" const passphrase = process.env.SETUP_PASSPHRASE;",
|
||||
" const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));",
|
||||
" settings.couchDB_DBNAME = 'setup-put-cat-db';",
|
||||
" settings.couchDB_URI = 'http://127.0.0.1:5999';",
|
||||
" settings.couchDB_USER = 'dummy';",
|
||||
" settings.couchDB_PASSWORD = 'dummy';",
|
||||
" settings.liveSync = false;",
|
||||
" settings.syncOnStart = false;",
|
||||
" settings.syncOnSave = false;",
|
||||
" if (process.env.RUNTIME_VAULT_PASSPHRASE !== undefined) {",
|
||||
" settings.passphrase = process.env.RUNTIME_VAULT_PASSPHRASE;",
|
||||
" }",
|
||||
" if (process.env.PRESERVE_SETTINGS !== 'true') {",
|
||||
" settings.couchDB_DBNAME = 'setup-put-cat-db';",
|
||||
" settings.couchDB_URI = 'http://127.0.0.1:5999';",
|
||||
" settings.couchDB_USER = 'dummy';",
|
||||
" settings.couchDB_PASSWORD = 'dummy';",
|
||||
" settings.liveSync = false;",
|
||||
" settings.syncOnStart = false;",
|
||||
" settings.syncOnSave = false;",
|
||||
" }",
|
||||
" const uri = await encodeSettingsToSetupURI(settings, passphrase);",
|
||||
" process.stdout.write(uri.trim());",
|
||||
"})();",
|
||||
@@ -41,13 +51,18 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
|
||||
await Deno.writeTextFile(scriptPath, script);
|
||||
|
||||
try {
|
||||
const env: Record<string, string> = {
|
||||
SETTINGS_FILE: settingsFile,
|
||||
SETUP_PASSPHRASE: setupPassphrase,
|
||||
PRESERVE_SETTINGS: preserveSettings ? "true" : "false",
|
||||
};
|
||||
if (runtimeVaultPassphrase !== undefined) {
|
||||
env.RUNTIME_VAULT_PASSPHRASE = runtimeVaultPassphrase;
|
||||
}
|
||||
const cmd = new Deno.Command("npx", {
|
||||
args: ["tsx", scriptPath],
|
||||
cwd: CLI_DIR,
|
||||
env: {
|
||||
SETTINGS_FILE: settingsFile,
|
||||
SETUP_PASSPHRASE: setupPassphrase,
|
||||
},
|
||||
env,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
@@ -112,7 +127,7 @@ export async function applyCouchdbSettings(
|
||||
export async function applyRemoteSyncSettings(
|
||||
settingsFile: string,
|
||||
options: {
|
||||
remoteType: "COUCHDB" | "MINIO";
|
||||
remoteType: "COUCHDB" | "MINIO" | "WEBDAV";
|
||||
couchdbUri?: string;
|
||||
couchdbUser?: string;
|
||||
couchdbPassword?: string;
|
||||
@@ -121,6 +136,7 @@ export async function applyRemoteSyncSettings(
|
||||
minioEndpoint?: string;
|
||||
minioAccessKey?: string;
|
||||
minioSecretKey?: string;
|
||||
webDAVConnectionURI?: string;
|
||||
encrypt?: boolean;
|
||||
passphrase?: string;
|
||||
enableCompression?: boolean;
|
||||
@@ -138,7 +154,7 @@ export async function applyRemoteSyncSettings(
|
||||
data.couchDB_USER = options.couchdbUser;
|
||||
data.couchDB_PASSWORD = options.couchdbPassword;
|
||||
data.couchDB_DBNAME = options.couchdbDbname;
|
||||
} else {
|
||||
} else if (options.remoteType === "MINIO") {
|
||||
data.remoteType = "MINIO";
|
||||
data.bucket = options.minioBucket;
|
||||
data.endpoint = options.minioEndpoint;
|
||||
@@ -146,15 +162,18 @@ export async function applyRemoteSyncSettings(
|
||||
data.secretKey = options.minioSecretKey;
|
||||
data.region = "auto";
|
||||
data.forcePathStyle = true;
|
||||
if (options.journalFormat !== undefined) {
|
||||
data.journalFormat = options.journalFormat;
|
||||
}
|
||||
if (options.expectedRepositoryId !== undefined) {
|
||||
data.expectedRepositoryId = options.expectedRepositoryId;
|
||||
}
|
||||
if (options.packReadPolicy !== undefined) {
|
||||
data.packReadPolicy = options.packReadPolicy;
|
||||
}
|
||||
} else {
|
||||
data.remoteType = "WEBDAV";
|
||||
data.webDAVactiveConnectionURI = options.webDAVConnectionURI;
|
||||
}
|
||||
if (options.journalFormat !== undefined) {
|
||||
data.journalFormat = options.journalFormat;
|
||||
}
|
||||
if (options.expectedRepositoryId !== undefined) {
|
||||
data.expectedRepositoryId = options.expectedRepositoryId;
|
||||
}
|
||||
if (options.packReadPolicy !== undefined) {
|
||||
data.packReadPolicy = options.packReadPolicy;
|
||||
}
|
||||
|
||||
data.liveSync = true;
|
||||
|
||||
@@ -12,6 +12,7 @@ const TASKS = [
|
||||
"test:e2e-matrix:minio-enc0",
|
||||
"test:e2e-matrix:minio-enc1",
|
||||
"test:adaptive-journal-s3",
|
||||
"test:adaptive-journal-webdav",
|
||||
] as const;
|
||||
|
||||
for (const [index, task] of TASKS.entries()) {
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { assert, assertEquals } from "@std/assert";
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { assertFilesEqual, runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
|
||||
import { applyRemoteSyncSettings, generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts";
|
||||
import { listWebDAVObjectKeys, readWebDAVObjectText, startWebDAV, stopWebDAV } from "./helpers/docker.ts";
|
||||
|
||||
const EXTERNAL_PACK_TEST_BYTES = 9 * 1024 * 1024;
|
||||
|
||||
function deterministicBytes(length: number, seed: number): Uint8Array {
|
||||
const bytes = new Uint8Array(length);
|
||||
let state = seed;
|
||||
for (let index = 0; index < bytes.byteLength; index += 1) {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
bytes[index] = state & 0xff;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function webDAVConnectionURI(endpoint: string, prefix: string): string {
|
||||
const endpointUrl = new URL(endpoint);
|
||||
const proxyUrl = new URL(`https://${endpointUrl.host}${endpointUrl.pathname}`);
|
||||
const username = Deno.env.get("WEBDAV_USERNAME") ?? "";
|
||||
const password = Deno.env.get("WEBDAV_PASSWORD") ?? "";
|
||||
proxyUrl.username = username;
|
||||
proxyUrl.password = password;
|
||||
if (endpointUrl.protocol === "http:") proxyUrl.searchParams.set("insecure", "true");
|
||||
proxyUrl.searchParams.set("prefix", prefix);
|
||||
return `sls+webdav:${proxyUrl.toString().slice("https:".length)}`;
|
||||
}
|
||||
|
||||
function setPackReadPolicy(connectionURI: string, policy: "range" | "whole-pack"): string {
|
||||
const url = new URL(connectionURI);
|
||||
url.searchParams.set("packReadPolicy", policy);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function selectAdaptiveJournal(connectionURI: string): string {
|
||||
const url = new URL(connectionURI);
|
||||
url.searchParams.set("journalFormat", "adaptive-v1");
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function remoteIdFromListing(listing: string): string {
|
||||
const line = listing
|
||||
.split(/\r?\n/u)
|
||||
.find((candidate) => candidate.includes("\tWebDAV Remote\t") || candidate.includes("\tWebDAV "));
|
||||
const id = line?.split("\t", 1)[0];
|
||||
if (!id) throw new Error(`WebDAV remote profile was not listed:\n${listing}`);
|
||||
return id;
|
||||
}
|
||||
|
||||
Deno.test("e2e: two CLI vaults synchronise through Adaptive Journal WebDAV", async () => {
|
||||
const suffix = `${Date.now()}-${Math.floor(Math.random() * 100000)}`;
|
||||
const endpoint = (Deno.env.get("WEBDAV_ENDPOINT") ?? "http://127.0.0.1:8088/dav").replace(/\/+$/u, "");
|
||||
const prefix = `adaptive-cli-${suffix}/`;
|
||||
const collectionEndpoint = `${endpoint}/${prefix}`;
|
||||
const connectionURI = webDAVConnectionURI(endpoint, prefix);
|
||||
const adaptiveConnectionURI = selectAdaptiveJournal(connectionURI);
|
||||
const vaultPassphrase = "adaptive-journal-webdav-cli-e2ee";
|
||||
const setupPassphrase = "adaptive-journal-webdav-cli-setup";
|
||||
|
||||
await using workDir = await TempDir.create("livesync-cli-adaptive-journal-webdav");
|
||||
const vaultA = workDir.join("vault-a");
|
||||
const vaultB = workDir.join("vault-b");
|
||||
const settingsA = workDir.join("settings-a.json");
|
||||
const settingsB = workDir.join("settings-b.json");
|
||||
const binarySourceA = workDir.join("source-a.bin");
|
||||
const binarySourceB = workDir.join("source-b.bin");
|
||||
const binaryDestinationA = workDir.join("destination-a.bin");
|
||||
const binaryDestinationB = workDir.join("destination-b.bin");
|
||||
await Deno.mkdir(vaultA, { recursive: true });
|
||||
await Deno.mkdir(vaultB, { recursive: true });
|
||||
|
||||
const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0";
|
||||
const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1";
|
||||
if (shouldStartDocker) await startWebDAV(endpoint);
|
||||
|
||||
try {
|
||||
await initSettingsFile(settingsA);
|
||||
await applyRemoteSyncSettings(settingsA, {
|
||||
remoteType: "WEBDAV",
|
||||
webDAVConnectionURI: connectionURI,
|
||||
encrypt: true,
|
||||
passphrase: vaultPassphrase,
|
||||
enableCompression: false,
|
||||
journalFormat: "adaptive-v1",
|
||||
packReadPolicy: "whole-pack",
|
||||
});
|
||||
const addedRemote = await runCliOrFail(
|
||||
vaultA,
|
||||
"--settings",
|
||||
settingsA,
|
||||
"remote-add",
|
||||
"WebDAV E2E",
|
||||
adaptiveConnectionURI
|
||||
);
|
||||
const remoteId = addedRemote.trim().split("\t", 1)[0];
|
||||
assert(remoteId, `remote-add did not return a profile ID:\n${addedRemote}`);
|
||||
const settingsAfterRemoteAdd = JSON.parse(await Deno.readTextFile(settingsA)) as {
|
||||
remoteConfigurations?: Record<string, unknown>;
|
||||
liveSync?: boolean;
|
||||
};
|
||||
assert(
|
||||
settingsAfterRemoteAdd.remoteConfigurations?.[remoteId],
|
||||
`remote-add did not persist profile ${remoteId}: ${Object.keys(settingsAfterRemoteAdd.remoteConfigurations ?? {}).join(", ")}`
|
||||
);
|
||||
assertEquals(settingsAfterRemoteAdd.liveSync, true, "remote-add changed the persisted synchronisation mode");
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "remote-activate", remoteId);
|
||||
|
||||
const textPath = "adaptive/text.md";
|
||||
const binaryPath = "adaptive/data.bin";
|
||||
await runCliWithInputOrFail(`created-by-a-${suffix}\n`, vaultA, "--settings", settingsA, "put", textPath);
|
||||
await Deno.writeFile(binarySourceA, deterministicBytes(EXTERNAL_PACK_TEST_BYTES, 0x1a2b3c4d));
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "push", binarySourceA, binaryPath);
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
|
||||
|
||||
const remoteListing = await runCliOrFail(vaultA, "--settings", settingsA, "remote-ls");
|
||||
assertEquals(remoteIdFromListing(remoteListing), remoteId);
|
||||
assert(
|
||||
remoteListing
|
||||
.split(/\r?\n/u)
|
||||
.some((line) => line.startsWith(`${remoteId}\t`) && line.includes("\tactive\t")),
|
||||
`Activated WebDAV profile was not listed as active:\n${remoteListing}`
|
||||
);
|
||||
const exportedConnection = (
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "remote-export", remoteId)
|
||||
).trim();
|
||||
assert(exportedConnection.startsWith("sls+webdav://"));
|
||||
assert(exportedConnection.includes("journalFormat=adaptive-v1"));
|
||||
assert(!exportedConnection.includes("packReadPolicy="));
|
||||
|
||||
const setupURI = await generateSetupUriFromSettings(settingsA, setupPassphrase, true, vaultPassphrase);
|
||||
await initSettingsFile(settingsB);
|
||||
await runCliWithInputOrFail(`${setupPassphrase}\n`, vaultB, "--settings", settingsB, "setup", setupURI);
|
||||
const settingsAfterSetup = JSON.parse(await Deno.readTextFile(settingsB)) as {
|
||||
encryptedPassphrase?: string;
|
||||
};
|
||||
assert(
|
||||
typeof settingsAfterSetup.encryptedPassphrase === "string" &&
|
||||
settingsAfterSetup.encryptedPassphrase.length > 0,
|
||||
"setup did not persist the encrypted Vault passphrase"
|
||||
);
|
||||
await runCliOrFail(
|
||||
vaultB,
|
||||
"--settings",
|
||||
settingsB,
|
||||
"remote-set",
|
||||
remoteId,
|
||||
setPackReadPolicy(exportedConnection, "range")
|
||||
);
|
||||
const rangeConnection = (await runCliOrFail(vaultB, "--settings", settingsB, "remote-export", remoteId)).trim();
|
||||
assert(rangeConnection.includes("packReadPolicy=range"));
|
||||
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
|
||||
assertEquals(
|
||||
sanitiseCatStdout(await runCliOrFail(vaultB, "--settings", settingsB, "cat", textPath)).trimEnd(),
|
||||
`created-by-a-${suffix}`
|
||||
);
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "pull", binaryPath, binaryDestinationB);
|
||||
await assertFilesEqual(binarySourceA, binaryDestinationB, "Adaptive Journal Range transfer differs");
|
||||
|
||||
await runCliWithInputOrFail(`updated-by-b-${suffix}\n`, vaultB, "--settings", settingsB, "put", textPath);
|
||||
await Deno.writeFile(binarySourceB, deterministicBytes(EXTERNAL_PACK_TEST_BYTES, 0x5e6f7788));
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "push", binarySourceB, binaryPath);
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
|
||||
assertEquals(
|
||||
sanitiseCatStdout(await runCliOrFail(vaultA, "--settings", settingsA, "cat", textPath)).trimEnd(),
|
||||
`updated-by-b-${suffix}`
|
||||
);
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "pull", binaryPath, binaryDestinationA);
|
||||
await assertFilesEqual(binarySourceB, binaryDestinationA, "Adaptive Journal whole-Pack transfer differs");
|
||||
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "rm", binaryPath);
|
||||
await runCliOrFail(vaultA, "--settings", settingsA, "sync");
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
|
||||
const deleted = await runCli(vaultB, "--settings", settingsB, "cat", binaryPath);
|
||||
assert(deleted.code !== 0, `Deleted binary remained readable:\n${deleted.combined}`);
|
||||
|
||||
const objectKeys = await listWebDAVObjectKeys(collectionEndpoint);
|
||||
assert(objectKeys.includes("a1~manifest.json"), `Adaptive manifest is missing:\n${objectKeys.join("\n")}`);
|
||||
for (const objectPrefix of ["a1~writer~", "a1~pack~", "a1~commit~"]) {
|
||||
assert(
|
||||
objectKeys.some((key) => key.startsWith(objectPrefix)),
|
||||
`Adaptive object with prefix ${objectPrefix} is missing:\n${objectKeys.join("\n")}`
|
||||
);
|
||||
}
|
||||
const packKeys = objectKeys.filter((key) => key.startsWith("a1~pack~"));
|
||||
assert(packKeys.length >= 2, `Expected external Packs from both CLI writers:\n${objectKeys.join("\n")}`);
|
||||
for (const legacyPrefix of ["a1~index~", "a1~delta~", "a1~metadata~"]) {
|
||||
assert(
|
||||
!objectKeys.some((key) => key.startsWith(legacyPrefix)),
|
||||
`Legacy Adaptive object with prefix ${legacyPrefix} was written:\n${objectKeys.join("\n")}`
|
||||
);
|
||||
}
|
||||
const manifest = JSON.parse(await readWebDAVObjectText(collectionEndpoint, "a1~manifest.json")) as {
|
||||
objectLayout?: unknown;
|
||||
};
|
||||
assertEquals(manifest.objectLayout, "commit-bundle-v1");
|
||||
assert(
|
||||
!objectKeys.some((key) => key.startsWith("a1~probe~")),
|
||||
`Adaptive capability probe objects were not removed:\n${objectKeys.join("\n")}`
|
||||
);
|
||||
assert(
|
||||
!objectKeys.includes("_00000000-milestone.json"),
|
||||
`Legacy Journal milestone was written into the Adaptive repository:\n${objectKeys.join("\n")}`
|
||||
);
|
||||
} finally {
|
||||
if (shouldStartDocker && !keepDocker) {
|
||||
await stopWebDAV().catch(() => {});
|
||||
}
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user