mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-11-05 15:58:19 +00:00
366 lines
18 KiB
TypeScript
366 lines
18 KiB
TypeScript
/**
|
|
* Verifies one complete Object Storage upload from a real Obsidian Vault,
|
|
* through LiveSync's local database and Journal Sync, to an S3-compatible
|
|
* service observed independently through the AWS SDK.
|
|
*
|
|
* The isolated Vault starts with Object Storage settings and the device-local
|
|
* compatibility acknowledgement already in place. Unconfigured start-up is
|
|
* intentionally inert and belongs to the onboarding scenario; compatibility
|
|
* review and visible setup have their own dedicated workflows. Supplying those
|
|
* prerequisites here keeps this scenario focused on the upload boundary.
|
|
*
|
|
* Note creation, local-database observation, one-shot synchronisation, request
|
|
* accounting, remote-object inspection, and prefix cleanup remain in one
|
|
* scenario so that a pass proves the same payload crossed every boundary.
|
|
* Separate successes would not prove that those observations belonged to the
|
|
* same upload.
|
|
*/
|
|
import { randomBytes } from "node:crypto";
|
|
import { execFile } from "node:child_process";
|
|
import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join, resolve } from "node:path";
|
|
import { promisify } from "node:util";
|
|
import { DEFAULT_SETTINGS, TweakValuesTemplate } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
|
import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
|
|
import { evalObsidianJson } from "../runner/cli.ts";
|
|
import { runCliJournalMigration } from "../runner/cliJournalMigration.ts";
|
|
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
|
|
import {
|
|
assertEqual,
|
|
configureObjectStorage,
|
|
createE2eObjectStoragePluginData,
|
|
createE2eObsidianDeviceLocalState,
|
|
prepareRemote,
|
|
pushLocalChanges,
|
|
waitForLiveSyncCoreReady,
|
|
type LocalDatabaseEntry,
|
|
} from "../runner/liveSyncWorkflow.ts";
|
|
import {
|
|
deleteObjectStoragePrefix,
|
|
ensureObjectStorageBucket,
|
|
listObjectStorageObjects,
|
|
loadObjectStorageConfig,
|
|
makeUniqueBucketPrefix,
|
|
readObjectStorageJson,
|
|
} from "../runner/objectStorage.ts";
|
|
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
|
|
import { createTemporaryVault } from "../runner/vault.ts";
|
|
import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../runner/remoteActivity.ts";
|
|
|
|
process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
|
|
|
|
const unicodePaths = process.env.E2E_OBSIDIAN_UNICODE_PATHS === "true";
|
|
const notePath = unicodePaths ? "Cafe\u0301/re\u0301sume\u0301.md" : "E2E/minio-upload.md";
|
|
const useIndependentIds = process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true";
|
|
const useCustomRequestHandler = process.env.E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER === "true";
|
|
const noteContent = [
|
|
"# Object Storage upload from real Obsidian",
|
|
"",
|
|
"This note is created through Obsidian and uploaded by Self-hosted LiveSync to S3-compatible Object Storage.",
|
|
"The test is intentionally small, but it crosses the real Obsidian, Journal Sync, and AWS SDK boundary.",
|
|
`Created at: ${new Date().toISOString()}`,
|
|
"",
|
|
].join("\n");
|
|
|
|
async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.ProcessEnv): Promise<LocalDatabaseEntry> {
|
|
return await evalObsidianJson<LocalDatabaseEntry>(
|
|
cliBinary,
|
|
[
|
|
"(async()=>{",
|
|
`const path=${JSON.stringify(notePath)};`,
|
|
`const content=${JSON.stringify(noteContent)};`,
|
|
"const core=app.plugins.plugins['obsidian-livesync'].core;",
|
|
"const parent=path.slice(0,path.lastIndexOf('/'));",
|
|
"if(!(await app.vault.adapter.exists(parent))) await app.vault.createFolder(parent);",
|
|
"const existing=app.vault.getAbstractFileByPath(path);",
|
|
"if(existing) await app.vault.delete(existing);",
|
|
"const createdFile=await app.vault.create(path,content);",
|
|
"const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));",
|
|
"let entry=false;",
|
|
"for(let i=0;i<40;i++){",
|
|
"await core.services.fileProcessing.commitPendingFileEvents();",
|
|
"entry=await core.localDatabase.getDBEntry(path,undefined,false,true).catch(()=>false);",
|
|
"if(entry&&entry._id&&Array.isArray(entry.children)&&entry.children.length>0) break;",
|
|
"await sleep(250);",
|
|
"}",
|
|
"if(!entry||!entry._id) throw new Error('Timed out waiting for local database entry');",
|
|
"return JSON.stringify({id:entry._id,path:createdFile.path,type:entry.type,children:entry.children||[]});",
|
|
"})()",
|
|
].join(""),
|
|
env
|
|
);
|
|
}
|
|
|
|
async function waitForObjectStorageObjects(prefix: string): Promise<string[]> {
|
|
const objectStorage = await loadObjectStorageConfig();
|
|
const timeoutMs = Number(process.env.E2E_OBSIDIAN_OBJECT_STORAGE_TIMEOUT_MS ?? 20000);
|
|
const deadline = Date.now() + timeoutMs;
|
|
let keys: string[] = [];
|
|
while (Date.now() < deadline) {
|
|
const objects = await listObjectStorageObjects(objectStorage, prefix);
|
|
keys = objects.flatMap((object) => (object.Key ? [object.Key] : []));
|
|
if (keys.length > 0) {
|
|
return keys;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
}
|
|
throw new Error(`Timed out waiting for Object Storage objects under ${prefix}. Last keys: ${keys.join(", ")}`);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const binary = requireObsidianBinary();
|
|
const cli = discoverObsidianCli();
|
|
if (!cli.binary) {
|
|
throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
|
|
}
|
|
|
|
const objectStorage = await loadObjectStorageConfig();
|
|
const bucketPrefix = makeUniqueBucketPrefix("minio-upload");
|
|
const vault = await createTemporaryVault();
|
|
let session: ObsidianLiveSyncSession | undefined;
|
|
|
|
try {
|
|
await ensureObjectStorageBucket(objectStorage);
|
|
|
|
console.log(`Using Obsidian executable: ${binary}`);
|
|
console.log(`Temporary vault: ${vault.path}`);
|
|
console.log(`Temporary Object Storage bucket: ${objectStorage.bucket}`);
|
|
console.log(`Temporary Object Storage prefix: ${bucketPrefix}`);
|
|
|
|
session = await startObsidianLiveSyncSession({
|
|
binary,
|
|
cliBinary: cli.binary,
|
|
vault,
|
|
startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
|
|
pluginData: createE2eObjectStoragePluginData(
|
|
{
|
|
...objectStorage,
|
|
bucketPrefix,
|
|
},
|
|
{ pathNormalizationVersion: unicodePaths ? 1 : 0 }
|
|
),
|
|
localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
|
|
});
|
|
await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
|
|
|
|
const configured = await configureObjectStorage(
|
|
cli.binary,
|
|
session.cliEnv,
|
|
{ ...objectStorage, bucketPrefix },
|
|
{
|
|
pathNormalizationVersion: unicodePaths ? 1 : 0,
|
|
...(useIndependentIds
|
|
? {
|
|
encrypt: true,
|
|
usePathObfuscation: true,
|
|
passphrase: randomBytes(32).toString("base64url"),
|
|
idDerivationVersion: 1,
|
|
idDerivationKey: await deriveIdKey(randomBytes(32).toString("base64url")),
|
|
}
|
|
: {}),
|
|
...(useCustomRequestHandler ? { useCustomRequestHandler: true } : {}),
|
|
}
|
|
);
|
|
await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
|
|
assertEqual(configured.isConfigured, true, "Self-hosted LiveSync was not marked as configured.");
|
|
assertEqual(configured.remoteType, "MINIO", "Remote type was not Object Storage.");
|
|
assertEqual(configured.endpoint, objectStorage.endpoint, "Configured Object Storage endpoint did not match.");
|
|
assertEqual(configured.bucket, objectStorage.bucket, "Configured Object Storage bucket did not match.");
|
|
assertEqual(configured.bucketPrefix, bucketPrefix, "Configured Object Storage bucket prefix did not match.");
|
|
assertEqual(configured.liveSync, false, "LiveSync should remain disabled during this one-shot workflow.");
|
|
|
|
await prepareRemote(cli.binary, session.cliEnv);
|
|
const activityBeforeUpload = await waitForRemoteActivityState(
|
|
session.remoteDebuggingPort,
|
|
REMOTE_ACTIVITY_EXPECTED_STATE.idle
|
|
);
|
|
const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);
|
|
if (useIndependentIds) {
|
|
if (
|
|
!/^f:[0-9a-f]{64}$/u.test(localEntry.id) ||
|
|
localEntry.children.some((child) => !/^h:\+[0-9a-f]{64}$/u.test(child))
|
|
) {
|
|
throw new Error(
|
|
`The real Obsidian Journal upload did not use independent document and Chunk IDs (document length ${localEntry.id.length}, Chunk lengths ${localEntry.children.map((child) => child.length).join(",")}).`
|
|
);
|
|
}
|
|
}
|
|
if (unicodePaths && !useIndependentIds) {
|
|
assertEqual(
|
|
localEntry.id,
|
|
notePath.normalize("NFC").toLowerCase(),
|
|
"Obsidian did not derive the canonical Metadata ID."
|
|
);
|
|
}
|
|
await pushLocalChanges(cli.binary, session.cliEnv);
|
|
const activityAfterUpload = await waitForRemoteActivityState(
|
|
session.remoteDebuggingPort,
|
|
REMOTE_ACTIVITY_EXPECTED_STATE.idle
|
|
);
|
|
if (activityAfterUpload.requestCount <= activityBeforeUpload.requestCount) {
|
|
throw new Error("Object Storage synchronisation did not advance the tracked remote-request count.");
|
|
}
|
|
assertEqual(
|
|
activityAfterUpload.responseCount,
|
|
activityAfterUpload.requestCount,
|
|
"Object Storage remote-request counters did not rebalance after synchronisation."
|
|
);
|
|
|
|
const keys = await waitForObjectStorageObjects(bucketPrefix);
|
|
if (useIndependentIds) {
|
|
const milestone = await readObjectStorageJson<{ encrypted_id_derivation_proof?: string }>(
|
|
objectStorage,
|
|
`${bucketPrefix}_00000000-milestone.json`
|
|
);
|
|
if (!milestone.encrypted_id_derivation_proof) {
|
|
throw new Error("The Journal milestone did not retain an encrypted ID agreement proof.");
|
|
}
|
|
}
|
|
|
|
if (unicodePaths) {
|
|
const cliRoot = await mkdtemp(join(tmpdir(), "livesync-nfc-journal-"));
|
|
try {
|
|
const databasePath = join(cliRoot, "db");
|
|
const vaultPath = join(cliRoot, "vault");
|
|
const settingsPath = join(cliRoot, "settings.json");
|
|
await mkdir(databasePath, { recursive: true });
|
|
await mkdir(dirname(join(vaultPath, notePath)), { recursive: true });
|
|
await writeFile(join(vaultPath, notePath), noteContent);
|
|
const sourceSettings = await evalObsidianJson<Record<string, unknown>>(
|
|
cli.binary,
|
|
[
|
|
"(async()=>{",
|
|
"const service=app.plugins.plugins['obsidian-livesync'].core.services.setting;",
|
|
"const s=service.currentSettings();",
|
|
`const keys=${JSON.stringify([...Object.keys(TweakValuesTemplate), "encrypt", "passphrase", "usePathObfuscation", "E2EEAlgorithm", "idDerivationKey"])};`,
|
|
"const values=Object.fromEntries(keys.map(key=>[key,s[key]]));",
|
|
"if(s.idDerivationVersion===1){",
|
|
"values.encryptedIdDerivationKey=await service.encryptConfigurationItem(s.idDerivationKey,s,'*');",
|
|
"values.idDerivationKey='';values.configPassphraseStore='';",
|
|
"}",
|
|
"return JSON.stringify(values);",
|
|
"})()",
|
|
].join(""),
|
|
session.cliEnv
|
|
);
|
|
await runCliJournalMigration(objectStorage, bucketPrefix, sourceSettings);
|
|
await writeFile(
|
|
settingsPath,
|
|
JSON.stringify({
|
|
...DEFAULT_SETTINGS,
|
|
...objectStorage,
|
|
...sourceSettings,
|
|
remoteType: "MINIO",
|
|
bucketPrefix,
|
|
pathNormalizationVersion: 1,
|
|
isConfigured: true,
|
|
})
|
|
);
|
|
const execute = promisify(execFile);
|
|
const run = async (...args: string[]) =>
|
|
await execute(
|
|
process.execPath,
|
|
[
|
|
resolve("src/apps/cli/dist/index.cjs"),
|
|
databasePath,
|
|
"--vault",
|
|
vaultPath,
|
|
"--settings",
|
|
settingsPath,
|
|
...args,
|
|
],
|
|
{ timeout: 60_000, maxBuffer: 4 * 1024 * 1024 }
|
|
);
|
|
await run("--force", "--write-settings", "fetch");
|
|
assertEqual(
|
|
await readFile(join(vaultPath, notePath), "utf8"),
|
|
noteContent,
|
|
"CLI Fetch changed the file contents."
|
|
);
|
|
assertEqual(
|
|
(await run("cat", notePath.normalize("NFC"))).stdout,
|
|
noteContent,
|
|
"CLI Fetch did not retain the remote Chunk content."
|
|
);
|
|
const listedPaths = (await run("ls")).stdout
|
|
.trim()
|
|
.split("\n")
|
|
.map((line) => line.split("\t")[0]);
|
|
assertEqual(listedPaths.length, 1, "CLI Fetch retained duplicate Metadata.");
|
|
assertEqual(
|
|
listedPaths[0].normalize("NFC"),
|
|
notePath.normalize("NFC"),
|
|
"CLI Fetch changed the logical path."
|
|
);
|
|
const fileInfo = JSON.parse((await run("info", listedPaths[0])).stdout) as { id: string };
|
|
assertEqual(fileInfo.id, localEntry.id, "CLI Fetch used a different Metadata ID.");
|
|
const expectedNames = await readdir(dirname(join(vaultPath, notePath)));
|
|
assertEqual(expectedNames.length, 1, "CLI Fetch created a duplicate physical file.");
|
|
assertEqual(
|
|
expectedNames[0],
|
|
notePath.split("/").slice(-1)[0],
|
|
"CLI Fetch renamed the NFD physical file."
|
|
);
|
|
const editedContent = noteContent + "\nEdited after CLI restart.\n";
|
|
await writeFile(join(vaultPath, notePath), editedContent);
|
|
const nextTime = new Date(Date.now() + 5000);
|
|
await utimes(join(vaultPath, notePath), nextTime, nextTime);
|
|
await run("mirror");
|
|
await run("sync");
|
|
await pushLocalChanges(cli.binary, session.cliEnv);
|
|
const sourceFile = join(vault.path, localEntry.path);
|
|
const deadline = Date.now() + 20_000;
|
|
while ((await readFile(sourceFile, "utf8")) !== editedContent && Date.now() < deadline) {
|
|
await new Promise((resolveWait) => setTimeout(resolveWait, 200));
|
|
}
|
|
assertEqual(
|
|
await readFile(sourceFile, "utf8"),
|
|
editedContent,
|
|
"Obsidian did not receive the CLI edit."
|
|
);
|
|
await run("rm", notePath.normalize("NFC"));
|
|
await run("sync");
|
|
await pushLocalChanges(cli.binary, session.cliEnv);
|
|
const deleteDeadline = Date.now() + 20_000;
|
|
let exists = true;
|
|
while (exists && Date.now() < deleteDeadline) {
|
|
exists = await readFile(sourceFile).then(
|
|
() => true,
|
|
(error: NodeJS.ErrnoException) => {
|
|
if (error.code === "ENOENT") return false;
|
|
throw error;
|
|
}
|
|
);
|
|
if (exists) await new Promise((resolveWait) => setTimeout(resolveWait, 200));
|
|
}
|
|
assertEqual(exists, false, "Obsidian did not reflect the CLI deletion.");
|
|
console.log(
|
|
"NFC_JOURNAL_ROUND_TRIP: canonical ID, NFD physical name, CLI Fetch, edit after process restart, and deletion verified."
|
|
);
|
|
} finally {
|
|
await rm(cliRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
console.log(
|
|
`Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
|
|
);
|
|
} finally {
|
|
if (session) {
|
|
await session.app.stop();
|
|
}
|
|
await vault.dispose();
|
|
if (process.env.E2E_OBSIDIAN_KEEP_OBJECT_STORAGE !== "true") {
|
|
await deleteObjectStoragePrefix(objectStorage, bucketPrefix).catch((error: unknown) => {
|
|
console.warn(error instanceof Error ? error.message : error);
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
main().catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.stack : error);
|
|
process.exit(1);
|
|
});
|