mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-11-03 06:48:18 +00:00
Preserve CLI database continuity after reconstruction
This commit is contained in:
1 parent
0e458e5e20
commit
d50ee9a83e
21 files changed
+1585
-60
No files matched your search
+36
-1
@@ -70,6 +70,7 @@ livesync-cli [database-path] [command] [args...]
|
||||
- `--force` / `-f`: Required by `rebuild` and `fetch` because they replace a database. It also retains its existing `init-settings` overwrite behaviour.
|
||||
- `--write-settings`: Write setting migrations and other lasting changes after the command succeeds.
|
||||
- `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option.
|
||||
- `fetch` and `rebuild` always retain the selected local database suffix after success. Other lasting changes from these commands still require this option.
|
||||
- All remaining commands leave the settings file unchanged by default.
|
||||
- Temporary values used to suspend synchronisation or select a remote for one command are never written.
|
||||
- `--compat-remote-admin-exit-zero`: Preserve the former zero exit code when `mark-resolved`, `lock-remote`, or `unlock-remote` returns a provider verification failure.
|
||||
@@ -84,8 +85,9 @@ livesync-cli [database-path] [command] [args...]
|
||||
- If `vault-path` is omitted, it defaults to `database-path` (compatibility mode).
|
||||
- Use this command to keep your local `.md` files in sync with the database.
|
||||
- `rebuild`: Rebuild the local and remote databases from the selected Vault. This replaces existing local and remote database state and requires `--force`.
|
||||
- This command may select a new local database suffix. Add `--write-settings` if later CLI commands should use the resulting database selection; otherwise the existing settings file remains unchanged.
|
||||
- The selected local database suffix is saved automatically, so later commands use the reconstructed database.
|
||||
- `fetch`: Replace the local database with the configured remote state, then reflect fetched files to the selected Vault. Existing local database state is discarded, so this command requires `--force`.
|
||||
- The selected local database suffix is saved automatically, including when `--write-settings` is omitted.
|
||||
- `ls [prefix]`: List files currently stored in the local database.
|
||||
- `push <src> <dst>`: Push a local file `<src>` into the database at path `<dst>`.
|
||||
- `pull <src> <dst>`: Pull a file `<src>` from the database into local file `<dst>`.
|
||||
@@ -105,6 +107,39 @@ livesync-cli [database-path] [command] [args...]
|
||||
|
||||
Remote-administration commands verify the resulting milestone state through the selected provider. The existing `[Verification]` lines remain suitable for scripts which inspect command output, while the default exit code now reflects whether that verification succeeded.
|
||||
|
||||
### Interrupted reconstruction and older runtime state
|
||||
|
||||
An interrupted `fetch` or `rebuild` leaves a record in
|
||||
`<database-path>/.livesync/reconstruction.json`. Ordinary synchronisation and
|
||||
mutating commands stop while reconstruction is incomplete. Preserve the Vault,
|
||||
database directory, and settings, check the reported failure, then retry the
|
||||
same command with `--force` and the same database, Vault, and settings paths.
|
||||
`rebuild` replaces remote state as well: confirm that the Vault remains the
|
||||
authoritative copy before retrying it. Do not remove the record merely to
|
||||
restart `daemon`.
|
||||
|
||||
If reconstruction finished but its settings update was interrupted, the next
|
||||
invocation completes that update before starting the core. If the settings
|
||||
file was independently edited, the CLI stops for review. The pending record
|
||||
can contain settings and credentials; protect it like the settings file.
|
||||
|
||||
Older CLI versions stored runtime state without recording its database and
|
||||
Vault owner. If unowned state is present, the CLI preserves it and stops
|
||||
ordinary mutating commands. Read commands remain available for inspection.
|
||||
Back up unique edits and establish the authoritative copy before explicitly
|
||||
using `fetch --force` or `rebuild --force`. The old runtime files remain
|
||||
available for diagnosis; they are not automatically imported into a new scope.
|
||||
|
||||
This check runs before scanning, event replay, and synchronisation. Its message
|
||||
identifies each affected file and distinguishes an unknown owner from a file
|
||||
which cannot be read or parsed. Read commands display the warning and continue
|
||||
inspection. After successful reconstruction for the selected database and Vault,
|
||||
retained legacy files are no longer inspected, even if they are malformed.
|
||||
Corruption in the currently selected runtime KV remains an initialisation error.
|
||||
|
||||
Use one CLI process at a time for each database directory. Separate runtime
|
||||
scopes do not make concurrent writers to shared device settings supported.
|
||||
|
||||
### Examples
|
||||
|
||||
```bash
|
||||
|
||||
@@ -78,12 +78,20 @@ describe("CLI database preparation", () => {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1, force = false) {
|
||||
async function start(
|
||||
command: CLICommand,
|
||||
runner: CliCommandRunner,
|
||||
exitCode = 1,
|
||||
force = false,
|
||||
writeSettings = false
|
||||
) {
|
||||
process.exitCode = undefined;
|
||||
process.argv = [
|
||||
"node",
|
||||
"livesync-cli",
|
||||
directory,
|
||||
...(force ? ["--force"] : []),
|
||||
...(writeSettings ? ["--write-settings"] : []),
|
||||
"--vault",
|
||||
vaultPath,
|
||||
"--settings",
|
||||
@@ -386,6 +394,194 @@ describe("CLI database preparation", () => {
|
||||
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
|
||||
});
|
||||
|
||||
it.each(["fetch", "rebuild"] as const)(
|
||||
"persists the selected database after %s without other command settings",
|
||||
async (command) => {
|
||||
const before = JSON.parse(await readFile(settingsPath, "utf-8"));
|
||||
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
|
||||
await core.services.setting.applyPartial({
|
||||
additionalSuffixOfDatabaseName: "headless-app",
|
||||
customChunkSize: before.customChunkSize + 10,
|
||||
});
|
||||
return true;
|
||||
});
|
||||
|
||||
await start(command, runner, 0, true);
|
||||
|
||||
const after = JSON.parse(await readFile(settingsPath, "utf-8"));
|
||||
expect(after.additionalSuffixOfDatabaseName).toBe("headless-app");
|
||||
expect(after.customChunkSize).toBe(before.customChunkSize);
|
||||
}
|
||||
);
|
||||
|
||||
it.each(["suffix", "vault"] as const)("isolates persistent state by selected %s", async (selection) => {
|
||||
await start("ls", async (_options, { core }) => {
|
||||
await core.services.keyValueDB.kvDB.set("scope-probe", "original");
|
||||
await core.services.config.setValue("scope-probe", "original");
|
||||
return false;
|
||||
});
|
||||
if (selection === "suffix") {
|
||||
const settings = JSON.parse(await readFile(settingsPath, "utf-8"));
|
||||
settings.additionalSuffixOfDatabaseName = "other";
|
||||
await writeFile(settingsPath, JSON.stringify(settings));
|
||||
} else {
|
||||
vaultPath = join(directory, "other-vault");
|
||||
await mkdir(vaultPath);
|
||||
}
|
||||
const observed: unknown[] = [];
|
||||
await start("ls", async (_options, { core }) => {
|
||||
observed.push(await core.services.keyValueDB.kvDB.get("scope-probe"));
|
||||
observed.push(await core.services.config.getValue("scope-probe"));
|
||||
return false;
|
||||
});
|
||||
expect(observed).toEqual([undefined, null]);
|
||||
});
|
||||
|
||||
it.each(["sync", "daemon", "mirror", "put"] as const)(
|
||||
"blocks %s after failed reconstruction before command preparation",
|
||||
async (command) => {
|
||||
await start("fetch", async () => false, 1, true);
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
const ready = vi.spyOn(ControlService.prototype, "onReady");
|
||||
|
||||
await start(command, runner);
|
||||
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(ready).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
|
||||
it("retains legacy queued work and blocks a scanner until explicit reconstruction succeeds", async () => {
|
||||
const legacy = join(directory, ".livesync/runtime/keyvalue-db.json");
|
||||
const legacyContents = JSON.stringify({
|
||||
"queued-files": ["notes/unique.md"],
|
||||
fileStatusMap: { "notes/local.md": 1 },
|
||||
});
|
||||
await mkdir(join(directory, ".livesync/runtime"), { recursive: true });
|
||||
await writeFile(legacy, legacyContents);
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
const scan = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles");
|
||||
|
||||
await start("mirror", runner);
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
expect(await readFile(legacy, "utf-8")).toBe(legacyContents);
|
||||
|
||||
await start("fetch", async () => true, 0, true);
|
||||
const later = vi.fn<CliCommandRunner>(async () => false);
|
||||
await start("sync", later);
|
||||
expect(later).toHaveBeenCalledOnce();
|
||||
expect(await readFile(legacy, "utf-8")).toBe(legacyContents);
|
||||
});
|
||||
|
||||
it("permits normal commands after reconstruction while preserving malformed legacy JSON", async () => {
|
||||
const legacy = join(directory, ".livesync/runtime/keyvalue-db.json");
|
||||
await mkdir(join(directory, ".livesync/runtime"), { recursive: true });
|
||||
await writeFile(legacy, '{"unfinished":');
|
||||
await start("fetch", async () => true, 0, true);
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
await start("sync", runner);
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
expect(await readFile(legacy, "utf-8")).toBe('{"unfinished":');
|
||||
});
|
||||
|
||||
it.each([
|
||||
["kv", '{"unfinished":', "cannot be parsed"],
|
||||
["snapshot", "[", "cannot be parsed"],
|
||||
["kv", '{"queued-files":["unique.md"]}', "owner is unknown"],
|
||||
["snapshot", '[{"path":"unique.md"}]', "owner is unknown"],
|
||||
])("diagnoses unowned %s (%s) before readiness and ignores it after recovery", async (kind, contents, reason) => {
|
||||
const legacy =
|
||||
kind === "kv"
|
||||
? join(directory, ".livesync/runtime/keyvalue-db.json")
|
||||
: join(vaultPath, ".livesync-snapshot.json");
|
||||
await mkdir(join(directory, ".livesync/runtime"), { recursive: true });
|
||||
await writeFile(legacy, contents);
|
||||
const messages = () => standardIo.writeStderr.mock.calls.flat().join("");
|
||||
const reader = vi.fn<CliCommandRunner>(async () => false);
|
||||
await start("ls", reader);
|
||||
expect(reader).toHaveBeenCalledOnce();
|
||||
expect(messages()).toContain(reason);
|
||||
expect(messages()).toContain(legacy);
|
||||
expect(messages()).toContain("inspection may continue");
|
||||
expect(messages().split("[Legacy state]")).toHaveLength(2);
|
||||
expect(standardIo.writeStdout.mock.calls.flat().join("")).not.toContain("[Legacy state]");
|
||||
|
||||
standardIo.writeStderr.mockClear();
|
||||
const writer = vi.fn<CliCommandRunner>(async () => false);
|
||||
const ready = vi.spyOn(ControlService.prototype, "onReady");
|
||||
await start("mirror", writer);
|
||||
expect(writer).not.toHaveBeenCalled();
|
||||
expect(ready).not.toHaveBeenCalled();
|
||||
expect(messages()).toContain("stopped before scanning");
|
||||
expect(messages()).toContain(reason);
|
||||
expect(messages().split("[Legacy state]")).toHaveLength(2);
|
||||
ready.mockRestore();
|
||||
|
||||
await start("fetch", async () => true, 0, true);
|
||||
standardIo.writeStderr.mockClear();
|
||||
const later = vi.fn<CliCommandRunner>(async () => false);
|
||||
await start("sync", later);
|
||||
expect(later).toHaveBeenCalledOnce();
|
||||
expect(messages()).not.toContain("[Legacy state]");
|
||||
expect(await readFile(legacy, "utf-8")).toBe(contents);
|
||||
});
|
||||
|
||||
it("blocks read commands which request settings writes while reconstruction is interrupted", async () => {
|
||||
await start("fetch", async () => false, 1, true);
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
const load = vi.spyOn(ControlService.prototype, "onLoad");
|
||||
|
||||
await start("ls", runner, 1, false, true);
|
||||
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(load).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("invalidates every Vault scope for the reset database and retains other database state", async () => {
|
||||
const originalVault = vaultPath;
|
||||
const otherVault = join(directory, "other-vault");
|
||||
await mkdir(otherVault);
|
||||
const select = async (suffix: string, selectedVault: string) => {
|
||||
vaultPath = selectedVault;
|
||||
const settings = JSON.parse(await readFile(settingsPath, "utf-8"));
|
||||
settings.additionalSuffixOfDatabaseName = suffix;
|
||||
await writeFile(settingsPath, JSON.stringify(settings));
|
||||
};
|
||||
for (const [suffix, selectedVault] of [
|
||||
["", originalVault],
|
||||
["", otherVault],
|
||||
["other", originalVault],
|
||||
]) {
|
||||
await select(suffix, selectedVault);
|
||||
await start("ls", async (_options, { core }) => {
|
||||
await core.services.keyValueDB.kvDB.set("scope-probe", "stored");
|
||||
return false;
|
||||
});
|
||||
}
|
||||
await select("", originalVault);
|
||||
let reset: boolean | undefined;
|
||||
await start("ls", async (_options, { core }) => {
|
||||
reset = await core.services.database.resetDatabase();
|
||||
return false;
|
||||
});
|
||||
expect(reset).toBe(true);
|
||||
|
||||
const observed: unknown[] = [];
|
||||
for (const [suffix, selectedVault] of [
|
||||
["", originalVault],
|
||||
["", otherVault],
|
||||
["other", originalVault],
|
||||
]) {
|
||||
await select(suffix, selectedVault);
|
||||
await start("ls", async (_options, { core }) => {
|
||||
observed.push(await core.services.keyValueDB.kvDB.get("scope-probe"));
|
||||
return false;
|
||||
});
|
||||
}
|
||||
expect(observed).toEqual([undefined, undefined, "stored"]);
|
||||
});
|
||||
|
||||
it("does not import vault files for standalone database commands", async () => {
|
||||
const storedPaths: string[] = [];
|
||||
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
|
||||
|
||||
+83
-19
@@ -41,6 +41,14 @@ import {
|
||||
reconcileDurableSettings,
|
||||
settingsEqual,
|
||||
} from "./settingsPersistence";
|
||||
import {
|
||||
inspectLegacyState,
|
||||
isMutatingCommand,
|
||||
isReconstructionCommand,
|
||||
readOptionalFile,
|
||||
ReconstructionState,
|
||||
writeFileAtomically,
|
||||
} from "./runtimeState";
|
||||
|
||||
const SETTINGS_FILE = ".livesync/settings.json";
|
||||
|
||||
@@ -381,7 +389,7 @@ export async function main(
|
||||
}
|
||||
|
||||
// Resolve database path
|
||||
const databasePath = path.resolve(options.databasePath!);
|
||||
let databasePath = path.resolve(options.databasePath!);
|
||||
// Check if database directory exists
|
||||
try {
|
||||
const stat = await fs.stat(databasePath);
|
||||
@@ -389,18 +397,19 @@ export async function main(
|
||||
writeStderrLine(standardIo, `Error: ${databasePath} is not a directory`);
|
||||
process.exit(1);
|
||||
}
|
||||
databasePath = await fs.realpath(databasePath);
|
||||
} catch {
|
||||
writeStderrLine(standardIo, `Error: Database directory ${databasePath} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Resolve settings path
|
||||
const settingsPath = options.settingsPath
|
||||
let settingsPath = options.settingsPath
|
||||
? path.resolve(options.settingsPath)
|
||||
: path.join(databasePath, SETTINGS_FILE);
|
||||
// Resolve vault path: mirror positional argument takes priority,
|
||||
// then --vault flag, otherwise fall back to databasePath.
|
||||
const vaultPath =
|
||||
let vaultPath =
|
||||
options.command === "mirror" && options.commandArgs[0]
|
||||
? path.resolve(options.commandArgs[0])
|
||||
: options.vaultPath
|
||||
@@ -414,11 +423,22 @@ export async function main(
|
||||
writeStderrLine(standardIo, `Error: Vault path ${vaultPath} is not a directory`);
|
||||
process.exit(1);
|
||||
}
|
||||
vaultPath = await fs.realpath(vaultPath);
|
||||
} catch {
|
||||
writeStderrLine(standardIo, `Error: Vault directory ${vaultPath} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
settingsPath = await fs.realpath(settingsPath).catch(() => settingsPath);
|
||||
const reconstruction = new ReconstructionState(databasePath, vaultPath, settingsPath);
|
||||
try {
|
||||
reconstruction.prepare(options.command, options.force ?? false, options.writeSettings ?? false);
|
||||
} catch (error) {
|
||||
writeStderrLine(standardIo, "[Error] Reconstruction recovery is required:", error);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
infoLog(`Self-hosted LiveSync CLI`);
|
||||
infoLog(`Database Path: ${databasePath}`);
|
||||
infoLog(`Vault Path: ${vaultPath}`);
|
||||
@@ -437,7 +457,7 @@ export async function main(
|
||||
}
|
||||
|
||||
// Create service context and hub
|
||||
const context = new NodeServiceContext(databasePath, standardIo);
|
||||
const context = new NodeServiceContext(databasePath, standardIo, vaultPath);
|
||||
const serviceHubInstance = new NodeServiceHub<NodeServiceContext>(databasePath, context);
|
||||
serviceHubInstance.API.addLog.setHandler((message: unknown, level: LOG_LEVEL) => {
|
||||
let levelStr = "";
|
||||
@@ -479,7 +499,7 @@ export async function main(
|
||||
|
||||
// Setup settings handlers
|
||||
const settingService = serviceHubInstance.setting;
|
||||
const originalSettingsText = await fs.readFile(settingsPath, "utf-8").catch(() => undefined);
|
||||
const originalSettingsText = readOptionalFile(settingsPath);
|
||||
let latestPreparedSettingsText: string | undefined;
|
||||
let preparedSettingsRevision = 0;
|
||||
let commandIsRunning = false;
|
||||
@@ -600,21 +620,19 @@ export async function main(
|
||||
|
||||
const writeSettingsAtomically = async (content: string | undefined): Promise<void> => {
|
||||
if (content === undefined || content === originalSettingsText) return;
|
||||
const temporaryPath = `${settingsPath}.${process.pid}.tmp`;
|
||||
await fs.mkdir(path.dirname(settingsPath), { recursive: true });
|
||||
try {
|
||||
await fs.writeFile(temporaryPath, content, "utf-8");
|
||||
await fs.rename(temporaryPath, settingsPath);
|
||||
if (options.verbose) {
|
||||
writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
await fs.unlink(temporaryPath).catch(() => {});
|
||||
throw error;
|
||||
writeFileAtomically(settingsPath, content);
|
||||
if (options.verbose) {
|
||||
writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`);
|
||||
}
|
||||
};
|
||||
|
||||
// Start the core
|
||||
let unloaded = false;
|
||||
const unload = async () => {
|
||||
if (unloaded) return;
|
||||
unloaded = true;
|
||||
await core.services.control.onUnload();
|
||||
};
|
||||
try {
|
||||
infoLog(`[Starting] Initializing LiveSync...`);
|
||||
|
||||
@@ -623,6 +641,31 @@ export async function main(
|
||||
writeStderrLine(standardIo, `[Error] Failed to initialize LiveSync`);
|
||||
process.exit(1);
|
||||
}
|
||||
if ((await core.services.keyValueDB.kvDB.get("cli-legacy-state-reviewed")) !== true) {
|
||||
const legacyIssues = inspectLegacyState(databasePath, vaultPath);
|
||||
if (legacyIssues.length > 0) {
|
||||
const reasons = {
|
||||
unowned: "database and Vault owner is unknown",
|
||||
invalid: "state cannot be parsed in its expected format; owner and contents cannot be established",
|
||||
unreadable: "state cannot be read; owner and contents cannot be established",
|
||||
};
|
||||
const details = legacyIssues.map((issue) => `${issue.path}: ${reasons[issue.reason]}`).join("; ");
|
||||
const advice =
|
||||
"Preserve these files, back up unique edits, and verify the authoritative source before choosing fetch --force or rebuild --force.";
|
||||
const blocked =
|
||||
!isReconstructionCommand(options.command) &&
|
||||
(isMutatingCommand(options.command) || options.writeSettings);
|
||||
if (blocked) {
|
||||
throw new Error(
|
||||
`[Legacy state] ${details}. Command '${options.command}' stopped before scanning, event replay, or synchronisation. ${advice}`
|
||||
);
|
||||
}
|
||||
const action = isReconstructionCommand(options.command)
|
||||
? "Explicit reconstruction will proceed; the legacy files will be retained."
|
||||
: "Read-only inspection may continue; the legacy files will be retained.";
|
||||
writeStderrLine(standardIo, `[Legacy state] ${details}. ${action} ${advice}`);
|
||||
}
|
||||
}
|
||||
const settingsAfterLoadText = latestPreparedSettingsText
|
||||
? preserveStoredSetting(latestPreparedSettingsText, originalSettingsText, "useIndexedDBAdapter")
|
||||
: originalSettingsText;
|
||||
@@ -655,7 +698,7 @@ export async function main(
|
||||
try {
|
||||
readyResult = await core.services.control.onReady();
|
||||
} finally {
|
||||
if (!readyResult) await core.services.control.onUnload();
|
||||
if (!readyResult) await unload();
|
||||
}
|
||||
if (!readyResult) {
|
||||
throw new Error("Failed to initialise LiveSync.");
|
||||
@@ -751,6 +794,21 @@ export async function main(
|
||||
settingsTextToCommit = settingsAfterLoadText;
|
||||
}
|
||||
}
|
||||
if (result && isReconstructionCommand(options.command)) {
|
||||
if (!options.writeSettings) {
|
||||
const durable =
|
||||
originalSettingsText === undefined
|
||||
? cloneSettings(durableSettingsBeforeSuspend)
|
||||
: (JSON.parse(originalSettingsText) as ObsidianLiveSyncSettings);
|
||||
durable.additionalSuffixOfDatabaseName =
|
||||
core.services.setting.currentSettings().additionalSuffixOfDatabaseName;
|
||||
settingsTextToCommit = JSON.stringify(durable, null, 2);
|
||||
}
|
||||
settingsTextToCommit ??= originalSettingsText;
|
||||
if (settingsTextToCommit === undefined)
|
||||
throw new Error("Reconstruction completed without database selection settings.");
|
||||
await core.services.keyValueDB.kvDB.set("cli-legacy-state-reviewed", true);
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`);
|
||||
@@ -764,10 +822,16 @@ export async function main(
|
||||
// Keep the process running
|
||||
await new Promise(() => {});
|
||||
} else {
|
||||
await core.services.control.onUnload();
|
||||
await writeSettingsAtomically(settingsTextToCommit);
|
||||
await unload();
|
||||
if (result && isReconstructionCommand(options.command)) {
|
||||
reconstruction.complete(settingsTextToCommit!);
|
||||
reconstruction.commit();
|
||||
} else {
|
||||
await writeSettingsAtomically(settingsTextToCommit);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await unload().catch(() => {});
|
||||
writeStderrLine(standardIo, `[Error] Failed to start:`, error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { watch as chokidarWatch, type FSWatcher } from "chokidar";
|
||||
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
|
||||
import { fsPromises as fs, getNodeFileCreationTime, path, type Stats } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
|
||||
import { writeFileAtomically } from "@/apps/cli/runtimeState";
|
||||
|
||||
/**
|
||||
* CLI-specific type guard adapter
|
||||
@@ -44,18 +45,19 @@ class CLITypeGuardAdapter implements IStorageEventTypeGuardAdapter<NodeFile, Nod
|
||||
* CLI-specific persistence adapter (file-based snapshot)
|
||||
*/
|
||||
class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
|
||||
private snapshotPath: string;
|
||||
private snapshotPath: () => string;
|
||||
|
||||
constructor(
|
||||
basePath: string,
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined,
|
||||
snapshotPath?: () => string
|
||||
) {
|
||||
this.snapshotPath = path.join(basePath, ".livesync-snapshot.json");
|
||||
this.snapshotPath = snapshotPath ?? (() => path.join(basePath, ".livesync-snapshot.json"));
|
||||
}
|
||||
|
||||
async saveSnapshot(snapshot: (FileEventItem | FileEventItemSentinel)[]): Promise<void> {
|
||||
try {
|
||||
await fs.writeFile(this.snapshotPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
||||
writeFileAtomically(this.snapshotPath(), JSON.stringify(snapshot, null, 2));
|
||||
} catch (error) {
|
||||
this.reportDiagnostic("Failed to save snapshot:", error);
|
||||
}
|
||||
@@ -63,7 +65,7 @@ class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
|
||||
|
||||
async loadSnapshot(): Promise<(FileEventItem | FileEventItemSentinel)[] | null> {
|
||||
try {
|
||||
const content = await fs.readFile(this.snapshotPath, "utf-8");
|
||||
const content = await fs.readFile(this.snapshotPath(), "utf-8");
|
||||
return JSON.parse(content) as (FileEventItem | FileEventItemSentinel)[];
|
||||
} catch {
|
||||
return null;
|
||||
@@ -221,10 +223,11 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte
|
||||
basePath: string,
|
||||
ignoreRules?: IgnoreRules,
|
||||
watchEnabled: boolean = false,
|
||||
reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
reportDiagnostic: CliDiagnosticReporter = () => undefined,
|
||||
snapshotPath?: () => string
|
||||
) {
|
||||
this.typeGuard = new CLITypeGuardAdapter();
|
||||
this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic);
|
||||
this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic, snapshotPath);
|
||||
this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic);
|
||||
this.status = new CLIStatusAdapter();
|
||||
this.converter = new CLIConverterAdapter();
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import type { IStorageEventWatchHandlers } from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
|
||||
import type { NodeFile } from "@/apps/cli/adapters/NodeTypes";
|
||||
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { runtimeScopeDirectory } from "../runtimeState";
|
||||
|
||||
// ── chokidar mock ──────────────────────────────────────────────────────────────
|
||||
// Must be hoisted before imports that pull in chokidar.
|
||||
@@ -44,6 +46,27 @@ describe("CLIStorageEventManagerAdapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps pending snapshots with the selected database and Vault pair", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-snapshot-"));
|
||||
let database = "first";
|
||||
let vault = "vault-a";
|
||||
const adapter = new CLIStorageEventManagerAdapter(directory, undefined, false, undefined, () =>
|
||||
path.join(runtimeScopeDirectory(directory, database, vault), "storage-events.json")
|
||||
);
|
||||
try {
|
||||
await adapter.persistence.saveSnapshot([{ type: "SENTINEL_FLUSH" }]);
|
||||
database = "second";
|
||||
expect(await adapter.persistence.loadSnapshot()).toBeNull();
|
||||
database = "first";
|
||||
vault = "vault-b";
|
||||
expect(await adapter.persistence.loadSnapshot()).toBeNull();
|
||||
vault = "vault-a";
|
||||
expect(await adapter.persistence.loadSnapshot()).toEqual([{ type: "SENTINEL_FLUSH" }]);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("beginWatch is no-op when watchEnabled=false", async () => {
|
||||
const adapter = new CLIStorageEventManagerAdapter("/base", undefined, false);
|
||||
const handlers = makeHandlers();
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
|
||||
import {
|
||||
StorageEventManagerBase,
|
||||
type StorageEventManagerBaseDependencies,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
|
||||
import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter";
|
||||
import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
|
||||
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
|
||||
import { runtimeScopeDirectory } from "@/apps/cli/runtimeState";
|
||||
import type { NodeServiceContext } from "@/apps/cli/services/NodeServiceContext";
|
||||
import { path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
// import type { IMinimumLiveSyncCommands } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
|
||||
|
||||
export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEventManagerAdapter> {
|
||||
@@ -16,12 +22,23 @@ export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEv
|
||||
ignoreRules?: IgnoreRules,
|
||||
watchEnabled?: boolean
|
||||
) {
|
||||
const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled, (message, detail) => {
|
||||
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
|
||||
if (detail !== undefined) {
|
||||
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
|
||||
}
|
||||
});
|
||||
const context = core.services.context as NodeServiceContext;
|
||||
const adapter = new CLIStorageEventManagerAdapter(
|
||||
basePath,
|
||||
ignoreRules,
|
||||
watchEnabled,
|
||||
(message, detail) => {
|
||||
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
|
||||
if (detail !== undefined) {
|
||||
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
|
||||
}
|
||||
},
|
||||
() =>
|
||||
nodePath.join(
|
||||
runtimeScopeDirectory(context.databasePath, dependencies.vaultService.getVaultName(), basePath),
|
||||
"storage-events.json"
|
||||
)
|
||||
);
|
||||
super(adapter, dependencies);
|
||||
this.core = core;
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"buildRun": "npm run build && npm run cli --",
|
||||
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
|
||||
"check": "tsc -p tsconfig.json",
|
||||
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
|
||||
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
|
||||
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
|
||||
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
|
||||
"test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",
|
||||
@@ -37,6 +37,7 @@
|
||||
"test:e2e:coverage": "node ../../../test/e2e-cli/run-coverage.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha256-js": "5.2.0",
|
||||
"chokidar": "^4.0.0",
|
||||
"minimatch": "^10.2.5",
|
||||
"octagonal-wheels": "^0.1.55",
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
// This module belongs exclusively to the Node.js CLI host.
|
||||
import { Sha256 } from "@aws-crypto/sha256-js";
|
||||
import { fs, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { CLICommand } from "./commands/types";
|
||||
|
||||
function sha256Hex(value: string): string {
|
||||
const hash = new Sha256();
|
||||
hash.update(new TextEncoder().encode(value));
|
||||
return Array.from(hash.digestSync(), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
export function runtimeScopeDirectory(databasePath: string, databaseName: string, vaultPath: string): string {
|
||||
const scope = sha256Hex(vaultPath);
|
||||
return path.join(runtimeDatabaseDirectory(databasePath, databaseName), scope);
|
||||
}
|
||||
|
||||
export function runtimeDatabaseDirectory(databasePath: string, databaseName: string): string {
|
||||
const database = sha256Hex(databaseName);
|
||||
return path.join(databasePath, ".livesync", "runtime", "scopes", database);
|
||||
}
|
||||
|
||||
export function writeFileAtomically(filePath: string, content: string): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
const temporaryPath = `${filePath}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
||||
try {
|
||||
const descriptor = fs.openSync(temporaryPath, "wx", 0o600);
|
||||
try {
|
||||
fs.writeFileSync(descriptor, content, "utf-8");
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
fs.renameSync(temporaryPath, filePath);
|
||||
syncDirectory(path.dirname(filePath));
|
||||
} finally {
|
||||
fs.rmSync(temporaryPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function syncDirectory(directory: string): void {
|
||||
// Windows does not support opening a directory for fsync.
|
||||
if (process.platform === "win32") return;
|
||||
const descriptor = fs.openSync(directory, "r");
|
||||
try {
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
export function readOptionalFile(filePath: string): string | undefined {
|
||||
try {
|
||||
return fs.readFileSync(filePath, "utf-8");
|
||||
} catch (error) {
|
||||
if ((error as { code?: string }).code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function isReconstructionCommand(command: CLICommand): command is "fetch" | "rebuild" {
|
||||
return command === "fetch" || command === "rebuild";
|
||||
}
|
||||
|
||||
const READ_COMMANDS = new Set<CLICommand>([
|
||||
"cat",
|
||||
"cat-rev",
|
||||
"ls",
|
||||
"info",
|
||||
"remote-ls",
|
||||
"remote-export",
|
||||
"remote-status",
|
||||
"p2p-peers",
|
||||
]);
|
||||
|
||||
export function isMutatingCommand(command: CLICommand): boolean {
|
||||
return !READ_COMMANDS.has(command);
|
||||
}
|
||||
|
||||
interface ReconstructionMarker {
|
||||
version: 1;
|
||||
command: "fetch" | "rebuild";
|
||||
databasePath: string;
|
||||
vaultPath: string;
|
||||
settingsPath: string;
|
||||
settingsBeforeHash: string;
|
||||
phase: "running" | "completed";
|
||||
settingsText?: string;
|
||||
}
|
||||
|
||||
function hashSettings(content: string | undefined): string {
|
||||
return sha256Hex(JSON.stringify(content ?? null));
|
||||
}
|
||||
|
||||
export class ReconstructionState {
|
||||
readonly markerPath: string;
|
||||
private marker: ReconstructionMarker | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly databasePath: string,
|
||||
private readonly vaultPath: string,
|
||||
private readonly settingsPath: string
|
||||
) {
|
||||
this.markerPath = path.join(databasePath, ".livesync", "reconstruction.json");
|
||||
}
|
||||
|
||||
prepare(command: CLICommand, force: boolean, writeSettings = false): void {
|
||||
const contents = readOptionalFile(this.markerPath);
|
||||
if (contents !== undefined) {
|
||||
const parsed: unknown = JSON.parse(contents);
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
throw new Error(`Invalid reconstruction state: ${this.markerPath}. Preserve it for recovery.`);
|
||||
}
|
||||
const marker = parsed as ReconstructionMarker;
|
||||
if (
|
||||
marker.version !== 1 ||
|
||||
!isReconstructionCommand(marker.command) ||
|
||||
(marker.phase !== "running" && marker.phase !== "completed") ||
|
||||
typeof marker.settingsPath !== "string" ||
|
||||
typeof marker.vaultPath !== "string" ||
|
||||
typeof marker.settingsBeforeHash !== "string" ||
|
||||
marker.databasePath !== this.databasePath ||
|
||||
(marker.phase === "completed" && typeof marker.settingsText !== "string")
|
||||
) {
|
||||
throw new Error(`Invalid reconstruction state: ${this.markerPath}. Preserve it for recovery.`);
|
||||
}
|
||||
this.marker = marker;
|
||||
if (marker.phase === "completed") {
|
||||
this.commit();
|
||||
} else if (
|
||||
(isMutatingCommand(command) || writeSettings) &&
|
||||
!(
|
||||
command === marker.command &&
|
||||
force &&
|
||||
marker.vaultPath === this.vaultPath &&
|
||||
marker.settingsPath === this.settingsPath
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`An interrupted ${marker.command} requires recovery. Retry ${marker.command} --force with the same database, Vault, and settings paths. State: ${this.markerPath}`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (isReconstructionCommand(command)) {
|
||||
this.marker = {
|
||||
version: 1,
|
||||
command,
|
||||
databasePath: this.databasePath,
|
||||
vaultPath: this.vaultPath,
|
||||
settingsPath: this.settingsPath,
|
||||
phase: "running",
|
||||
settingsBeforeHash: hashSettings(readOptionalFile(this.settingsPath)),
|
||||
};
|
||||
this.save();
|
||||
}
|
||||
}
|
||||
|
||||
complete(settingsText: string): void {
|
||||
if (!this.marker || this.marker.phase !== "running") throw new Error("Reconstruction was not started.");
|
||||
this.marker = { ...this.marker, phase: "completed", settingsText };
|
||||
this.save();
|
||||
}
|
||||
|
||||
commit(): void {
|
||||
const marker = this.marker;
|
||||
if (!marker || marker.phase !== "completed" || marker.settingsText === undefined) {
|
||||
throw new Error("Reconstruction has no completed settings update.");
|
||||
}
|
||||
const current = readOptionalFile(marker.settingsPath);
|
||||
if (current !== marker.settingsText && hashSettings(current) !== marker.settingsBeforeHash) {
|
||||
throw new Error(
|
||||
`The settings file changed after reconstruction: ${marker.settingsPath}. Preserve the reconstruction state and review both settings versions before continuing.`
|
||||
);
|
||||
}
|
||||
if (current !== marker.settingsText) writeFileAtomically(marker.settingsPath, marker.settingsText);
|
||||
this.clear();
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
fs.rmSync(this.markerPath, { force: true });
|
||||
syncDirectory(path.dirname(this.markerPath));
|
||||
this.marker = undefined;
|
||||
}
|
||||
|
||||
private save(): void {
|
||||
writeFileAtomically(this.markerPath, JSON.stringify(this.marker, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
export interface LegacyStateIssue {
|
||||
path: string;
|
||||
reason: "unowned" | "invalid" | "unreadable";
|
||||
}
|
||||
|
||||
export function inspectLegacyState(databasePath: string, vaultPath: string): LegacyStateIssue[] {
|
||||
const issues: LegacyStateIssue[] = [];
|
||||
for (const [filePath, expectArray] of [
|
||||
[path.join(databasePath, ".livesync", "runtime", "keyvalue-db.json"), false],
|
||||
[path.join(vaultPath, ".livesync-snapshot.json"), true],
|
||||
] as const) {
|
||||
let contents: string | undefined;
|
||||
try {
|
||||
contents = readOptionalFile(filePath);
|
||||
} catch {
|
||||
issues.push({ path: filePath, reason: "unreadable" });
|
||||
continue;
|
||||
}
|
||||
if (contents === undefined) continue;
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(contents);
|
||||
} catch {
|
||||
issues.push({ path: filePath, reason: "invalid" });
|
||||
continue;
|
||||
}
|
||||
if (value === null || typeof value !== "object" || Array.isArray(value) !== expectArray) {
|
||||
issues.push({ path: filePath, reason: "invalid" });
|
||||
} else if (Object.keys(value).length > 0) {
|
||||
issues.push({ path: filePath, reason: "unowned" });
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
ReconstructionState,
|
||||
inspectLegacyState,
|
||||
readOptionalFile,
|
||||
runtimeDatabaseDirectory,
|
||||
runtimeScopeDirectory,
|
||||
} from "./runtimeState";
|
||||
|
||||
describe("CLI runtime scope compatibility", () => {
|
||||
it("retains SHA-256 directory names for database and Unicode Vault paths", () => {
|
||||
expect(runtimeScopeDirectory("state", "headless-app", "/vault/日本語😀")).toBe(
|
||||
path.join(
|
||||
"state",
|
||||
".livesync",
|
||||
"runtime",
|
||||
"scopes",
|
||||
"511afda5613706862fa4add801b3f9b19e0db2acacc07b9218f2d01ff41ca61d",
|
||||
"542d31d6918ad8bf915e204e41e479fdbd7251690f182fd802378323b4c6de23"
|
||||
)
|
||||
);
|
||||
expect(path.basename(runtimeDatabaseDirectory("state", ""))).toBe(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("CLI reconstruction state", () => {
|
||||
let directory: string;
|
||||
let settingsPath: string;
|
||||
let state: ReconstructionState;
|
||||
|
||||
beforeEach(() => {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-reconstruction-"));
|
||||
settingsPath = path.join(directory, "settings.json");
|
||||
fs.writeFileSync(settingsPath, JSON.stringify({ additionalSuffixOfDatabaseName: "original" }));
|
||||
state = new ReconstructionState(directory, path.join(directory, "vault"), settingsPath);
|
||||
});
|
||||
|
||||
afterEach(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
|
||||
it("retains the persisted settings fingerprint across hash implementations", () => {
|
||||
state.prepare("fetch", true);
|
||||
expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({
|
||||
settingsBeforeHash: "29e04e3dd1d10b352678a922ff5a03fbe529d3505e9cf46c2ecc860f44478ae8",
|
||||
});
|
||||
});
|
||||
|
||||
it("persists progress before returning and permits only the matching forced retry", () => {
|
||||
state.prepare("fetch", true);
|
||||
expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({
|
||||
phase: "running",
|
||||
command: "fetch",
|
||||
});
|
||||
if (process.platform !== "win32") expect(fs.statSync(state.markerPath).mode & 0o777).toBe(0o600);
|
||||
expect(() => state.prepare("sync", false)).toThrow("interrupted fetch");
|
||||
expect(() => state.prepare("ls", false, true)).toThrow("interrupted fetch");
|
||||
expect(() => state.prepare("fetch", false)).toThrow("interrupted fetch");
|
||||
expect(() => state.prepare("rebuild", true)).toThrow("interrupted fetch");
|
||||
expect(() =>
|
||||
new ReconstructionState(directory, path.join(directory, "other"), settingsPath).prepare("fetch", true)
|
||||
).toThrow("interrupted fetch");
|
||||
expect(() =>
|
||||
new ReconstructionState(
|
||||
directory,
|
||||
path.join(directory, "vault"),
|
||||
path.join(directory, "other.json")
|
||||
).prepare("fetch", true)
|
||||
).toThrow("interrupted fetch");
|
||||
expect(() => state.prepare("ls", false)).not.toThrow();
|
||||
expect(() => state.prepare("fetch", true)).not.toThrow();
|
||||
});
|
||||
|
||||
it("recovers completed settings before permitting another command", () => {
|
||||
state.prepare("fetch", true);
|
||||
const completed = JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app", retained: true });
|
||||
state.complete(completed);
|
||||
|
||||
new ReconstructionState(directory, path.join(directory, "vault"), settingsPath).prepare("sync", false);
|
||||
|
||||
expect(fs.readFileSync(settingsPath, "utf-8")).toBe(completed);
|
||||
expect(readOptionalFile(state.markerPath)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["normal completion", "restart"] as const)(
|
||||
"preserves user settings changed during reconstruction at %s",
|
||||
(phase) => {
|
||||
state.prepare("rebuild", true);
|
||||
state.complete(JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app" }));
|
||||
const edited = JSON.stringify({ additionalSuffixOfDatabaseName: "user-selection", unique: "edit" });
|
||||
fs.writeFileSync(settingsPath, edited);
|
||||
|
||||
const commit =
|
||||
phase === "restart"
|
||||
? () =>
|
||||
new ReconstructionState(directory, path.join(directory, "vault"), settingsPath).prepare(
|
||||
"sync",
|
||||
false
|
||||
)
|
||||
: () => state.commit();
|
||||
|
||||
expect(commit).toThrow("settings file changed");
|
||||
expect(fs.readFileSync(settingsPath, "utf-8")).toBe(edited);
|
||||
expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({ phase: "completed" });
|
||||
}
|
||||
);
|
||||
|
||||
it("clears a completed marker after an interruption following the settings rename", () => {
|
||||
state.prepare("fetch", true);
|
||||
const completed = JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app" });
|
||||
state.complete(completed);
|
||||
fs.writeFileSync(settingsPath, completed);
|
||||
state.prepare("ls", false);
|
||||
expect(readOptionalFile(state.markerPath)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["null", "[]", "{}", '{"version":2}', '{"version":1,"phase":"completed"}'])(
|
||||
"refuses malformed state %s without clearing it",
|
||||
(contents) => {
|
||||
fs.mkdirSync(path.dirname(state.markerPath), { recursive: true });
|
||||
fs.writeFileSync(state.markerPath, contents);
|
||||
expect(() => state.prepare("sync", false)).toThrow("Invalid reconstruction state");
|
||||
expect(fs.readFileSync(state.markerPath, "utf-8")).toBe(contents);
|
||||
}
|
||||
);
|
||||
|
||||
it.each(["null", "[]", '"text"', "{", ""])("reports invalid legacy KV %s without changing it", (contents) => {
|
||||
const legacy = path.join(directory, ".livesync/runtime/keyvalue-db.json");
|
||||
fs.mkdirSync(path.dirname(legacy), { recursive: true });
|
||||
fs.writeFileSync(legacy, contents);
|
||||
expect(inspectLegacyState(directory, directory)).toEqual([{ path: legacy, reason: "invalid" }]);
|
||||
expect(fs.readFileSync(legacy, "utf-8")).toBe(contents);
|
||||
});
|
||||
|
||||
it("reports both unreadable KV and invalid snapshot instead of hiding the second problem", () => {
|
||||
const legacy = path.join(directory, ".livesync/runtime/keyvalue-db.json");
|
||||
const snapshot = path.join(directory, ".livesync-snapshot.json");
|
||||
fs.mkdirSync(legacy, { recursive: true });
|
||||
fs.writeFileSync(snapshot, "{}");
|
||||
expect(inspectLegacyState(directory, directory)).toEqual([
|
||||
{ path: legacy, reason: "unreadable" },
|
||||
{ path: snapshot, reason: "invalid" },
|
||||
]);
|
||||
expect(fs.statSync(legacy).isDirectory()).toBe(true);
|
||||
expect(fs.readFileSync(snapshot, "utf-8")).toBe("{}");
|
||||
});
|
||||
|
||||
it("preserves non-empty unowned legacy state and ignores empty legacy containers", () => {
|
||||
const legacy = path.join(directory, ".livesync", "runtime", "keyvalue-db.json");
|
||||
fs.mkdirSync(path.dirname(legacy), { recursive: true });
|
||||
fs.writeFileSync(legacy, "{}");
|
||||
fs.writeFileSync(path.join(directory, ".livesync-snapshot.json"), "[]");
|
||||
expect(inspectLegacyState(directory, directory)).toEqual([]);
|
||||
const queued = JSON.stringify({ "queued-files": ["unique-edit.md"] });
|
||||
fs.writeFileSync(legacy, queued);
|
||||
expect(inspectLegacyState(directory, directory)).toEqual([{ path: legacy, reason: "unowned" }]);
|
||||
expect(fs.readFileSync(legacy, "utf-8")).toBe(queued);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,7 @@ import type { IVaultService } from "@vrtmrz/livesync-commonlib/compat/services/b
|
||||
import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
|
||||
import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils";
|
||||
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { readOptionalFile, writeFileAtomically } from "@/apps/cli/runtimeState";
|
||||
|
||||
const NODE_KV_TYPED_KEY = "__nodeKvType";
|
||||
const NODE_KV_VALUES_KEY = "values";
|
||||
@@ -103,12 +104,11 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
}
|
||||
|
||||
private load() {
|
||||
try {
|
||||
const loaded = JSON.parse(nodeFs.readFileSync(this.filePath, "utf-8")) as Record<string, unknown>;
|
||||
this.data = new Map(Object.entries(loaded).map(([key, value]) => [key, deserializeFromNodeKV(value)]));
|
||||
} catch {
|
||||
this.data = new Map();
|
||||
}
|
||||
const contents = readOptionalFile(this.filePath);
|
||||
if (contents === undefined) return;
|
||||
const loaded: unknown = JSON.parse(contents);
|
||||
if (!isRecord(loaded) || Array.isArray(loaded)) throw new Error("Invalid Node key-value database");
|
||||
this.data = new Map(Object.entries(loaded).map(([key, value]) => [key, deserializeFromNodeKV(value)]));
|
||||
}
|
||||
|
||||
private flush() {
|
||||
@@ -116,7 +116,7 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
const serializable = Object.fromEntries(
|
||||
[...this.data.entries()].map(([key, value]) => [key, serializeForNodeKV(value)])
|
||||
);
|
||||
nodeFs.writeFileSync(this.filePath, JSON.stringify(serializable, null, 2), "utf-8");
|
||||
writeFileAtomically(this.filePath, JSON.stringify(serializable, null, 2));
|
||||
}
|
||||
|
||||
async get<T>(key: IDBValidKey): Promise<T> {
|
||||
@@ -163,9 +163,7 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
return filtered;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.flush();
|
||||
}
|
||||
async close(): Promise<void> {}
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
this.data.clear();
|
||||
@@ -185,7 +183,8 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
{
|
||||
private _kvDB: KeyValueDatabase | undefined;
|
||||
private _simpleStore: SimpleStore<unknown> | undefined;
|
||||
private filePath: string;
|
||||
private filePath: string | (() => string);
|
||||
private openedFilePath: string | undefined;
|
||||
private _log = createInstanceLogFunction("NodeKeyValueDBService");
|
||||
|
||||
get simpleStore() {
|
||||
@@ -202,7 +201,7 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
return this._kvDB;
|
||||
}
|
||||
|
||||
constructor(context: T, dependencies: NodeKeyValueDBDependencies<T>, filePath: string) {
|
||||
constructor(context: T, dependencies: NodeKeyValueDBDependencies<T>, filePath: string | (() => string)) {
|
||||
super(context);
|
||||
this.filePath = filePath;
|
||||
|
||||
@@ -215,7 +214,13 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
|
||||
private async openKeyValueDB(): Promise<boolean> {
|
||||
try {
|
||||
this._kvDB = new NodeFileKeyValueDatabase(this.filePath);
|
||||
const filePath = typeof this.filePath === "string" ? this.filePath : this.filePath();
|
||||
if (this._kvDB && this.openedFilePath === filePath) return true;
|
||||
await this._kvDB?.close();
|
||||
this._kvDB = undefined;
|
||||
this.openedFilePath = undefined;
|
||||
this._kvDB = new NodeFileKeyValueDatabase(filePath);
|
||||
this.openedFilePath = filePath;
|
||||
return true;
|
||||
} catch (ex) {
|
||||
this._log("Failed to open Node key-value database", LOG_LEVEL_NOTICE);
|
||||
@@ -226,8 +231,10 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
|
||||
private async _everyOnResetDatabase(): Promise<boolean> {
|
||||
try {
|
||||
await this._kvDB?.del("queued-files");
|
||||
if (!(await this.openKeyValueDB())) return false;
|
||||
await this._kvDB?.destroy();
|
||||
this._kvDB = undefined;
|
||||
this.openedFilePath = undefined;
|
||||
return await this.openKeyValueDB();
|
||||
} catch (ex) {
|
||||
this._log("Failed to reset Node key-value database", LOG_LEVEL_NOTICE);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { createServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
|
||||
import type { NodeKeyValueDBDependencies } from "./NodeKeyValueDBService";
|
||||
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
|
||||
@@ -45,3 +46,70 @@ describe("NodeKeyValueDBService.openSimpleStore", () => {
|
||||
await expect(store.get("key")).rejects.toThrow("KeyValueDB is not initialized yet");
|
||||
});
|
||||
});
|
||||
|
||||
describe("NodeKeyValueDBService persistence", () => {
|
||||
let directory: string;
|
||||
|
||||
beforeEach(() => {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-kv-"));
|
||||
});
|
||||
afterEach(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
|
||||
function compose(filePath: string | (() => string)) {
|
||||
const handlers: Record<string, () => Promise<boolean>> = {};
|
||||
const event = (name: string) => ({
|
||||
addHandler: (handler: () => Promise<boolean>) => {
|
||||
handlers[name] = handler;
|
||||
},
|
||||
});
|
||||
const dependencies = {
|
||||
appLifecycle: { onSettingLoaded: event("load") },
|
||||
databaseEvents: {
|
||||
onResetDatabase: event("reset"),
|
||||
onDatabaseInitialisation: event("initialise"),
|
||||
onUnloadDatabase: event("unload"),
|
||||
onCloseDatabase: event("close"),
|
||||
},
|
||||
vault: {},
|
||||
} as unknown as NodeKeyValueDBDependencies;
|
||||
return { service: new NodeKeyValueDBService(createServiceContext(), dependencies, filePath), handlers };
|
||||
}
|
||||
|
||||
it("retargets existing SimpleStore handles after database selection and resets only the selected file", async () => {
|
||||
let selected = path.join(directory, "first.json");
|
||||
const { service, handlers } = compose(() => selected);
|
||||
const store = service.openSimpleStore("journal");
|
||||
expect(await handlers.load()).toBe(true);
|
||||
await store.set("checkpoint", new Set(["first"]));
|
||||
selected = path.join(directory, "second.json");
|
||||
expect(await handlers.initialise()).toBe(true);
|
||||
expect(await store.get("checkpoint")).toBeUndefined();
|
||||
await store.set("checkpoint", "second");
|
||||
expect(await handlers.reset()).toBe(true);
|
||||
expect(await store.get("checkpoint")).toBeUndefined();
|
||||
selected = path.join(directory, "first.json");
|
||||
expect(await handlers.initialise()).toBe(true);
|
||||
expect(await store.get("checkpoint")).toEqual(new Set(["first"]));
|
||||
});
|
||||
|
||||
it("does not overwrite newer data when a stale handle closes", async () => {
|
||||
const filePath = path.join(directory, "state.json");
|
||||
const first = compose(filePath);
|
||||
await first.handlers.load();
|
||||
await first.service.kvDB.set("queued-files", ["original.md"]);
|
||||
const second = compose(filePath);
|
||||
await second.handlers.load();
|
||||
await first.service.kvDB.set("queued-files", ["unique-edit.md"]);
|
||||
await second.handlers.close();
|
||||
expect(JSON.parse(fs.readFileSync(filePath, "utf-8"))["queued-files"]).toEqual(["unique-edit.md"]);
|
||||
});
|
||||
|
||||
it.each(["{", "null", "[]"])("refuses corrupt state %s without replacing it", async (contents) => {
|
||||
const filePath = path.join(directory, "state.json");
|
||||
fs.writeFileSync(filePath, contents);
|
||||
const { service, handlers } = compose(filePath);
|
||||
expect(await handlers.load()).toBe(false);
|
||||
expect(() => service.kvDB).toThrow("not initialized");
|
||||
expect(fs.readFileSync(filePath, "utf-8")).toBe(contents);
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,8 @@ import { ServiceContext, type StandardIo } from "@vrtmrz/livesync-commonlib/cont
|
||||
export class NodeServiceContext extends ServiceContext {
|
||||
constructor(
|
||||
readonly databasePath: string,
|
||||
readonly standardIo: StandardIo
|
||||
readonly standardIo: StandardIo,
|
||||
readonly runtimeVaultPath: string = databasePath
|
||||
) {
|
||||
super({ events: eventHub, translate: translateLiveSyncMessage });
|
||||
}
|
||||
|
||||
@@ -24,11 +24,14 @@ import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
|
||||
import { createNodeLocalStorage } from "./NodeLocalStorage";
|
||||
import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService";
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { fs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService";
|
||||
import { PouchDB } from "@/apps/cli/lib/pouchdb-node";
|
||||
import { NodeServiceContext } from "./NodeServiceContext";
|
||||
import { setLang } from "@/common/translation";
|
||||
import { runtimeDatabaseDirectory, runtimeScopeDirectory } from "@/apps/cli/runtimeState";
|
||||
import { createDeletableValueStore } from "@vrtmrz/value-store";
|
||||
import type { SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService";
|
||||
|
||||
export { NodeServiceContext } from "./NodeServiceContext";
|
||||
|
||||
@@ -86,17 +89,31 @@ class NodeUIService<T extends ServiceContext> extends UIService<T> {
|
||||
}
|
||||
}
|
||||
|
||||
class NodeSettingService<T extends NodeServiceContext> extends InjectableSettingService<T> {
|
||||
declare readonly config: InjectableSettingService<T>["config"];
|
||||
|
||||
constructor(context: T, dependencies: SettingServiceDependencies) {
|
||||
super(context, dependencies);
|
||||
const databaseConfig = this.config;
|
||||
const vaultScope = nodePath.basename(runtimeScopeDirectory(context.databasePath, "", context.runtimeVaultPath));
|
||||
this.config = createDeletableValueStore({
|
||||
get: (key) => databaseConfig.getValue(`${key}-${vaultScope}`),
|
||||
set: (key, value) => databaseConfig.setValue(`${key}-${vaultScope}`, value),
|
||||
delete: (key) => databaseConfig.delete(`${key}-${vaultScope}`),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServiceHub<T> {
|
||||
constructor(basePath: string, context: T) {
|
||||
const runtimeDir = nodePath.join(basePath, ".livesync", "runtime");
|
||||
const localStoragePath = nodePath.join(runtimeDir, "local-storage.json");
|
||||
const keyValueDBPath = nodePath.join(runtimeDir, "keyvalue-db.json");
|
||||
|
||||
const API = new HeadlessAPIService<T>(context);
|
||||
const conflict = new InjectableConflictService(context);
|
||||
const fileProcessing = new InjectableFileProcessingService(context);
|
||||
|
||||
const setting = new InjectableSettingService<T>(context, {
|
||||
const setting = new NodeSettingService<T>(context, {
|
||||
APIService: API,
|
||||
onDisplayLanguageChanged: setLang,
|
||||
deviceValues: createNodeLocalStorage(localStoragePath),
|
||||
@@ -118,8 +135,13 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
settingService: setting,
|
||||
APIService: API,
|
||||
});
|
||||
const scopeDirectory = () => runtimeScopeDirectory(basePath, vault.getVaultName(), context.runtimeVaultPath);
|
||||
const test = new InjectableTestService(context);
|
||||
const databaseEvents = new InjectableDatabaseEventService(context);
|
||||
databaseEvents.onResetDatabase.addHandler(async () => {
|
||||
fs.rmSync(runtimeDatabaseDirectory(basePath, vault.getVaultName()), { force: true, recursive: true });
|
||||
return true;
|
||||
});
|
||||
const path = new PathServiceCompat(context, {
|
||||
settingService: setting,
|
||||
});
|
||||
@@ -156,7 +178,7 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
databaseEvents,
|
||||
vault,
|
||||
},
|
||||
keyValueDBPath
|
||||
() => nodePath.join(scopeDirectory(), "keyvalue-db.json")
|
||||
);
|
||||
|
||||
const control = new ControlService(context, {
|
||||
|
||||
Reference in new issue
Block a user