mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-07-26 06:22:58 +00:00
Protect conflict chunks during garbage collection
This commit is contained in:
@@ -5,7 +5,6 @@ import {
|
||||
LOG_LEVEL_NOTICE,
|
||||
LOG_LEVEL_VERBOSE,
|
||||
REMOTE_COUCHDB,
|
||||
REMOTE_P2P,
|
||||
type DocumentID,
|
||||
type EntryDoc,
|
||||
type EntryLeaf,
|
||||
@@ -53,8 +52,7 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
|
||||
name: "Garbage Collection V3 (advanced, beta)",
|
||||
icon: "trash-2",
|
||||
checkCallback: (checking) => {
|
||||
const isApplicableRemote =
|
||||
this.settings.remoteType === REMOTE_COUCHDB || this.settings.remoteType === REMOTE_P2P;
|
||||
const isApplicableRemote = this.settings.remoteType === REMOTE_COUCHDB;
|
||||
if (!this.settings.useEdgeCaseMode || !this._isDatabaseReady() || !isApplicableRemote) {
|
||||
return false;
|
||||
}
|
||||
@@ -464,7 +462,7 @@ Note: **Make sure to synchronise all devices before deletion.**
|
||||
const confirmMessage = `This function deletes unused chunks from the device. If there are differences between devices, some chunks may be missing when resolving conflicts.
|
||||
Be sure to synchronise before executing.
|
||||
|
||||
However, if you have deleted them, you may be able to recover them by performing Hatch -> Recreate missing chunks for all files.
|
||||
If chunks used by current Vault files are deleted, Hatch -> Recreate chunks for current Vault files can recreate them only from files currently present in the Vault. It cannot recover unreadable historical or conflict content.
|
||||
|
||||
Are you ready to delete unused chunks?`;
|
||||
|
||||
@@ -926,41 +924,22 @@ This may indicate that some devices have not completed synchronisation, which co
|
||||
const gcStartTime = Date.now();
|
||||
// Perform Garbage Collection (new implementation).
|
||||
const localDatabase = this.localDatabase.localDatabase;
|
||||
const usedChunks = new Set<DocumentID>();
|
||||
const allChunks = new Map<DocumentID, string>();
|
||||
|
||||
const IDs = this.localDatabase.findEntryNames("", "", {});
|
||||
let i = 0;
|
||||
const doc_count = (await localDatabase.info()).doc_count;
|
||||
for await (const id of IDs) {
|
||||
const doc = await this.localDatabase.getRaw(id as DocumentID);
|
||||
i++;
|
||||
if (i % 100 == 0) {
|
||||
this._notice(`Garbage Collection: Scanned ${i} / ~${doc_count} `, "gc-scanning");
|
||||
}
|
||||
if (!doc) continue;
|
||||
if ("children" in doc) {
|
||||
const children = (doc.children || []) as DocumentID[];
|
||||
for (const chunkId of children) {
|
||||
usedChunks.add(chunkId);
|
||||
}
|
||||
} else if (doc.type === EntryTypes.CHUNK) {
|
||||
allChunks.set(doc._id, doc._rev);
|
||||
}
|
||||
}
|
||||
// Use the revision-aware reachability scan. Reading only winning revisions
|
||||
// would make chunks used exclusively by live conflict branches look unused.
|
||||
const { used: usedChunks, existing: allChunks } = await this.localDatabase.allChunks();
|
||||
this._notice(
|
||||
`Garbage Collection: Scanning completed. Total chunks: ${allChunks.size}, Used chunks: ${usedChunks.size}`,
|
||||
"gc-scanning"
|
||||
);
|
||||
|
||||
const unusedChunks = [...allChunks.keys()].filter((e) => !usedChunks.has(e));
|
||||
const unusedChunks = [...allChunks.entries()].filter(([chunkId]) => !usedChunks.has(chunkId));
|
||||
this._notice(`Garbage Collection: Found ${unusedChunks.length} unused chunks to delete.`, "gc-scanning");
|
||||
const deleteChunkDocs = unusedChunks.map(
|
||||
(chunkId) =>
|
||||
([chunkId, chunk]) =>
|
||||
({
|
||||
_id: chunkId,
|
||||
_id: chunkId as DocumentID,
|
||||
_deleted: true,
|
||||
_rev: allChunks.get(chunkId),
|
||||
_rev: chunk._rev,
|
||||
}) as EntryLeaf
|
||||
);
|
||||
const response = await localDatabase.bulkDocs(deleteChunkDocs);
|
||||
|
||||
@@ -24,7 +24,12 @@ vi.mock("@/common/events", () => ({
|
||||
onEvent: vi.fn(),
|
||||
},
|
||||
}));
|
||||
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
REMOTE_COUCHDB,
|
||||
REMOTE_MINIO,
|
||||
REMOTE_P2P,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte";
|
||||
import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites";
|
||||
|
||||
@@ -87,6 +92,9 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
|
||||
settings.useEdgeCaseMode = true;
|
||||
expect(garbageCollect?.checkCallback?.(true)).toBe(true);
|
||||
|
||||
settings.remoteType = REMOTE_P2P;
|
||||
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
|
||||
|
||||
settings.remoteType = REMOTE_MINIO;
|
||||
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
|
||||
});
|
||||
@@ -176,4 +184,129 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
|
||||
expect(askSelectStringDialogue).not.toHaveBeenCalled();
|
||||
expect(applyPartial).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("describes the current chunk-recreation action without promising historical recovery", async () => {
|
||||
const maintenance = Object.create(LocalDatabaseMaintenance.prototype) as LocalDatabaseMaintenance;
|
||||
const askSelectStringDialogue = vi.fn().mockResolvedValue("Cancel");
|
||||
Object.assign(maintenance, {
|
||||
core: {
|
||||
confirm: {
|
||||
askSelectStringDialogue,
|
||||
},
|
||||
},
|
||||
_log: vi.fn(),
|
||||
});
|
||||
vi.spyOn(maintenance, "ensureAvailable").mockResolvedValue(true);
|
||||
vi.spyOn(maintenance, "trackChanges").mockResolvedValue(undefined);
|
||||
|
||||
await maintenance.performGC();
|
||||
|
||||
const message = vi.mocked(askSelectStringDialogue).mock.calls[0]?.[0] as string;
|
||||
expect(message).toContain("Hatch -> Recreate chunks for current Vault files");
|
||||
expect(message).toContain("only from files currently present in the Vault");
|
||||
expect(message).not.toContain("Recreate missing chunks for all files");
|
||||
});
|
||||
});
|
||||
|
||||
describe("LocalDatabaseMaintenance Garbage Collection V3", () => {
|
||||
it("keeps chunks referenced by a live conflict revision and deletes only unreachable chunks", async () => {
|
||||
const maintenance = Object.create(LocalDatabaseMaintenance.prototype) as LocalDatabaseMaintenance;
|
||||
const pushModes: string[] = [];
|
||||
const deletedChunks: Array<{ _id: string; _rev?: string; _deleted?: boolean }> = [];
|
||||
const allChunks = vi.fn(async () => ({
|
||||
used: new Set(["h:winner", "h:conflict"]),
|
||||
existing: new Map([
|
||||
["h:winner", { _id: "h:winner", _rev: "1-winner", type: "leaf", data: "winner" }],
|
||||
["h:conflict", { _id: "h:conflict", _rev: "1-conflict", type: "leaf", data: "conflict" }],
|
||||
["h:obsolete", { _id: "h:obsolete", _rev: "1-obsolete", type: "leaf", data: "obsolete" }],
|
||||
]),
|
||||
}));
|
||||
const rawDocuments = new Map<string, object>([
|
||||
[
|
||||
"note.md",
|
||||
{
|
||||
_id: "note.md",
|
||||
_rev: "2-winner",
|
||||
_conflicts: ["2-conflict"],
|
||||
type: "plain",
|
||||
children: ["h:winner"],
|
||||
},
|
||||
],
|
||||
["h:winner", { _id: "h:winner", _rev: "1-winner", type: "leaf", data: "winner" }],
|
||||
["h:conflict", { _id: "h:conflict", _rev: "1-conflict", type: "leaf", data: "conflict" }],
|
||||
["h:obsolete", { _id: "h:obsolete", _rev: "1-obsolete", type: "leaf", data: "obsolete" }],
|
||||
]);
|
||||
const findEntryNames = vi.fn(async function* () {
|
||||
yield* rawDocuments.keys();
|
||||
});
|
||||
const getRaw = vi.fn(async (id: string) => rawDocuments.get(id));
|
||||
const localDatabase = {
|
||||
allChunks,
|
||||
localDatabase: {
|
||||
info: vi.fn(async () => ({ doc_count: rawDocuments.size })),
|
||||
bulkDocs: vi.fn(async (docs: Array<{ _id: string; _rev?: string; _deleted?: boolean }>) => {
|
||||
deletedChunks.push(...docs);
|
||||
return docs.map(({ _id }) => ({ ok: true, id: _id, rev: "2-deleted" }));
|
||||
}),
|
||||
},
|
||||
findEntryNames,
|
||||
getRaw,
|
||||
};
|
||||
const replicator = {
|
||||
openOneShotReplication: vi.fn(
|
||||
async (
|
||||
_settings: typeof DEFAULT_SETTINGS,
|
||||
_showResult: boolean,
|
||||
_ignoreCleanLock: boolean,
|
||||
mode: string
|
||||
) => {
|
||||
pushModes.push(mode);
|
||||
return true;
|
||||
}
|
||||
),
|
||||
getConnectedDeviceList: vi.fn(async () => ({
|
||||
accepted_nodes: ["device-a"],
|
||||
node_info: {
|
||||
"device-a": {
|
||||
progress: "10-local",
|
||||
device_name: "Device A",
|
||||
app_version: "1.12.7",
|
||||
plugin_version: "1.0.0-beta.0",
|
||||
},
|
||||
},
|
||||
})),
|
||||
};
|
||||
Object.assign(maintenance, {
|
||||
core: {
|
||||
settings: {
|
||||
...DEFAULT_SETTINGS,
|
||||
remoteType: REMOTE_COUCHDB,
|
||||
},
|
||||
replicator,
|
||||
confirm: {
|
||||
askSelectStringDialogue: vi.fn(async () => "Proceed Garbage Collection"),
|
||||
},
|
||||
},
|
||||
localDatabase,
|
||||
_notice: vi.fn(),
|
||||
});
|
||||
vi.spyOn(maintenance, "ensureAvailable").mockResolvedValue(true);
|
||||
vi.spyOn(maintenance, "compactDatabase").mockResolvedValue(undefined);
|
||||
vi.spyOn(maintenance, "clearHash").mockImplementation(() => undefined);
|
||||
|
||||
await maintenance.gcv3();
|
||||
|
||||
expect(allChunks).toHaveBeenCalledOnce();
|
||||
expect(findEntryNames).not.toHaveBeenCalled();
|
||||
expect(getRaw).not.toHaveBeenCalled();
|
||||
expect(deletedChunks).toEqual([
|
||||
{
|
||||
_id: "h:obsolete",
|
||||
_rev: "1-obsolete",
|
||||
_deleted: true,
|
||||
},
|
||||
]);
|
||||
expect(pushModes).toEqual(["sync", "pushOnly"]);
|
||||
expect(maintenance.compactDatabase).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -187,7 +187,7 @@ export function paneMaintenance(
|
||||
)
|
||||
.addOnUpdate(this.onlyOnMinIO);
|
||||
});
|
||||
void addPanel(paneEl, "Garbage Collection V3 (Beta)", (e) => e, this.onlyOnP2POrCouchDB).then((paneEl) => {
|
||||
void addPanel(paneEl, "Garbage Collection V3 (Beta)", (e) => e, this.onlyOnCouchDB).then((paneEl) => {
|
||||
new Setting(paneEl)
|
||||
.setName("Perform Garbage Collection")
|
||||
.setDesc("Perform Garbage Collection to remove unused chunks and reduce database size.")
|
||||
|
||||
Reference in New Issue
Block a user