Protect conflict chunks during garbage collection

This commit is contained in:
vorotamoroz
2026-07-24 17:09:22 +00:00
parent 07cba4ce83
commit 20fb027055
9 changed files with 220 additions and 36 deletions
@@ -5,7 +5,6 @@ import {
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
REMOTE_COUCHDB,
REMOTE_P2P,
type DocumentID,
type EntryDoc,
type EntryLeaf,
@@ -53,8 +52,7 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
name: "Garbage Collection V3 (advanced, beta)",
icon: "trash-2",
checkCallback: (checking) => {
const isApplicableRemote =
this.settings.remoteType === REMOTE_COUCHDB || this.settings.remoteType === REMOTE_P2P;
const isApplicableRemote = this.settings.remoteType === REMOTE_COUCHDB;
if (!this.settings.useEdgeCaseMode || !this._isDatabaseReady() || !isApplicableRemote) {
return false;
}
@@ -464,7 +462,7 @@ Note: **Make sure to synchronise all devices before deletion.**
const confirmMessage = `This function deletes unused chunks from the device. If there are differences between devices, some chunks may be missing when resolving conflicts.
Be sure to synchronise before executing.
However, if you have deleted them, you may be able to recover them by performing Hatch -> Recreate missing chunks for all files.
If chunks used by current Vault files are deleted, Hatch -> Recreate chunks for current Vault files can recreate them only from files currently present in the Vault. It cannot recover unreadable historical or conflict content.
Are you ready to delete unused chunks?`;
@@ -926,41 +924,22 @@ This may indicate that some devices have not completed synchronisation, which co
const gcStartTime = Date.now();
// Perform Garbage Collection (new implementation).
const localDatabase = this.localDatabase.localDatabase;
const usedChunks = new Set<DocumentID>();
const allChunks = new Map<DocumentID, string>();
const IDs = this.localDatabase.findEntryNames("", "", {});
let i = 0;
const doc_count = (await localDatabase.info()).doc_count;
for await (const id of IDs) {
const doc = await this.localDatabase.getRaw(id as DocumentID);
i++;
if (i % 100 == 0) {
this._notice(`Garbage Collection: Scanned ${i} / ~${doc_count} `, "gc-scanning");
}
if (!doc) continue;
if ("children" in doc) {
const children = (doc.children || []) as DocumentID[];
for (const chunkId of children) {
usedChunks.add(chunkId);
}
} else if (doc.type === EntryTypes.CHUNK) {
allChunks.set(doc._id, doc._rev);
}
}
// Use the revision-aware reachability scan. Reading only winning revisions
// would make chunks used exclusively by live conflict branches look unused.
const { used: usedChunks, existing: allChunks } = await this.localDatabase.allChunks();
this._notice(
`Garbage Collection: Scanning completed. Total chunks: ${allChunks.size}, Used chunks: ${usedChunks.size}`,
"gc-scanning"
);
const unusedChunks = [...allChunks.keys()].filter((e) => !usedChunks.has(e));
const unusedChunks = [...allChunks.entries()].filter(([chunkId]) => !usedChunks.has(chunkId));
this._notice(`Garbage Collection: Found ${unusedChunks.length} unused chunks to delete.`, "gc-scanning");
const deleteChunkDocs = unusedChunks.map(
(chunkId) =>
([chunkId, chunk]) =>
({
_id: chunkId,
_id: chunkId as DocumentID,
_deleted: true,
_rev: allChunks.get(chunkId),
_rev: chunk._rev,
}) as EntryLeaf
);
const response = await localDatabase.bulkDocs(deleteChunkDocs);
@@ -24,7 +24,12 @@ vi.mock("@/common/events", () => ({
onEvent: vi.fn(),
},
}));
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO } from "@vrtmrz/livesync-commonlib/compat/common/types";
import {
DEFAULT_SETTINGS,
REMOTE_COUCHDB,
REMOTE_MINIO,
REMOTE_P2P,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte";
import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites";
@@ -87,6 +92,9 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
settings.useEdgeCaseMode = true;
expect(garbageCollect?.checkCallback?.(true)).toBe(true);
settings.remoteType = REMOTE_P2P;
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
settings.remoteType = REMOTE_MINIO;
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
});
@@ -176,4 +184,129 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
expect(askSelectStringDialogue).not.toHaveBeenCalled();
expect(applyPartial).not.toHaveBeenCalled();
});
it("describes the current chunk-recreation action without promising historical recovery", async () => {
const maintenance = Object.create(LocalDatabaseMaintenance.prototype) as LocalDatabaseMaintenance;
const askSelectStringDialogue = vi.fn().mockResolvedValue("Cancel");
Object.assign(maintenance, {
core: {
confirm: {
askSelectStringDialogue,
},
},
_log: vi.fn(),
});
vi.spyOn(maintenance, "ensureAvailable").mockResolvedValue(true);
vi.spyOn(maintenance, "trackChanges").mockResolvedValue(undefined);
await maintenance.performGC();
const message = vi.mocked(askSelectStringDialogue).mock.calls[0]?.[0] as string;
expect(message).toContain("Hatch -> Recreate chunks for current Vault files");
expect(message).toContain("only from files currently present in the Vault");
expect(message).not.toContain("Recreate missing chunks for all files");
});
});
describe("LocalDatabaseMaintenance Garbage Collection V3", () => {
it("keeps chunks referenced by a live conflict revision and deletes only unreachable chunks", async () => {
const maintenance = Object.create(LocalDatabaseMaintenance.prototype) as LocalDatabaseMaintenance;
const pushModes: string[] = [];
const deletedChunks: Array<{ _id: string; _rev?: string; _deleted?: boolean }> = [];
const allChunks = vi.fn(async () => ({
used: new Set(["h:winner", "h:conflict"]),
existing: new Map([
["h:winner", { _id: "h:winner", _rev: "1-winner", type: "leaf", data: "winner" }],
["h:conflict", { _id: "h:conflict", _rev: "1-conflict", type: "leaf", data: "conflict" }],
["h:obsolete", { _id: "h:obsolete", _rev: "1-obsolete", type: "leaf", data: "obsolete" }],
]),
}));
const rawDocuments = new Map<string, object>([
[
"note.md",
{
_id: "note.md",
_rev: "2-winner",
_conflicts: ["2-conflict"],
type: "plain",
children: ["h:winner"],
},
],
["h:winner", { _id: "h:winner", _rev: "1-winner", type: "leaf", data: "winner" }],
["h:conflict", { _id: "h:conflict", _rev: "1-conflict", type: "leaf", data: "conflict" }],
["h:obsolete", { _id: "h:obsolete", _rev: "1-obsolete", type: "leaf", data: "obsolete" }],
]);
const findEntryNames = vi.fn(async function* () {
yield* rawDocuments.keys();
});
const getRaw = vi.fn(async (id: string) => rawDocuments.get(id));
const localDatabase = {
allChunks,
localDatabase: {
info: vi.fn(async () => ({ doc_count: rawDocuments.size })),
bulkDocs: vi.fn(async (docs: Array<{ _id: string; _rev?: string; _deleted?: boolean }>) => {
deletedChunks.push(...docs);
return docs.map(({ _id }) => ({ ok: true, id: _id, rev: "2-deleted" }));
}),
},
findEntryNames,
getRaw,
};
const replicator = {
openOneShotReplication: vi.fn(
async (
_settings: typeof DEFAULT_SETTINGS,
_showResult: boolean,
_ignoreCleanLock: boolean,
mode: string
) => {
pushModes.push(mode);
return true;
}
),
getConnectedDeviceList: vi.fn(async () => ({
accepted_nodes: ["device-a"],
node_info: {
"device-a": {
progress: "10-local",
device_name: "Device A",
app_version: "1.12.7",
plugin_version: "1.0.0-beta.0",
},
},
})),
};
Object.assign(maintenance, {
core: {
settings: {
...DEFAULT_SETTINGS,
remoteType: REMOTE_COUCHDB,
},
replicator,
confirm: {
askSelectStringDialogue: vi.fn(async () => "Proceed Garbage Collection"),
},
},
localDatabase,
_notice: vi.fn(),
});
vi.spyOn(maintenance, "ensureAvailable").mockResolvedValue(true);
vi.spyOn(maintenance, "compactDatabase").mockResolvedValue(undefined);
vi.spyOn(maintenance, "clearHash").mockImplementation(() => undefined);
await maintenance.gcv3();
expect(allChunks).toHaveBeenCalledOnce();
expect(findEntryNames).not.toHaveBeenCalled();
expect(getRaw).not.toHaveBeenCalled();
expect(deletedChunks).toEqual([
{
_id: "h:obsolete",
_rev: "1-obsolete",
_deleted: true,
},
]);
expect(pushModes).toEqual(["sync", "pushOnly"]);
expect(maintenance.compactDatabase).toHaveBeenCalledOnce();
});
});
@@ -187,7 +187,7 @@ export function paneMaintenance(
)
.addOnUpdate(this.onlyOnMinIO);
});
void addPanel(paneEl, "Garbage Collection V3 (Beta)", (e) => e, this.onlyOnP2POrCouchDB).then((paneEl) => {
void addPanel(paneEl, "Garbage Collection V3 (Beta)", (e) => e, this.onlyOnCouchDB).then((paneEl) => {
new Setting(paneEl)
.setName("Perform Garbage Collection")
.setDesc("Perform Garbage Collection to remove unused chunks and reduce database size.")