Compare commits

...

4 Commits

Author SHA1 Message Date
vorotamoroz
fae0a9d76a bump 2023-04-13 17:33:28 +09:00
vorotamoroz
9a27c9bfe5 - Actions for maintaining databases moved to the 🎛️Maintain databases.
- Clean-up of unreferenced chunks has been implemented on an **experimental**.
2023-04-13 17:33:17 +09:00
vorotamoroz
5e75917b8d bump 2023-04-12 12:08:35 +09:00
vorotamoroz
3322d13b55 - Fixed:
- `Fetch` and `Rebuild database` will work more safely.
- Case-sensitive renaming now works fine.
  Revoked the logic which was made at #130, however, looks fine now.
2023-04-12 12:08:08 +09:00
9 changed files with 355 additions and 239 deletions

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.18.3",
"version": "0.18.5",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.18.3",
"version": "0.18.5",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.18.3",
"version": "0.18.5",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.18.3",
"version": "0.18.5",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",

View File

@@ -5,6 +5,7 @@ import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { askSelectString, askYesNo, askString } from "./utils";
import { decrypt, encrypt } from "./lib/src/e2ee_v2";
import { LiveSyncCommands } from "./LiveSyncCommands";
import { delay } from "./lib/src/utils";
export class SetupLiveSync extends LiveSyncCommands {
onunload() { }
@@ -97,7 +98,8 @@ export class SetupLiveSync extends LiveSyncCommands {
const setupAsNew = "Set it up as secondary or subsequent device";
const setupAgain = "Reconfigure and reconstitute the data";
const setupManually = "Leave everything to me";
newSettingW.syncInternalFiles = false;
newSettingW.usePluginSync = false;
const setupType = await askSelectString(this.app, "How would you like to set it up?", [setupAsNew, setupAgain, setupJustImport, setupManually]);
if (setupType == setupJustImport) {
this.plugin.settings = newSettingW;
@@ -106,11 +108,7 @@ export class SetupLiveSync extends LiveSyncCommands {
} else if (setupType == setupAsNew) {
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.plugin.saveSettings();
await this.plugin.resetLocalDatabase();
await this.plugin.localDatabase.initializeDatabase();
await this.plugin.markRemoteResolved();
await this.plugin.replicate(true);
await this.fetchLocal();
} else if (setupType == setupAgain) {
const confirm = "I know this operation will rebuild all my databases with files on this device, and files that are on the remote database and I didn't synchronize to any other devices will be lost and want to proceed indeed.";
if (await askSelectString(this.app, "Do you really want to do this?", ["Cancel", confirm]) != confirm) {
@@ -118,15 +116,7 @@ export class SetupLiveSync extends LiveSyncCommands {
}
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.plugin.saveSettings();
await this.plugin.resetLocalDatabase();
await this.plugin.localDatabase.initializeDatabase();
await this.plugin.initializeDatabase(true);
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.markRemoteResolved();
await this.plugin.replicate(true);
await this.rebuildEverything();
} else if (setupType == setupManually) {
const keepLocalDB = await askYesNo(this.app, "Keep local DB?");
const keepRemoteDB = await askYesNo(this.app, "Keep remote DB?");
@@ -134,6 +124,8 @@ export class SetupLiveSync extends LiveSyncCommands {
// nothing to do. so peaceful.
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
this.suspendAllSync();
this.suspendExtraSync();
await this.plugin.saveSettings();
const replicate = await askYesNo(this.app, "Unlock and replicate?");
if (replicate == "yes") {
@@ -189,4 +181,52 @@ export class SetupLiveSync extends LiveSyncCommands {
Logger("Couldn't parse or decrypt configuration uri.", LOG_LEVEL.NOTICE);
}
}
suspendExtraSync() {
Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
this.plugin.settings.syncInternalFiles = false;
this.plugin.settings.usePluginSync = false;
this.plugin.settings.autoSweepPlugins = false;
}
suspendAllSync() {
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncAfterMerge = false;
//this.suspendExtraSync();
}
async fetchLocal() {
this.suspendExtraSync();
await this.plugin.realizeSettingSyncMode();
await this.plugin.resetLocalDatabase();
await delay(1000);
await this.plugin.markRemoteResolved();
await this.plugin.openDatabase();
this.plugin.isReady = true;
await delay(500);
await this.plugin.replicateAllFromServer(true);
}
async rebuildRemote() {
this.suspendExtraSync();
await this.plugin.realizeSettingSyncMode();
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await delay(500);
await this.plugin.replicateAllToServer(true);
}
async rebuildEverything() {
this.suspendExtraSync();
await this.plugin.realizeSettingSyncMode();
await this.plugin.resetLocalDatabase();
await delay(1000);
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await delay(500);
await this.plugin.replicateAllToServer(true);
}
}

View File

@@ -1,5 +1,5 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, ConfigPassphraseStore, RemoteDBSettings } from "./lib/src/types";
import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, ConfigPassphraseStore, RemoteDBSettings, NewEntry } from "./lib/src/types";
import { delay } from "./lib/src/utils";
import { Semaphore } from "./lib/src/semaphore";
import { versionNumberString2Number } from "./lib/src/strbin";
@@ -8,22 +8,25 @@ import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
import { testCrypt } from "./lib/src/e2ee_v2";
import ObsidianLiveSyncPlugin from "./main";
const requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, key?: string, body?: string) => {
const _requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, path?: string, body?: any, method?: string) => {
const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
const encoded = window.btoa(utf8str);
const authHeader = "Basic " + encoded;
// const origin = "capacitor://localhost";
const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin };
const uri = `${baseUri}/_node/_local/_config${key ? "/" + key : ""}`;
const uri = `${baseUri}/${path}`;
const requestParam: RequestUrlParam = {
url: uri,
method: body ? "PUT" : "GET",
method: method || (body ? "PUT" : "GET"),
headers: transformedHeaders,
contentType: "application/json",
body: body ? JSON.stringify(body) : undefined,
};
return await requestUrl(requestParam);
}
const requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, key?: string, body?: string, method?: string) => {
const uri = `_node/_local/_config${key ? "/" + key : ""}`;
return await _requestToCouchDB(baseUri, username, password, origin, uri, body, method);
};
export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
plugin: ObsidianLiveSyncPlugin;
@@ -74,7 +77,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
<label class='sls-setting-label c-40'><input type='radio' name='disp' value='40' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🔧</div></label>
<label class='sls-setting-label c-50 wizardHidden'><input type='radio' name='disp' value='50' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🧰</div></label>
<label class='sls-setting-label c-60 wizardHidden'><input type='radio' name='disp' value='60' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🔌</div></label>
<!-- <label class='sls-setting-label c-70 wizardHidden'><input type='radio' name='disp' value='70' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🚑</div></label>-->
<label class='sls-setting-label c-70 wizardHidden'><input type='radio' name='disp' value='70' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🎛️</div></label>
`;
const menuTabs = w.querySelectorAll(".sls-setting-label");
const changeDisplay = (screen: string) => {
@@ -420,25 +423,16 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (!encrypt) {
passphrase = "";
}
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncAfterMerge = false;
this.plugin.addOnSetup.suspendAllSync();
this.plugin.addOnSetup.suspendExtraSync();
this.plugin.settings.encrypt = encrypt;
this.plugin.settings.passphrase = passphrase;
this.plugin.settings.useDynamicIterationCount = useDynamicIterationCount;
this.plugin.settings.usePathObfuscation = usePathObfuscation;
await this.plugin.saveSettings();
markDirtyControl();
if (sendToServer) {
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
await this.plugin.addOnSetup.rebuildRemote()
} else {
await this.plugin.markRemoteResolved();
await this.plugin.replicate(true);
@@ -489,78 +483,30 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (!encrypt) {
passphrase = "";
}
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncAfterMerge = false;
this.plugin.settings.syncInternalFiles = false;
this.plugin.settings.usePluginSync = false;
this.plugin.addOnSetup.suspendAllSync();
this.plugin.addOnSetup.suspendExtraSync();
this.plugin.settings.encrypt = encrypt;
this.plugin.settings.passphrase = passphrase;
this.plugin.settings.useDynamicIterationCount = useDynamicIterationCount;
this.plugin.settings.usePathObfuscation = usePathObfuscation;
Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
Logger("All synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
await this.plugin.saveSettings();
markDirtyControl();
applyDisplayEnabled();
// @ts-ignore
this.plugin.app.setting.close()
this.plugin.app.setting.close();
await delay(2000);
if (method == "localOnly") {
await this.plugin.resetLocalDatabase();
await delay(1000);
await this.plugin.markRemoteResolved();
await this.plugin.openDatabase();
this.plugin.isReady = true;
await this.plugin.replicateAllFromServer(true);
await this.plugin.addOnSetup.fetchLocal();
}
if (method == "remoteOnly") {
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
await this.plugin.addOnSetup.rebuildRemote();
}
if (method == "rebuildBothByThisDevice") {
await this.plugin.resetLocalDatabase();
await delay(1000);
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
await this.plugin.addOnSetup.rebuildEverything();
}
}
new Setting(containerRemoteDatabaseEl)
.setName("Overwrite remote database")
.setDesc("Overwrite remote database with local DB and passphrase.")
.setClass("wizardHidden")
.addButton((button) =>
button
.setButtonText("Send")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("remoteOnly");
})
)
new Setting(containerRemoteDatabaseEl)
.setName("Rebuild everything")
.setDesc("Rebuild local and remote database with local files.")
.setClass("wizardHidden")
.addButton((button) =>
button
.setButtonText("Rebuild")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("rebuildBothByThisDevice");
})
)
new Setting(containerRemoteDatabaseEl)
.setName("Test Database Connection")
.setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.")
@@ -744,19 +690,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text: "",
});
new Setting(containerRemoteDatabaseEl)
.setName("Lock remote database")
.setDesc("Lock remote database to prevent synchronization with other devices.")
.setClass("wizardHidden")
.addButton((button) =>
button
.setButtonText("Lock")
.setDisabled(false)
.setWarning()
.onClick(async () => {
await this.plugin.markRemoteLocked();
})
);
let rebuildRemote = false;
new Setting(containerRemoteDatabaseEl)
@@ -821,21 +754,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
})
);
new Setting(containerLocalDatabaseEl)
.setName("Fetch rebuilt DB")
.setDesc("Restore or reconstruct local database from remote database.")
.setClass("wizardHidden")
.addButton((button) =>
button
.setButtonText("Fetch")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("localOnly");
})
)
let newDatabaseName = this.plugin.settings.additionalSuffixOfDatabaseName + "";
new Setting(containerLocalDatabaseEl)
.setName("Database suffix")
@@ -1149,32 +1067,42 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.addButton((button) => {
button.setButtonText("Merge")
.onClick(async () => {
this.plugin.settings.syncInternalFiles = true;
this.plugin.addOnSetup.suspendExtraSync();
this.display();
Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL.NOTICE);
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("safe", true);
this.plugin.settings.syncInternalFiles = true;
await this.plugin.saveSettings();
this.display();
Logger(`Done!`, LOG_LEVEL.NOTICE);
})
})
.addButton((button) => {
button.setButtonText("Fetch")
.onClick(async () => {
this.plugin.settings.syncInternalFiles = true;
this.display();
this.plugin.addOnSetup.suspendExtraSync();
// @ts-ignore
this.plugin.app.setting.close()
Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL.NOTICE);
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pullForce", true);
this.plugin.settings.syncInternalFiles = true;
await this.plugin.saveSettings();
Logger(`Restarting the app is strongly recommended!`, LOG_LEVEL.NOTICE);
this.display();
Logger(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL.NOTICE);
// this.display();
})
})
.addButton((button) => {
button.setButtonText("Overwrite")
.onClick(async () => {
this.plugin.settings.syncInternalFiles = true;
this.display();
this.plugin.addOnSetup.suspendExtraSync();
// @ts-ignore
this.plugin.app.setting.close()
Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL.NOTICE);
// this.display();
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pushForce", true);
this.plugin.settings.syncInternalFiles = true;
await this.plugin.saveSettings();
this.display();
Logger(`Done!`, LOG_LEVEL.NOTICE);
// this.display();
})
});
}
@@ -1714,18 +1642,59 @@ ${stringifyYaml(pluginConfig)}`;
})
);
new Setting(containerHatchEl)
.setName("Discard local database to reset or uninstall Self-hosted LiveSync")
.addButton((button) =>
button
.setButtonText("Discard")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase();
})
);
const localDatabaseCleanUp = async (force: boolean) => {
const usedMap = new Map();
const existMap = new Map();
const db = this.plugin.localDatabase.localDatabase;
if ((db as any)?.adapter != "indexeddb") {
if (force) {
Logger("Fetch from the remote database", LOG_LEVEL.NOTICE, "clean-up-db");
await rebuildDB("localOnly");
return;
} else {
Logger("This feature requires enabling `Use new adapter`. Please enable it", LOG_LEVEL.NOTICE, "clean-up-db");
return;
}
}
Logger(`The remote database locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
const xx = await db.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
for (const xxd of xx.rows) {
const chunk = xxd.id
existMap.set(chunk, xxd.value.rev);
}
const x = await db.find({ limit: 999999999, selector: { children: { $exists: true, $type: "array" } }, fields: ["_id", "mtime", "children"] });
for (const temp of x.docs) {
for (const chunk of (temp as NewEntry).children) {
usedMap.set(chunk, (usedMap.has(chunk) ? usedMap.get(chunk) : 0) + 1);
existMap.delete(chunk);
}
}
const payload = {} as Record<string, string[]>;
for (const [id, rev] of existMap) {
payload[id] = [rev];
}
const removeItems = Object.keys(payload).length;
if (removeItems == 0) {
Logger(`No unreferenced chunks found`, LOG_LEVEL.NOTICE, "clean-up-db");
return;
}
Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
for (const [id, rev] of existMap) {
//@ts-ignore
const ret = await db.purge(id, rev);
Logger(ret, LOG_LEVEL.VERBOSE);
}
this.plugin.localDatabase.refreshSettings();
Logger(`Compacting local database...`, LOG_LEVEL.NOTICE, "clean-up-db");
await db.compact();
Logger("Done!", LOG_LEVEL.NOTICE, "clean-up-db");
}
addScreenElement("50", containerHatchEl);
// With great respect, thank you TfTHacker!
@@ -1803,73 +1772,171 @@ ${stringifyYaml(pluginConfig)}`;
addScreenElement("60", containerPluginSettings);
// const containerCorruptedDataEl = containerEl.createDiv();
const containerMaintenanceEl = containerEl.createDiv();
// containerCorruptedDataEl.createEl("h3", { text: "Corrupted or missing data" });
// containerCorruptedDataEl.createEl("h4", { text: "Corrupted" });
// if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {
// const cx = containerCorruptedDataEl.createEl("div", { text: "If you have a copy of these files on any device, simply edit them once and sync. If not, there's nothing we can do except deleting them. sorry.." });
// for (const k in this.plugin.localDatabase.corruptedEntries) {
// const xx = cx.createEl("div", { text: `${k}` });
containerMaintenanceEl.createEl("h3", { text: "Maintain databases" });
// const ba = xx.createEl("button", { text: `Delete this` }, (e) => {
// e.addEventListener("click", async () => {
// await this.plugin.localDatabase.deleteDBEntry(k as string as FilePathWithPrefix /* should be explained */);
// xx.remove();
// });
// });
// ba.addClass("mod-warning");
// //TODO: FIX LATER
// // xx.createEl("button", { text: `Restore from file` }, (e) => {
// // e.addEventListener("click", async () => {
// // const f = await this.app.vault.getFiles().filter((e) => this.plugin.path2id(e.path) == k);
// // if (f.length == 0) {
// // Logger("Not found in vault", LOG_LEVEL.NOTICE);
// // return;
// // }
// // await this.plugin.updateIntoDB(f[0]);
// // xx.remove();
// // });
// // });
// // xx.addClass("mod-warning");
// }
// } else {
// containerCorruptedDataEl.createEl("div", { text: "There is no corrupted data." });
// }
// containerCorruptedDataEl.createEl("h4", { text: "Missing or waiting" });
// if (Object.keys(this.plugin.queuedFiles).length > 0) {
// const cx = containerCorruptedDataEl.createEl("div", {
// text: "These files have missing or waiting chunks. Perhaps these chunks will arrive in a while after replication. But if they don't, you have to restore it's database entry from a existing local file by hitting the button below.",
// });
// const files = [...new Set([...this.plugin.queuedFiles.map((e) => e.entry._id)])];
// for (const k of files) {
// const xx = cx.createEl("div", { text: `${this.plugin.id2path(k)}` });
containerMaintenanceEl.createEl("h4", { text: "The remote database" });
// const ba = xx.createEl("button", { text: `Delete this` }, (e) => {
// e.addEventListener("click", async () => {
// await this.plugin.localDatabase.deleteDBEntry(k);
// xx.remove();
// });
// });
// ba.addClass("mod-warning");
// xx.createEl("button", { text: `Restore from file` }, (e) => {
// e.addEventListener("click", async () => {
// const f = await this.app.vault.getFiles().filter((e) => this.plugin.path2id(e.path) == k);
// if (f.length == 0) {
// Logger("Not found in vault", LOG_LEVEL.NOTICE);
// return;
// }
// await this.plugin.updateIntoDB(f[0]);
// xx.remove();
// });
// });
// xx.addClass("mod-warning");
// }
// } else {
// containerCorruptedDataEl.createEl("div", { text: "There is no missing or waiting chunk." });
// }
// applyDisplayEnabled();
// addScreenElement("70", containerCorruptedDataEl);
new Setting(containerMaintenanceEl)
.setName("Lock remote database")
.setDesc("Lock remote database to prevent synchronization with other devices.")
.addButton((button) =>
button
.setButtonText("Lock")
.setDisabled(false)
.setWarning()
.onClick(async () => {
await this.plugin.markRemoteLocked();
})
);
new Setting(containerMaintenanceEl)
.setName("Overwrite remote database")
.setDesc("Overwrite remote database with local DB and passphrase.")
.addButton((button) =>
button
.setButtonText("Send")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("remoteOnly");
})
)
new Setting(containerMaintenanceEl)
.setName("(Experimental) Clean the remote database")
.setDesc("")
.addButton((button) =>
button.setButtonText("Perform cleaning")
.setDisabled(false)
.setWarning()
.onClick(async () => {
// @ts-ignore
this.plugin.app.setting.close()
try {
const usedMap = new Map();
const existMap = new Map();
const ret = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile);
if (typeof ret === "string") {
Logger(`Connect error: ${ret}`, LOG_LEVEL.NOTICE, "clean-up-db");
return;
}
const info = ret.info;
Logger(JSON.stringify(info), LOG_LEVEL.VERBOSE, "clean-up-db");
Logger(`Database data-size:${(info as any)?.data_size ?? "-"}, disk-size: ${(info as any)?.disk_size ?? "-"}`);
Logger(`The remote database locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
await this.plugin.markRemoteLocked();
Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
const db = ret.db;
const xx = await db.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
for (const xxd of xx.rows) {
const chunk = xxd.id
existMap.set(chunk, xxd.value.rev);
}
const x = await db.find({ limit: 999999999, selector: { children: { $exists: true, $type: "array" } }, fields: ["_id", "mtime", "children"] });
for (const temp of x.docs) {
for (const chunk of (temp as NewEntry).children) {
usedMap.set(chunk, (usedMap.has(chunk) ? usedMap.get(chunk) : 0) + 1);
existMap.delete(chunk);
}
}
const payload = {} as Record<string, string[]>;
for (const [id, rev] of existMap) {
payload[id] = [rev];
}
const removeItems = Object.keys(payload).length;
if (removeItems == 0) {
Logger(`No unreferenced chunk found`, LOG_LEVEL.NOTICE, "clean-up-db");
return;
}
Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
const rets = await _requestToCouchDB(
`${this.plugin.settings.couchDB_URI}/${this.plugin.settings.couchDB_DBNAME}`,
this.plugin.settings.couchDB_USER,
this.plugin.settings.couchDB_PASSWORD,
undefined,
"_purge",
payload, "POST");
// const result = await rets();
Logger(JSON.stringify(rets.text), LOG_LEVEL.VERBOSE);
Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
await db.compact();
const endInfo = await db.info();
Logger(`Result database data-size:${(endInfo as any)?.data_size ?? "-"}, disk-size: ${(endInfo as any)?.disk_size ?? "-"}`);
Logger(JSON.stringify(endInfo), LOG_LEVEL.VERBOSE, "clean-up-db");
Logger(`Local database cleaning up...`);
await localDatabaseCleanUp(true);
} catch (ex) {
Logger("Failed to clean up db.")
Logger(ex, LOG_LEVEL.VERBOSE);
}
})
);
containerMaintenanceEl.createEl("h4", { text: "The local database" });
new Setting(containerMaintenanceEl)
.setName("Fetch rebuilt DB")
.setDesc("Restore or reconstruct local database from remote database.")
.addButton((button) =>
button
.setButtonText("Fetch")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("localOnly");
})
)
new Setting(containerMaintenanceEl)
.setName("(Experimental) Clean the local database")
.setDesc("")
.addButton((button) =>
button.setButtonText("Perform cleaning")
.setDisabled(false)
.setWarning()
.onClick(async () => {
// @ts-ignore
this.plugin.app.setting.close()
await localDatabaseCleanUp(false);
await this.plugin.markRemoteResolved();
})
);
new Setting(containerMaintenanceEl)
.setName("Discard local database to reset or uninstall Self-hosted LiveSync")
.addButton((button) =>
button
.setButtonText("Discard")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase();
})
);
containerMaintenanceEl.createEl("h4", { text: "Both databases" });
new Setting(containerMaintenanceEl)
.setName("Rebuild everything")
.setDesc("Rebuild local and remote database with local files.")
.addButton((button) =>
button
.setButtonText("Rebuild")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("rebuildBothByThisDevice");
})
)
applyDisplayEnabled();
addScreenElement("70", containerMaintenanceEl);
applyDisplayEnabled();
if (this.selectedScreen == "") {

View File

@@ -57,8 +57,8 @@ export class StorageEventManagerObsidian extends StorageEventManager {
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
if (file instanceof TFile) {
this.appendWatchEvent([
{ type: "CREATE", file },
{ type: "DELETE", file: { path: oldFile as FilePath, mtime: file.stat.mtime, ctime: file.stat.ctime, size: file.stat.size, deleted: true } },
{ type: "CREATE", file },
], ctx);
}
}

View File

@@ -4,7 +4,7 @@ import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "di
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, RequestUrlParam, RequestUrlResponse, requestUrl } from "./deps";
import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, DatabaseConnectingStatus, EntryHasPath, DocumentID, FilePathWithPrefix, FilePath, AnyEntry } from "./lib/src/types";
import { InternalFileInfo, queueItem, CacheData, FileEventItem, FileWatchEventQueueMax } from "./types";
import { delay, getDocData, isDocContentSame } from "./lib/src/utils";
import { getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { LogDisplayModal } from "./LogDisplayModal";
@@ -387,25 +387,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin
try {
if (this.isRedFlagRaised() || this.isRedFlag2Raised() || this.isRedFlag3Raised()) {
this.settings.batchSave = false;
this.settings.liveSync = false;
this.settings.periodicReplication = false;
this.settings.syncOnSave = false;
this.settings.syncOnStart = false;
this.settings.syncOnFileOpen = false;
this.settings.syncAfterMerge = false;
this.settings.autoSweepPlugins = false;
this.settings.usePluginSync = false;
this.addOnSetup.suspendAllSync();
this.addOnSetup.suspendExtraSync();
this.settings.suspendFileWatching = true;
this.settings.syncInternalFiles = false;
await this.saveSettings();
if (this.isRedFlag2Raised()) {
Logger(`${FLAGMD_REDFLAG2} has been detected! Self-hosted LiveSync suspends all sync and rebuild everything.`, LOG_LEVEL.NOTICE);
await this.resetLocalDatabase();
await this.initializeDatabase(true);
await this.markRemoteLocked();
await this.tryResetRemoteDatabase();
await this.markRemoteLocked();
await this.replicateAllToServer(true);
await this.addOnSetup.rebuildEverything();
await this.deleteRedFlag2();
if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
this.settings.suspendFileWatching = false;
@@ -415,12 +403,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
}
} else if (this.isRedFlag3Raised()) {
Logger(`${FLAGMD_REDFLAG3} has been detected! Self-hosted LiveSync will discard the local database and fetch everything from the remote once again.`, LOG_LEVEL.NOTICE);
await this.resetLocalDatabase();
await delay(1000);
await this.markRemoteResolved();
await this.openDatabase();
this.isReady = true;
await this.replicateAllFromServer(true);
await this.addOnSetup.fetchLocal();
await this.deleteRedFlag3();
if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
this.settings.suspendFileWatching = false;
@@ -912,8 +895,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin
const file = queue.args.file;
const key = `file-last-proc-${queue.type}-${file.path}`;
const last = Number(await this.kvDB.get(key) || 0);
let mtime = file.mtime;
if (queue.type == "DELETE") {
await this.deleteFromDBbyPath(file.path);
mtime = file.mtime - 1;
const keyD1 = `file-last-proc-CREATE-${file.path}`;
const keyD2 = `file-last-proc-CHANGED-${file.path}`;
await this.kvDB.set(keyD1, mtime);
await this.kvDB.set(keyD2, mtime);
} else if (queue.type == "INTERNAL") {
await this.addOnHiddenFileSync.watchVaultRawEventsAsync(file.path);
} else {
@@ -930,6 +919,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin
const cache = queue.args.cache;
if (queue.type == "CREATE" || queue.type == "CHANGED") {
const keyD1 = `file-last-proc-DELETED-${file.path}`;
await this.kvDB.set(keyD1, mtime);
if (!await this.updateIntoDB(targetFile, false, cache)) {
Logger(`DB -> STORAGE: failed, cancel the relative operations: ${targetFile.path}`, LOG_LEVEL.INFO);
// cancel running queues and remove one of atomic operation
@@ -942,7 +933,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
await this.watchVaultRenameAsync(targetFile, queue.args.oldPath);
}
}
await this.kvDB.set(key, file.mtime);
await this.kvDB.set(key, mtime);
} while (this.vaultManager.getQueueLength() > 0);
return true;
})
@@ -1114,13 +1105,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
// This occurs not only when files are deleted, but also when conflicts are resolved.
// We have to check no other revisions are left.
const lastDocs = await this.localDatabase.getDBEntry(path);
if (path != file.path) {
Logger(`delete skipped: ${file.path} :Not exactly matched`, LOG_LEVEL.VERBOSE);
}
if (lastDocs === false) {
await this.deleteVaultItem(file);
} else {
// it perhaps delete some revisions.
// may be we have to reload this
await this.pullFile(path, null, true);
Logger(`delete skipped:${lastDocs._id}`, LOG_LEVEL.VERBOSE);
Logger(`delete skipped:${file.path}`, LOG_LEVEL.VERBOSE);
}
return;
}
@@ -1381,8 +1375,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
//---> Sync
async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): Promise<void> {
const docsSorted = docs.sort((a, b) => b.mtime - a.mtime);
L1:
for (const change of docs) {
for (const change of docsSorted) {
for (const proc of this.addOns) {
if (await proc.parseReplicationResultItem(change)) {
continue L1;

View File

@@ -325,14 +325,16 @@ export function isValidPath(filename: string) {
let touchedFiles: string[] = [];
export function getAbstractFileByPath(path: FilePath): TAbstractFile | null {
// Hidden API but so useful.
// @ts-ignore
if ("getAbstractFileByPathInsensitive" in app.vault && (app.vault.adapter?.insensitive ?? false)) {
// @ts-ignore
return app.vault.getAbstractFileByPathInsensitive(path);
} else {
return app.vault.getAbstractFileByPath(path);
}
// Disabled temporary.
return app.vault.getAbstractFileByPath(path);
// // Hidden API but so useful.
// // @ts-ignore
// if ("getAbstractFileByPathInsensitive" in app.vault && (app.vault.adapter?.insensitive ?? false)) {
// // @ts-ignore
// return app.vault.getAbstractFileByPathInsensitive(path);
// } else {
// return app.vault.getAbstractFileByPath(path);
// }
}
export function trimPrefix(target: string, prefix: string) {
return target.startsWith(prefix) ? target.substring(prefix.length) : target;

View File

@@ -20,6 +20,18 @@ Note: **When changing this configuration, we need to rebuild both of the local a
- No longer `Touch hidden files`.
- 0.18.3
- Fixed Pop-up is now correctly shown after hidden file synchronisation.
- 0.18.4
- Fixed:
- `Fetch` and `Rebuild database` will work more safely.
- Case-sensitive renaming now works fine.
Revoked the logic which was made at #130, however, looks fine now.
- 0.18.5
- Improved:
- Actions for maintaining databases moved to the `🎛Maintain databases`.
- Clean-up of unreferenced chunks has been implemented on an **experimental**.
- This feature requires enabling `Use new adapter`.
- Be sure to fully all devices synchronised before perform it.
- After cleaning up the remote, all devices will be locked out. If we are sure had it be synchronised, we can perform only cleaning-up locally. If not, we have to perform `Fetch`.
### 0.17.0
- 0.17.0 has no surfaced changes but the design of saving chunks has been changed. They have compatibility but changing files after upgrading makes different chunks than before 0.16.x.