From fddc466b0fbfd3d390c70079d6ea03c25c9dd3f9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 19 Jul 2022 17:57:29 +0900 Subject: [PATCH] New Feature - Hidden file sync. --- src/LocalPouchDB.ts | 14 +- src/ObsidianLiveSyncSettingTab.ts | 55 ++++- src/lib | 2 +- src/main.ts | 369 +++++++++++++++++++++++++++++- src/types.ts | 8 + 5 files changed, 439 insertions(+), 9 deletions(-) diff --git a/src/LocalPouchDB.ts b/src/LocalPouchDB.ts index 9da1a2c..4aa4c5f 100644 --- a/src/LocalPouchDB.ts +++ b/src/LocalPouchDB.ts @@ -20,6 +20,7 @@ import { MILSTONE_DOCID, DatabaseConnectingStatus, ChunkVersionRange, + NoteEntry, } from "./lib/src/types"; import { RemoteDBSettings } from "./lib/src/types"; import { resolveWithIgnoreKnownError, runWithLock, shouldSplitAsPlainText, splitPieces2, enableEncryption } from "./lib/src/utils"; @@ -304,7 +305,7 @@ export class LocalPouchDB { } else { obj = await this.localDatabase.get(id); } - + const deleted = "deleted" in obj ? obj.deleted : undefined; if (obj.type && obj.type == "leaf") { //do nothing for leaf; return false; @@ -330,6 +331,8 @@ export class LocalPouchDB { _conflicts: obj._conflicts, children: children, datatype: type, + deleted: deleted, + type: type }; return doc; } @@ -350,6 +353,7 @@ export class LocalPouchDB { } else { obj = await this.localDatabase.get(id); } + const deleted = "deleted" in obj ? obj.deleted : undefined; if (obj.type && obj.type == "leaf") { //do nothing for leaf; @@ -358,7 +362,7 @@ export class LocalPouchDB { //Check it out and fix docs to regular case if (!obj.type || (obj.type && obj.type == "notes")) { - const note = obj as Entry; + const note = obj as NoteEntry; const doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = { data: note.data, _id: note._id, @@ -370,6 +374,8 @@ export class LocalPouchDB { _conflicts: obj._conflicts, children: [], datatype: "newnote", + deleted: deleted, + type: "newnote", }; if (typeof this.corruptedEntries[doc._id] != "undefined") { delete this.corruptedEntries[doc._id]; @@ -414,6 +420,8 @@ export class LocalPouchDB { children: obj.children, datatype: obj.type, _conflicts: obj._conflicts, + deleted: deleted, + type: obj.type }; if (dump) { Logger(`therefore:`); @@ -684,7 +692,7 @@ export class LocalPouchDB { throw ex; } } - const r = await this.localDatabase.put(newDoc, { force: true }); + const r = await this.localDatabase.put(newDoc, { force: true }); if (typeof this.corruptedEntries[note._id] != "undefined") { delete this.corruptedEntries[note._id]; } diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts index 136e5b3..082b556 100644 --- a/src/ObsidianLiveSyncSettingTab.ts +++ b/src/ObsidianLiveSyncSettingTab.ts @@ -765,9 +765,60 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { await this.plugin.saveSettings(); }) ); + + containerSyncSettingEl.createEl("h3", { + text: sanitizeHTMLToDom(`Experimental`), + }); + new Setting(containerSyncSettingEl) + .setName("Sync hidden files.") + .addToggle((toggle) => + toggle.setValue(this.plugin.settings.syncInternalFiles).onChange(async (value) => { + this.plugin.settings.syncInternalFiles = value; + await this.plugin.saveSettings(); + }) + ); + new Setting(containerSyncSettingEl) + .setName("Scan hidden files before replication.") + .addToggle((toggle) => + toggle.setValue(this.plugin.settings.syncInternalFilesBeforeReplication).onChange(async (value) => { + this.plugin.settings.syncInternalFilesBeforeReplication = value; + await this.plugin.saveSettings(); + }) + ); + new Setting(containerSyncSettingEl) + .setName("Scan hidden files periodicaly.") + .addText((text) => { + text.setPlaceholder("") + .setValue(this.plugin.settings.syncInternalFilesInterval + "") + .onChange(async (value) => { + let v = Number(value); + if (isNaN(v) || v < 10) { + v = 10; + } + this.plugin.settings.syncInternalFilesInterval = v; + await this.plugin.saveSettings(); + }); + text.inputEl.setAttribute("type", "number"); + }); + new Setting(containerSyncSettingEl) + .setName("Skip patterns") + .setDesc( + "Regular expression" + ) + .addTextArea((text) => + text + .setValue(this.plugin.settings.syncInternalFilesIgnorePatterns) + .setPlaceholder("\\/node_modules\\/, \\/\\.git\\/") + .onChange(async (value) => { + this.plugin.settings.syncInternalFilesIgnorePatterns = value; + await this.plugin.saveSettings(); + }) + ); + containerSyncSettingEl.createEl("h3", { + text: sanitizeHTMLToDom(`Advanced settings`), + }); containerSyncSettingEl.createEl("div", { - text: sanitizeHTMLToDom(`Advanced settings
- If you reached the payload size limit when using IBM Cloudant, please set batch size and batch limit to a lower value.`), + text: `If you reached the payload size limit when using IBM Cloudant, please set batch size and batch limit to a lower value.`, }); new Setting(containerSyncSettingEl) .setName("Batch size") diff --git a/src/lib b/src/lib index 548265c..1f67fb6 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 548265c7016f2829412900220bcae2ec145abfe6 +Subproject commit 1f67fb604c7e3ae7c3f3640203cb58c248e67be8 diff --git a/src/main.ts b/src/main.ts index 7fe8b18..da716d7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,8 +1,8 @@ import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest, Modal, App, FuzzySuggestModal, Setting } from "obsidian"; import { diff_match_patch } from "diff-match-patch"; -import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID } from "./lib/src/types"; -import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList } from "./types"; +import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry } from "./lib/src/types"; +import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo } from "./types"; import { base64ToString, arrayBufferToBase64, @@ -18,6 +18,7 @@ import { NewNotice, getLocks, Parallels, + WrappedNotice, } from "./lib/src/utils"; import { Logger, setLogger } from "./lib/src/logger"; import { LocalPouchDB } from "./LocalPouchDB"; @@ -500,6 +501,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin { this.showPluginSyncModal(); }, }); + + this.addCommand({ + id: "livesync-scaninternal", + name: "Sync hidden files", + callback: () => { + this.syncInternalFilesAndDatabase("safe", true); + }, + }); } pluginDialog: PluginDialogModal = null; @@ -531,6 +540,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } this.clearPeriodicSync(); this.clearPluginSweep(); + this.clearInternalFileScan(); if (this.localDatabase != null) { this.localDatabase.closeReplication(); this.localDatabase.close(); @@ -1124,6 +1134,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin { const now = new Date().getTime(); if (queue.missingChildren.length == 0) { queue.done = true; + if (queue.entry._id.startsWith("i:")) { + //system file + const filename = id2path(queue.entry._id.substring("i:".length)); + Logger(`Applying hidden file, ${queue.entry._id} (${queue.entry._rev}) change...`); + await this.syncInternalFilesAndDatabase("pull", false, false, [filename]) + Logger(`Applied hidden file, ${queue.entry._id} (${queue.entry._rev}) change...`); + } if (isValidPath(id2path(queue.entry._id))) { Logger(`Applying ${queue.entry._id} (${queue.entry._rev}) change...`); await this.handleDBChanged(queue.entry); @@ -1162,7 +1179,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } async parseIncomingDoc(doc: PouchDB.Core.ExistingDocument) { const skipOldFile = this.settings.skipOlderFilesOnSync && false; //patched temporary. - if (skipOldFile) { + if ((!doc._id.startsWith("i:")) && skipOldFile) { const info = this.app.vault.getAbstractFileByPath(id2path(doc._id)); if (info && info instanceof TFile) { @@ -1304,6 +1321,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { this.localDatabase.closeReplication(); this.clearPeriodicSync(); this.clearPluginSweep(); + this.clearInternalFileScan(); await this.applyBatchChange(); // disable all sync temporary. if (this.suspended) return; @@ -1314,8 +1332,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin { this.localDatabase.openReplication(this.settings, true, false, this.parseReplicationResult); this.refreshStatusText(); } + if (this.settings.syncInternalFiles) { + await this.syncInternalFilesAndDatabase("safe", false); + } this.setPeriodicSync(); this.setPluginSweep(); + this.setPeriodicInternalFileScan(); } lastMessage = ""; @@ -1414,6 +1436,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin { await this.sweepPlugin(false); } await this.loadQueuedFiles(); + if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication) { + await this.syncInternalFilesAndDatabase("push", showMessage); + } this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult); } @@ -1877,6 +1902,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { size: file.stat.size, children: [], datatype: datatype, + type: datatype, }; //upsert should locked const msg = `DB <- STORAGE (${datatype}) `; @@ -2016,6 +2042,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { size: 0, children: [], datatype: "plain", + type: "plain" }; Logger(`check diff:${m.name}(${m.id})`, LOG_LEVEL.VERBOSE); await runWithLock("plugin-" + m.id, false, async () => { @@ -2091,4 +2118,340 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } }); } + + + periodicInternalFileScanHandler: number = null; + + clearInternalFileScan() { + if (this.periodicInternalFileScanHandler != null) { + clearInterval(this.periodicInternalFileScanHandler); + this.periodicInternalFileScanHandler = null; + } + } + + setPeriodicInternalFileScan() { + if (this.periodicInternalFileScanHandler != null) { + this.clearInternalFileScan(); + } + if (this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval > 0) { + this.periodicPluginSweepHandler = this.setInterval(async () => await this.periodicInternalFileScan(), this.settings.syncInternalFilesInterval * 1000); + } + } + + async periodicInternalFileScan() { + await this.syncInternalFilesAndDatabase("push", false); + } + + async getFiles( + path: string, + ignoreList: string[], + filter: RegExp[] + ) { + const w = await this.app.vault.adapter.list(path); + let files = [ + ...w.files + .filter((e) => !ignoreList.some((ee) => e.endsWith(ee))) + .filter((e) => !filter || filter.some((ee) => e.match(ee))), + ]; + L1: for (const v of w.folders) { + for (const ignore of ignoreList) { + if (v.endsWith(ignore)) { + continue L1; + } + } + files = files.concat(await this.getFiles(v, ignoreList, filter)); + } + return files; + } + + async scanInternalFiles(): Promise { + const ignoreFiles = ["node_modules", ".git", "obsidian-pouch"]; + const root = this.app.vault.getRoot(); + const findRoot = root.path; + const filenames = (await this.getFiles(findRoot, ignoreFiles, null)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash")); + const files = filenames.map(async e => { + return { + path: e, + stat: await this.app.vault.adapter.stat(e) + } + }); + const result: InternalFileInfo[] = []; + for (const f of files) { + const w = await f; + result.push({ + ...w, + ...w.stat + }) + } + return result; + } + + async storeInternaFileToDatabase(file: InternalFileInfo, forceWrite = false) { + const id = "i:" + path2id(file.path); + const contentBin = await this.app.vault.adapter.readBinary(file.path); + const content = await arrayBufferToBase64(contentBin); + const mtime = file.mtime; + await runWithLock("file-" + id, false, async () => { + const old = await this.localDatabase.getDBEntry(id, null, false, false); + let saveData: LoadedEntry; + if (old === false) { + saveData = { + _id: id, + data: content, + mtime, + ctime: mtime, + datatype: "newnote", + size: file.size, + children: [], + deleted: false, + type: "newnote", + } + } else { + if (old.data == content && !forceWrite) { + // Logger(`internal files STORAGE --> DB:${file.path}: Not changed`); + return; + } + saveData = + { + ...old, + data: content, + mtime, + size: file.size, + datatype: "newnote", + children: [], + deleted: false, + type: "newnote", + } + } + await this.localDatabase.putDBEntry(saveData, true); + Logger(`internal files STORAGE --> DB:${file.path}: Done`); + }); + } + + async deleteInternaFileOnDatabase(filename: string, forceWrite = false) { + const id = "i:" + path2id(filename); + const mtime = new Date().getTime(); + await runWithLock("file-" + id, false, async () => { + const old = await this.localDatabase.getDBEntry(id, null, false, false) as InternalFileEntry | false; + let saveData: InternalFileEntry; + if (old === false) { + saveData = { + _id: id, + mtime, + ctime: mtime, + size: 0, + children: [], + deleted: true, + type: "newnote", + } + } else { + if (old.deleted) { + Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`); + return; + } + saveData = + { + ...old, + mtime, + size: 0, + children: [], + deleted: true, + type: "newnote", + } + } + await this.localDatabase.localDatabase.put(saveData); + Logger(`STORAGE -x> DB:${filename}: (hidden) Done`); + + }); + } + async ensureDirectoryEx(fullpath: string) { + const pathElements = fullpath.split("/"); + pathElements.pop(); + let c = ""; + for (const v of pathElements) { + c += v; + try { + await this.app.vault.adapter.mkdir(c); + } catch (ex) { + // basically skip exceptions. + if (ex.message && ex.message == "Folder already exists.") { + // especialy this message is. + } else { + Logger("Folder Create Error"); + Logger(ex); + } + } + c += "/"; + } + } + async extractInternaFileFromDatabase(filename: string, force = false) { + const isExists = await this.app.vault.adapter.exists(filename); + const id = "i:" + path2id(filename); + + return await runWithLock("file-" + id, false, async () => { + const fileOnDB = await this.localDatabase.getDBEntry(id, null, false, false) as false | LoadedEntry; + if (fileOnDB === false) throw new Error(`File not found on database.:${id}`); + const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false; + if (deleted) { + if (!isExists) { + Logger(`STORAGE