Compare commits

...

6 Commits

Author SHA1 Message Date
vorotamoroz
799e604eb2 bump 2022-10-21 18:22:02 +09:00
vorotamoroz
d9b69d9a1b Fixed:
- Fixed the Infinity loop
2022-10-21 18:20:03 +09:00
vorotamoroz
c18b5c24b4 bump 2022-10-14 17:39:16 +09:00
vorotamoroz
07f16e3d7d Added missing log updates. 2022-10-14 17:37:25 +09:00
vorotamoroz
486f1aa4a0 Bump 2022-10-05 17:14:52 +09:00
vorotamoroz
075c6beb68 New feature:
- Monitor hidden files, Now we can use internal file sync without scan.
Fixed:
- Periodic synchronisation sometimes failed.
- Status-display had not been cleared in some cases.
- `Skip patterns default` has been changed to more clear name.
2022-10-05 17:14:32 +09:00
8 changed files with 151 additions and 70 deletions

View File

@@ -1,7 +1,7 @@
{ {
"id": "obsidian-livesync", "id": "obsidian-livesync",
"name": "Self-hosted LiveSync", "name": "Self-hosted LiveSync",
"version": "0.15.10", "version": "0.16.2",
"minAppVersion": "0.9.12", "minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz", "author": "vorotamoroz",

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.15.10", "version": "0.16.2",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.15.10", "version": "0.16.2",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",

View File

@@ -1,6 +1,6 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.15.10", "version": "0.16.2",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js", "main": "main.js",
"type": "module", "type": "module",

View File

@@ -962,8 +962,17 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings(); await this.plugin.saveSettings();
}) })
); );
new Setting(containerSyncSettingEl)
.setName("Monitor changes to internal files")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.watchInternalFileChanges).onChange(async (value) => {
this.plugin.settings.watchInternalFileChanges = value;
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Scan for hidden files before replication") .setName("Scan for hidden files before replication")
.setDesc("This configuration will be ignored if monitoring changes is enabled.")
.addToggle((toggle) => .addToggle((toggle) =>
toggle.setValue(this.plugin.settings.syncInternalFilesBeforeReplication).onChange(async (value) => { toggle.setValue(this.plugin.settings.syncInternalFilesBeforeReplication).onChange(async (value) => {
this.plugin.settings.syncInternalFilesBeforeReplication = value; this.plugin.settings.syncInternalFilesBeforeReplication = value;
@@ -972,7 +981,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
); );
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Scan hidden files periodically") .setName("Scan hidden files periodically")
.setDesc("Seconds, 0 to disable.") .setDesc("Seconds, 0 to disable. This configuration will be ignored if monitoring changes is enabled.")
.addText((text) => { .addText((text) => {
text.setPlaceholder("") text.setPlaceholder("")
.setValue(this.plugin.settings.syncInternalFilesInterval + "") .setValue(this.plugin.settings.syncInternalFilesInterval + "")
@@ -988,7 +997,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}); });
let skipPatternTextArea: TextAreaComponent = null; let skipPatternTextArea: TextAreaComponent = null;
const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, \\/obsidian-livesync\\/"; const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, \\/obsidian-livesync\\/";
const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$"; const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$";
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Skip patterns") .setName("Skip patterns")
.setDesc( .setDesc(
@@ -1007,7 +1016,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} }
); );
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Skip patterns defaults") .setName("Restore the skip pattern to default")
.addButton((button) => { .addButton((button) => {
button.setButtonText("Default") button.setButtonText("Default")
.onClick(async () => { .onClick(async () => {

Submodule src/lib updated: 122225513a...8d9f82ed9b

View File

@@ -98,7 +98,7 @@ let touchedFiles: string[] = [];
function touch(file: TFile | string) { function touch(file: TFile | string) {
const f = file instanceof TFile ? file : app.vault.getAbstractFileByPath(file) as TFile; const f = file instanceof TFile ? file : app.vault.getAbstractFileByPath(file) as TFile;
const key = `${f.path}-${f.stat.mtime}-${f.stat.size}`; const key = `${f.path}-${f.stat.mtime}-${f.stat.size}`;
touchedFiles.push(key); touchedFiles.unshift(key);
touchedFiles = touchedFiles.slice(0, 100); touchedFiles = touchedFiles.slice(0, 100);
} }
function recentlyTouched(file: TFile) { function recentlyTouched(file: TFile) {
@@ -111,9 +111,9 @@ function clearTouched() {
} }
type CacheData = string | ArrayBuffer; type CacheData = string | ArrayBuffer;
type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "RENAME"; type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "RENAME" | "INTERNAL";
type FileEventArgs = { type FileEventArgs = {
file: TAbstractFile; file: TAbstractFile | InternalFileInfo;
cache?: CacheData; cache?: CacheData;
oldPath?: string; oldPath?: string;
ctx?: any; ctx?: any;
@@ -297,14 +297,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.watchVaultCreate = this.watchVaultCreate.bind(this); this.watchVaultCreate = this.watchVaultCreate.bind(this);
this.watchVaultDelete = this.watchVaultDelete.bind(this); this.watchVaultDelete = this.watchVaultDelete.bind(this);
this.watchVaultRename = this.watchVaultRename.bind(this); this.watchVaultRename = this.watchVaultRename.bind(this);
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
this.watchWorkspaceOpen = debounce(this.watchWorkspaceOpen.bind(this), 1000, false); this.watchWorkspaceOpen = debounce(this.watchWorkspaceOpen.bind(this), 1000, false);
this.watchWindowVisibility = debounce(this.watchWindowVisibility.bind(this), 1000, false); this.watchWindowVisibility = debounce(this.watchWindowVisibility.bind(this), 1000, false);
this.watchOnline = debounce(this.watchOnline.bind(this), 500, false); this.watchOnline = debounce(this.watchOnline.bind(this), 500, false);
this.parseReplicationResult = this.parseReplicationResult.bind(this); this.parseReplicationResult = this.parseReplicationResult.bind(this);
this.periodicSync = this.periodicSync.bind(this);
this.setPeriodicSync = this.setPeriodicSync.bind(this); this.setPeriodicSync = this.setPeriodicSync.bind(this);
this.periodicSync = this.periodicSync.bind(this);
this.loadQueuedFiles = this.loadQueuedFiles.bind(this);
this.getPluginList = this.getPluginList.bind(this); this.getPluginList = this.getPluginList.bind(this);
// this.registerWatchEvents(); // this.registerWatchEvents();
@@ -718,6 +720,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.registerEvent(this.app.vault.on("delete", this.watchVaultDelete)); this.registerEvent(this.app.vault.on("delete", this.watchVaultDelete));
this.registerEvent(this.app.vault.on("rename", this.watchVaultRename)); this.registerEvent(this.app.vault.on("rename", this.watchVaultRename));
this.registerEvent(this.app.vault.on("create", this.watchVaultCreate)); this.registerEvent(this.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API
this.registerEvent(this.app.vault.on("raw", this.watchVaultRawEvents));
} }
registerWatchEvents() { registerWatchEvents() {
@@ -769,9 +773,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
// Cache file and waiting to can be proceed. // Cache file and waiting to can be proceed.
async appendWatchEvent(type: FileEventType, file: TAbstractFile, oldPath?: string, ctx?: any) { async appendWatchEvent(type: FileEventType, file: TAbstractFile | InternalFileInfo, oldPath?: string, ctx?: any) {
// check really we can process. // check really we can process.
if (!this.isTargetFile(file)) return; if (file instanceof TFile && !this.isTargetFile(file)) return;
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
let cache: null | string | ArrayBuffer; let cache: null | string | ArrayBuffer;
@@ -837,36 +841,45 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
} }
clearTrigger("applyBatchAuto"); clearTrigger("applyBatchAuto");
const ret = await runWithLock("procFiles", false, async () => { const ret = await runWithLock("procFiles", true, async () => {
const procs = [...this.watchedFileEventQueue]; do {
this.watchedFileEventQueue = []; const procs = [...this.watchedFileEventQueue];
for (const queue of procs) { this.watchedFileEventQueue = [];
const file = queue.args.file; for (const queue of procs) {
const key = `file-last-proc-${queue.type}-${file.path}`; const file = queue.args.file;
const last = Number(await this.localDatabase.kvDB.get(key) || 0); const key = `file-last-proc-${queue.type}-${file.path}`;
if (file instanceof TFile && file.stat.mtime == last) { const last = Number(await this.localDatabase.kvDB.get(key) || 0);
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL.VERBOSE); if (file instanceof TFile && file.stat.mtime == last) {
continue; Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL.VERBOSE);
} continue;
const cache = queue.args.cache; }
if ((queue.type == "CREATE" || queue.type == "CHANGED") && file instanceof TFile) {
await this.updateIntoDB(file, false, cache); const cache = queue.args.cache;
} if ((queue.type == "CREATE" || queue.type == "CHANGED") && file instanceof TFile) {
if (queue.type == "DELETE") { await this.updateIntoDB(file, false, cache);
}
if (queue.type == "DELETE") {
if (file instanceof TFile) {
await this.deleteFromDB(file);
} else if (file instanceof TFolder) {
await this.deleteFolderOnDB(file);
}
}
if (queue.type == "RENAME") {
if (file instanceof TFile) {
await this.watchVaultRenameAsync(file, queue.args.oldPath);
}
}
if (queue.type == "INTERNAL") {
await this.watchVaultRawEventsAsync(file.path);
}
if (file instanceof TFile) { if (file instanceof TFile) {
await this.deleteFromDB(file); await this.localDatabase.kvDB.set(key, file.stat.mtime);
} else if (file instanceof TFolder) {
await this.deleteFolderOnDB(file);
} }
} }
if (queue.type == "RENAME") { this.refreshStatusText();
await this.watchVaultRenameAsync(file, queue.args.oldPath); } while (this.watchedFileEventQueue.length != 0);
} return true;
if (file instanceof TFile) {
await this.localDatabase.kvDB.set(key, file.stat.mtime);
}
}
this.refreshStatusText();
}) })
this.refreshStatusText(); this.refreshStatusText();
return ret; return ret;
@@ -907,9 +920,55 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
async applyBatchChange() { async applyBatchChange() {
return await this.procFileEvent(true); if (this.settings.batchSave) {
return await this.procFileEvent(true);
}
} }
// Watch raw events (Internal API)
watchVaultRawEvents(path: string) {
if (!this.settings.syncInternalFiles) return;
if (!this.settings.watchInternalFileChanges) return;
if (!path.startsWith(this.app.vault.configDir)) return;
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
if (ignorePatterns.some(e => path.match(e))) return;
this.appendWatchEvent("INTERNAL", { path, mtime: 0, ctime: 0, size: 0 }, "", null);
}
recentProcessedInternalFiles = [] as string[];
async watchVaultRawEventsAsync(path: string) {
const stat = await this.app.vault.adapter.stat(path);
// sometimes folder is coming.
if (stat && stat.type != "file") return;
const storageMTime = ~~((stat && stat.mtime || 0) / 1000);
const key = `${path}-${storageMTime}`;
if (this.recentProcessedInternalFiles.contains(key)) {
//If recently processed, it may caused by self.
return;
}
this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100);
const id = filename2idInternalChunk(path);
const filesOnDB = await this.localDatabase.getDBEntryMeta(id);
const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000);
// Skip unchanged file.
if (dbMTime == storageMTime) {
// Logger(`STORAGE --> DB:${path}: (hidden) Nothing changed`);
return;
}
// Do not compare timestamp. Always local data should be preferred except this plugin wrote one.
if (storageMTime == 0) {
await this.deleteInternalFileOnDatabase(path);
} else {
await this.storeInternalFileToDatabase({ path: path, ...stat });
}
}
GetAllFilesRecursively(file: TAbstractFile): TFile[] { GetAllFilesRecursively(file: TAbstractFile): TFile[] {
if (file instanceof TFile) { if (file instanceof TFile) {
return [file]; return [file];
@@ -1236,6 +1295,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
} }
); );
this.refreshStatusText();
} }
async handleDBChangedAsync(change: EntryBody) { async handleDBChangedAsync(change: EntryBody) {
@@ -1296,6 +1356,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.syncInternalFilesAndDatabase("pull", false, false, w); await this.syncInternalFilesAndDatabase("pull", false, false, w);
Logger(`Applying hidden ${w.length} files changed`); Logger(`Applying hidden ${w.length} files changed`);
}); });
this.refreshStatusText();
} }
procInternalFile(filename: string) { procInternalFile(filename: string) {
this.procInternalFiles.push(filename); this.procInternalFiles.push(filename);
@@ -1602,6 +1663,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
updateStatusBarText() { } updateStatusBarText() { }
async replicate(showMessage?: boolean) { async replicate(showMessage?: boolean) {
if (!this.isReady) return;
if (this.settings.versionUpFlash != "") { if (this.settings.versionUpFlash != "") {
Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE); Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE);
return; return;
@@ -1611,7 +1673,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.sweepPlugin(false); await this.sweepPlugin(false);
} }
await this.loadQueuedFiles(); await this.loadQueuedFiles();
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication) { if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication && !this.settings.watchInternalFileChanges) {
await this.syncInternalFilesAndDatabase("push", showMessage); await this.syncInternalFilesAndDatabase("push", showMessage);
} }
this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult); this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
@@ -2333,7 +2395,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (this.periodicInternalFileScanHandler != null) { if (this.periodicInternalFileScanHandler != null) {
this.clearInternalFileScan(); this.clearInternalFileScan();
} }
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval > 0) { if (this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval > 0 && !this.settings.watchInternalFileChanges) {
this.periodicPluginSweepHandler = this.setInterval(async () => await this.periodicInternalFileScan(), this.settings.syncInternalFilesInterval * 1000); this.periodicPluginSweepHandler = this.setInterval(async () => await this.periodicInternalFileScan(), this.settings.syncInternalFilesInterval * 1000);
} }
} }

View File

@@ -1,5 +1,15 @@
### 0.16.0
- Now hidden files need not be scanned. Changes will be detected automatically.
- If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.
- Due to using an internal API, this feature may become unusable with a major update. If this happens, please disable this once.
#### Minors
- 0.16.1 Added missing log updates.
- 0.16.2 Fixed many problems caused by combinations of `Sync On Save` and the tracking logic that changed at 0.15.6.
### 0.15.0 ### 0.15.0
- Outdated configuration items have been removed. - Outdated configuration items have been removed.
- Setup wizard has been implemented! - Setup wizard has been implemented!
I appreciate for reviewing and giving me advice @Pouhon158! I appreciate for reviewing and giving me advice @Pouhon158!
@@ -18,29 +28,4 @@ I appreciate for reviewing and giving me advice @Pouhon158!
- The boot sequence has been corrected and now boots smoothly. - The boot sequence has been corrected and now boots smoothly.
- Auto applying of batch save will be processed earlier than before. - Auto applying of batch save will be processed earlier than before.
### 0.14.1
- The target selecting filter was implemented.
Now we can set what files are synchronised by regular expression.
- We can configure the size of chunks.
We can use larger chunks to improve performance.
(This feature can not be used with IBM Cloudant)
- Read chunks online.
Now we can synchronise only metadata and retrieve chunks on demand. It reduces local database size and time for replication.
- Added this note.
- Use local chunks in preference to remote them if present,
#### Recommended configuration for Self-hosted CouchDB
- Set chunk size to around 100 to 250 (10MB - 25MB per chunk)
- *Set batch size to 100 and batch limit to 20 (0.14.2)*
- Be sure to `Read chunks online` checked.
#### Minors
- 0.14.2 Fixed issue about retrieving files if synchronisation has been interrupted or failed
- 0.14.3 New test items have been added to `Check database configuration`.
- 0.14.4 Fixed issue of importing configurations.
- 0.14.5 Auto chunk size adjusting implemented.
- 0.14.6 Change Target to ES2018
- 0.14.7 Refactor and fix typos.
- 0.14.8 Refactored again. There should be no change in behaviour, but please let me know if there is any.
... To continue on to `updates_old.md`. ... To continue on to `updates_old.md`.

View File

@@ -1,3 +1,28 @@
### 0.14.1
- The target selecting filter was implemented.
Now we can set what files are synchronised by regular expression.
- We can configure the size of chunks.
We can use larger chunks to improve performance.
(This feature can not be used with IBM Cloudant)
- Read chunks online.
Now we can synchronise only metadata and retrieve chunks on demand. It reduces local database size and time for replication.
- Added this note.
- Use local chunks in preference to remote them if present,
#### Recommended configuration for Self-hosted CouchDB
- Set chunk size to around 100 to 250 (10MB - 25MB per chunk)
- *Set batch size to 100 and batch limit to 20 (0.14.2)*
- Be sure to `Read chunks online` checked.
#### Minors
- 0.14.2 Fixed issue about retrieving files if synchronisation has been interrupted or failed
- 0.14.3 New test items have been added to `Check database configuration`.
- 0.14.4 Fixed issue of importing configurations.
- 0.14.5 Auto chunk size adjusting implemented.
- 0.14.6 Change Target to ES2018
- 0.14.7 Refactor and fix typos.
- 0.14.8 Refactored again. There should be no change in behaviour, but please let me know if there is any.
### 0.13.0 ### 0.13.0
- The metadata of the deleted files will be kept on the database by default. If you want to delete this as the previous version, please turn on `Delete metadata of deleted files.`. And, if you have upgraded from the older version, please ensure every device has been upgraded. - The metadata of the deleted files will be kept on the database by default. If you want to delete this as the previous version, please turn on `Delete metadata of deleted files.`. And, if you have upgraded from the older version, please ensure every device has been upgraded.