Compare commits

...

11 Commits

Author SHA1 Message Date
vorotamoroz
799e604eb2 bump 2022-10-21 18:22:02 +09:00
vorotamoroz
d9b69d9a1b Fixed:
- Fixed the Infinity loop
2022-10-21 18:20:03 +09:00
vorotamoroz
c18b5c24b4 bump 2022-10-14 17:39:16 +09:00
vorotamoroz
07f16e3d7d Added missing log updates. 2022-10-14 17:37:25 +09:00
vorotamoroz
486f1aa4a0 Bump 2022-10-05 17:14:52 +09:00
vorotamoroz
075c6beb68 New feature:
- Monitor hidden files, Now we can use internal file sync without scan.
Fixed:
- Periodic synchronisation sometimes failed.
- Status-display had not been cleared in some cases.
- `Skip patterns default` has been changed to more clear name.
2022-10-05 17:14:32 +09:00
vorotamoroz
d6121b0c1e bump 2022-10-03 10:57:39 +09:00
vorotamoroz
3292a48054 Fixed
- The boot sequence has been corrected and now boots smoothly.
- Auto applying of batch save will be processed earlier than before.
2022-10-03 10:52:31 +09:00
vorotamoroz
ee37764040 bump 2022-10-02 01:48:50 +09:00
vorotamoroz
b6f7fced22 Use new library for batching the chunk retrieving 2022-10-02 01:45:44 +09:00
vorotamoroz
13456c0854 Fixed: deleted debug message 2022-10-02 01:44:39 +09:00
8 changed files with 182 additions and 99 deletions

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.15.8",
"version": "0.16.2",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

34
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.15.8",
"version": "0.16.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.15.8",
"version": "0.16.2",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",
@@ -30,7 +30,7 @@
"eslint": "^7.32.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.25.2",
"obsidian": "^0.15.4",
"obsidian": "^0.16.3",
"postcss": "^8.4.14",
"postcss-load-config": "^3.1.4",
"rollup": "^2.32.1",
@@ -2571,9 +2571,9 @@
}
},
"node_modules/moment": {
"version": "2.29.3",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz",
"integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==",
"version": "2.29.4",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
"dev": true,
"engines": {
"node": "*"
@@ -2671,13 +2671,13 @@
}
},
"node_modules/obsidian": {
"version": "0.15.4",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.15.4.tgz",
"integrity": "sha512-FE11CxxpVD6t/DBvjLvlT7q7YYW91ubTqPKIIp286LdnyLipS8Xi3Tif8i8ALPv87Vg9obKM43aWcPsYLxLllQ==",
"version": "0.16.3",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.16.3.tgz",
"integrity": "sha512-hal9qk1A0GMhHSeLr2/+o3OpLmImiP+Y+sx2ewP13ds76KXsziG96n+IPFT0mSkup1zSwhEu+DeRhmbcyCCXWw==",
"dev": true,
"dependencies": {
"@types/codemirror": "0.0.108",
"moment": "2.29.3"
"moment": "2.29.4"
},
"peerDependencies": {
"@codemirror/state": "^6.0.0",
@@ -5490,9 +5490,9 @@
}
},
"moment": {
"version": "2.29.3",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz",
"integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==",
"version": "2.29.4",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
"dev": true
},
"ms": {
@@ -5560,13 +5560,13 @@
}
},
"obsidian": {
"version": "0.15.4",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.15.4.tgz",
"integrity": "sha512-FE11CxxpVD6t/DBvjLvlT7q7YYW91ubTqPKIIp286LdnyLipS8Xi3Tif8i8ALPv87Vg9obKM43aWcPsYLxLllQ==",
"version": "0.16.3",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.16.3.tgz",
"integrity": "sha512-hal9qk1A0GMhHSeLr2/+o3OpLmImiP+Y+sx2ewP13ds76KXsziG96n+IPFT0mSkup1zSwhEu+DeRhmbcyCCXWw==",
"dev": true,
"requires": {
"@types/codemirror": "0.0.108",
"moment": "2.29.3"
"moment": "2.29.4"
}
},
"once": {

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.15.8",
"version": "0.16.2",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",
@@ -27,7 +27,7 @@
"eslint": "^7.32.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.25.2",
"obsidian": "^0.15.4",
"obsidian": "^0.16.3",
"postcss": "^8.4.14",
"postcss-load-config": "^3.1.4",
"rollup": "^2.32.1",

View File

@@ -67,9 +67,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
element.removeClass("selected");
(element.querySelector("input[type=radio]") as HTMLInputElement).checked = false;
});
console.log(`.sls-setting-label.c-${screen}`)
w.querySelectorAll(`.sls-setting-label.c-${screen}`).forEach((element) => {
console.log(element)
element.addClass("selected");
(element.querySelector("input[type=radio]") as HTMLInputElement).checked = true;
});
@@ -381,15 +379,15 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
})
)
.addButton((button) =>
button
.setButtonText("Apply w/o rebuilding")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-right")
.onClick(async () => {
await applyEncryption(false);
})
);
button
.setButtonText("Apply w/o rebuilding")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-right")
.onClick(async () => {
await applyEncryption(false);
})
);
const rebuildDB = async (method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice") => {
@@ -964,8 +962,17 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl)
.setName("Monitor changes to internal files")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.watchInternalFileChanges).onChange(async (value) => {
this.plugin.settings.watchInternalFileChanges = value;
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl)
.setName("Scan for hidden files before replication")
.setDesc("This configuration will be ignored if monitoring changes is enabled.")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.syncInternalFilesBeforeReplication).onChange(async (value) => {
this.plugin.settings.syncInternalFilesBeforeReplication = value;
@@ -974,7 +981,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
);
new Setting(containerSyncSettingEl)
.setName("Scan hidden files periodically")
.setDesc("Seconds, 0 to disable.")
.setDesc("Seconds, 0 to disable. This configuration will be ignored if monitoring changes is enabled.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.syncInternalFilesInterval + "")
@@ -990,7 +997,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
});
let skipPatternTextArea: TextAreaComponent = null;
const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, \\/obsidian-livesync\\/";
const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$";
const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$";
new Setting(containerSyncSettingEl)
.setName("Skip patterns")
.setDesc(
@@ -1009,7 +1016,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}
);
new Setting(containerSyncSettingEl)
.setName("Skip patterns defaults")
.setName("Restore the skip pattern to default")
.addButton((button) => {
button.setButtonText("Default")
.onClick(async () => {

Submodule src/lib updated: d8d83b7f46...8d9f82ed9b

View File

@@ -98,7 +98,7 @@ let touchedFiles: string[] = [];
function touch(file: TFile | string) {
const f = file instanceof TFile ? file : app.vault.getAbstractFileByPath(file) as TFile;
const key = `${f.path}-${f.stat.mtime}-${f.stat.size}`;
touchedFiles.push(key);
touchedFiles.unshift(key);
touchedFiles = touchedFiles.slice(0, 100);
}
function recentlyTouched(file: TFile) {
@@ -111,9 +111,9 @@ function clearTouched() {
}
type CacheData = string | ArrayBuffer;
type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "RENAME";
type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "RENAME" | "INTERNAL";
type FileEventArgs = {
file: TAbstractFile;
file: TAbstractFile | InternalFileInfo;
cache?: CacheData;
oldPath?: string;
ctx?: any;
@@ -297,21 +297,23 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.watchVaultCreate = this.watchVaultCreate.bind(this);
this.watchVaultDelete = this.watchVaultDelete.bind(this);
this.watchVaultRename = this.watchVaultRename.bind(this);
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
this.watchWorkspaceOpen = debounce(this.watchWorkspaceOpen.bind(this), 1000, false);
this.watchWindowVisibility = debounce(this.watchWindowVisibility.bind(this), 1000, false);
this.watchOnline = debounce(this.watchOnline.bind(this), 500, false);
this.parseReplicationResult = this.parseReplicationResult.bind(this);
this.periodicSync = this.periodicSync.bind(this);
this.setPeriodicSync = this.setPeriodicSync.bind(this);
this.periodicSync = this.periodicSync.bind(this);
this.loadQueuedFiles = this.loadQueuedFiles.bind(this);
this.getPluginList = this.getPluginList.bind(this);
// this.registerWatchEvents();
this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
this.registerFileWatchEvents();
this.app.workspace.onLayoutReady(async () => {
this.registerFileWatchEvents();
if (this.localDatabase.isReady)
try {
if (this.isRedFlagRaised()) {
@@ -718,6 +720,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.registerEvent(this.app.vault.on("delete", this.watchVaultDelete));
this.registerEvent(this.app.vault.on("rename", this.watchVaultRename));
this.registerEvent(this.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API
this.registerEvent(this.app.vault.on("raw", this.watchVaultRawEvents));
}
registerWatchEvents() {
@@ -769,9 +773,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
// Cache file and waiting to can be proceed.
async appendWatchEvent(type: FileEventType, file: TAbstractFile, oldPath?: string, ctx?: any) {
async appendWatchEvent(type: FileEventType, file: TAbstractFile | InternalFileInfo, oldPath?: string, ctx?: any) {
// check really we can process.
if (!this.isTargetFile(file)) return;
if (file instanceof TFile && !this.isTargetFile(file)) return;
if (this.settings.suspendFileWatching) return;
let cache: null | string | ArrayBuffer;
@@ -830,43 +834,52 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (!applyBatch && this.watchedFileEventQueue.length < FileWatchEventQueueMax) {
// Defer till applying batch save or queue has been grown enough.
// or 120 seconds after.
setTrigger("applyBatchAuto", 120000, () => {
setTrigger("applyBatchAuto", 30000, () => {
this.procFileEvent(true);
})
return;
}
}
clearTrigger("applyBatchAuto");
const ret = await runWithLock("procFiles", false, async () => {
const procs = [...this.watchedFileEventQueue];
this.watchedFileEventQueue = [];
for (const queue of procs) {
const file = queue.args.file;
const key = `file-last-proc-${queue.type}-${file.path}`;
const last = Number(await this.localDatabase.kvDB.get(key) || 0);
if (file instanceof TFile && file.stat.mtime == last) {
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL.VERBOSE);
continue;
}
const cache = queue.args.cache;
if ((queue.type == "CREATE" || queue.type == "CHANGED") && file instanceof TFile) {
await this.updateIntoDB(file, false, cache);
}
if (queue.type == "DELETE") {
const ret = await runWithLock("procFiles", true, async () => {
do {
const procs = [...this.watchedFileEventQueue];
this.watchedFileEventQueue = [];
for (const queue of procs) {
const file = queue.args.file;
const key = `file-last-proc-${queue.type}-${file.path}`;
const last = Number(await this.localDatabase.kvDB.get(key) || 0);
if (file instanceof TFile && file.stat.mtime == last) {
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL.VERBOSE);
continue;
}
const cache = queue.args.cache;
if ((queue.type == "CREATE" || queue.type == "CHANGED") && file instanceof TFile) {
await this.updateIntoDB(file, false, cache);
}
if (queue.type == "DELETE") {
if (file instanceof TFile) {
await this.deleteFromDB(file);
} else if (file instanceof TFolder) {
await this.deleteFolderOnDB(file);
}
}
if (queue.type == "RENAME") {
if (file instanceof TFile) {
await this.watchVaultRenameAsync(file, queue.args.oldPath);
}
}
if (queue.type == "INTERNAL") {
await this.watchVaultRawEventsAsync(file.path);
}
if (file instanceof TFile) {
await this.deleteFromDB(file);
} else if (file instanceof TFolder) {
await this.deleteFolderOnDB(file);
await this.localDatabase.kvDB.set(key, file.stat.mtime);
}
}
if (queue.type == "RENAME") {
await this.watchVaultRenameAsync(file, queue.args.oldPath);
}
if (file instanceof TFile) {
await this.localDatabase.kvDB.set(key, file.stat.mtime);
}
}
this.refreshStatusText();
this.refreshStatusText();
} while (this.watchedFileEventQueue.length != 0);
return true;
})
this.refreshStatusText();
return ret;
@@ -907,9 +920,55 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async applyBatchChange() {
return await this.procFileEvent(true);
if (this.settings.batchSave) {
return await this.procFileEvent(true);
}
}
// Watch raw events (Internal API)
watchVaultRawEvents(path: string) {
if (!this.settings.syncInternalFiles) return;
if (!this.settings.watchInternalFileChanges) return;
if (!path.startsWith(this.app.vault.configDir)) return;
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
if (ignorePatterns.some(e => path.match(e))) return;
this.appendWatchEvent("INTERNAL", { path, mtime: 0, ctime: 0, size: 0 }, "", null);
}
recentProcessedInternalFiles = [] as string[];
async watchVaultRawEventsAsync(path: string) {
const stat = await this.app.vault.adapter.stat(path);
// sometimes folder is coming.
if (stat && stat.type != "file") return;
const storageMTime = ~~((stat && stat.mtime || 0) / 1000);
const key = `${path}-${storageMTime}`;
if (this.recentProcessedInternalFiles.contains(key)) {
//If recently processed, it may caused by self.
return;
}
this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100);
const id = filename2idInternalChunk(path);
const filesOnDB = await this.localDatabase.getDBEntryMeta(id);
const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000);
// Skip unchanged file.
if (dbMTime == storageMTime) {
// Logger(`STORAGE --> DB:${path}: (hidden) Nothing changed`);
return;
}
// Do not compare timestamp. Always local data should be preferred except this plugin wrote one.
if (storageMTime == 0) {
await this.deleteInternalFileOnDatabase(path);
} else {
await this.storeInternalFileToDatabase({ path: path, ...stat });
}
}
GetAllFilesRecursively(file: TAbstractFile): TFile[] {
if (file instanceof TFile) {
return [file];
@@ -1236,6 +1295,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
}
);
this.refreshStatusText();
}
async handleDBChangedAsync(change: EntryBody) {
@@ -1296,6 +1356,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.syncInternalFilesAndDatabase("pull", false, false, w);
Logger(`Applying hidden ${w.length} files changed`);
});
this.refreshStatusText();
}
procInternalFile(filename: string) {
this.procInternalFiles.push(filename);
@@ -1602,6 +1663,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
updateStatusBarText() { }
async replicate(showMessage?: boolean) {
if (!this.isReady) return;
if (this.settings.versionUpFlash != "") {
Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE);
return;
@@ -1611,7 +1673,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.sweepPlugin(false);
}
await this.loadQueuedFiles();
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication) {
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication && !this.settings.watchInternalFileChanges) {
await this.syncInternalFilesAndDatabase("push", showMessage);
}
this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
@@ -2333,7 +2395,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (this.periodicInternalFileScanHandler != null) {
this.clearInternalFileScan();
}
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval > 0) {
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval > 0 && !this.settings.watchInternalFileChanges) {
this.periodicPluginSweepHandler = this.setInterval(async () => await this.periodicInternalFileScan(), this.settings.syncInternalFilesInterval * 1000);
}
}

View File

@@ -1,5 +1,15 @@
### 0.16.0
- Now hidden files need not be scanned. Changes will be detected automatically.
- If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.
- Due to using an internal API, this feature may become unusable with a major update. If this happens, please disable this once.
#### Minors
- 0.16.1 Added missing log updates.
- 0.16.2 Fixed many problems caused by combinations of `Sync On Save` and the tracking logic that changed at 0.15.6.
### 0.15.0
- Outdated configuration items have been removed.
- Outdated configuration items have been removed.
- Setup wizard has been implemented!
I appreciate for reviewing and giving me advice @Pouhon158!
@@ -13,30 +23,9 @@ I appreciate for reviewing and giving me advice @Pouhon158!
- 0.15.6 File tracking logic has been refined.
- 0.15.7 Fixed bug about renaming file.
- 0.15.8 Fixed bug about deleting empty directory, weird behaviour on boot-sequence on mobile devices.
### 0.14.1
- The target selecting filter was implemented.
Now we can set what files are synchronised by regular expression.
- We can configure the size of chunks.
We can use larger chunks to improve performance.
(This feature can not be used with IBM Cloudant)
- Read chunks online.
Now we can synchronise only metadata and retrieve chunks on demand. It reduces local database size and time for replication.
- Added this note.
- Use local chunks in preference to remote them if present,
#### Recommended configuration for Self-hosted CouchDB
- Set chunk size to around 100 to 250 (10MB - 25MB per chunk)
- *Set batch size to 100 and batch limit to 20 (0.14.2)*
- Be sure to `Read chunks online` checked.
#### Minors
- 0.14.2 Fixed issue about retrieving files if synchronisation has been interrupted or failed
- 0.14.3 New test items have been added to `Check database configuration`.
- 0.14.4 Fixed issue of importing configurations.
- 0.14.5 Auto chunk size adjusting implemented.
- 0.14.6 Change Target to ES2018
- 0.14.7 Refactor and fix typos.
- 0.14.8 Refactored again. There should be no change in behaviour, but please let me know if there is any.
- 0.15.9 Improved chunk retrieving, now chunks are retrieved in batch on continuous requests.
- 0.15.10 Fixed:
- The boot sequence has been corrected and now boots smoothly.
- Auto applying of batch save will be processed earlier than before.
... To continue on to `updates_old.md`.

View File

@@ -1,3 +1,28 @@
### 0.14.1
- The target selecting filter was implemented.
Now we can set what files are synchronised by regular expression.
- We can configure the size of chunks.
We can use larger chunks to improve performance.
(This feature can not be used with IBM Cloudant)
- Read chunks online.
Now we can synchronise only metadata and retrieve chunks on demand. It reduces local database size and time for replication.
- Added this note.
- Use local chunks in preference to remote them if present,
#### Recommended configuration for Self-hosted CouchDB
- Set chunk size to around 100 to 250 (10MB - 25MB per chunk)
- *Set batch size to 100 and batch limit to 20 (0.14.2)*
- Be sure to `Read chunks online` checked.
#### Minors
- 0.14.2 Fixed issue about retrieving files if synchronisation has been interrupted or failed
- 0.14.3 New test items have been added to `Check database configuration`.
- 0.14.4 Fixed issue of importing configurations.
- 0.14.5 Auto chunk size adjusting implemented.
- 0.14.6 Change Target to ES2018
- 0.14.7 Refactor and fix typos.
- 0.14.8 Refactored again. There should be no change in behaviour, but please let me know if there is any.
### 0.13.0
- The metadata of the deleted files will be kept on the database by default. If you want to delete this as the previous version, please turn on `Delete metadata of deleted files.`. And, if you have upgraded from the older version, please ensure every device has been upgraded.