Compare commits

...

7 Commits

Author SHA1 Message Date
vorotamoroz
3940260d42 bump 2023-03-02 12:56:59 +09:00
vorotamoroz
b16333c604 Implemented:
- `Resolve all conflicted files` has been implemented.
Fixed:
- Fixed a problem about reading chunks online when a file has more chunks than the concurrency limit.
Rollbacked:
- Logs are kept only for 100 lines, again.
2023-03-02 12:54:41 +09:00
vorotamoroz
7bf6d1f663 update dependencies 2023-03-02 12:51:46 +09:00
vorotamoroz
7046928068 bump 2023-03-01 12:59:48 +09:00
vorotamoroz
333fcbaaeb - Fixed:
- Requests of reading chunks online are now split into a reasonable(and configurable) size.
    - No longer error message will be shown on Linux devices with hidden file synchronisation.
  - Improved:
    - The interval of reading chunks online is now configurable.
    - Boot sequence has been speeded up, more.
  - Misc:
    - Messages on the boot sequence will now be more detailed. If you want to see them, please enable the verbose log.
    - Logs became be kept for 1000 lines while the verbose log is enabled.
2023-03-01 12:58:29 +09:00
vorotamoroz
009f92c307 bump 2023-02-28 17:25:46 +09:00
vorotamoroz
3e541bd061 Fixed:
- Some messages have been refined.
- Boot sequence has been speeded up.
- Opening the local database multiple times in a short duration has been suppressed.
2023-02-28 17:15:43 +09:00
8 changed files with 1389 additions and 866 deletions

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.17.27",
"version": "0.17.30",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

1941
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.17.27",
"version": "0.17.30",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",
@@ -16,31 +16,31 @@
"@types/diff-match-patch": "^1.0.32",
"@types/pouchdb": "^6.4.0",
"@types/pouchdb-browser": "^6.1.3",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",
"@typescript-eslint/eslint-plugin": "^5.54.0",
"@typescript-eslint/parser": "^5.54.0",
"builtin-modules": "^3.3.0",
"esbuild": "0.15.15",
"esbuild-svelte": "^0.7.3",
"eslint": "^8.28.0",
"eslint": "^8.35.0",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-import": "^2.27.5",
"events": "^3.3.0",
"obsidian": "^0.16.3",
"postcss": "^8.4.19",
"obsidian": "^1.1.1",
"postcss": "^8.4.21",
"postcss-load-config": "^4.0.1",
"pouchdb-adapter-http": "^8.0.0",
"pouchdb-adapter-idb": "^8.0.0",
"pouchdb-adapter-indexeddb": "^8.0.0",
"pouchdb-core": "^8.0.0",
"pouchdb-find": "^8.0.0",
"pouchdb-mapreduce": "^8.0.0",
"pouchdb-replication": "^8.0.0",
"pouchdb-utils": "file:src/lib/src/patches/pouchdb-utils",
"svelte": "^3.53.1",
"svelte-preprocess": "^4.10.7",
"pouchdb-adapter-http": "^8.0.1",
"pouchdb-adapter-idb": "^8.0.1",
"pouchdb-adapter-indexeddb": "^8.0.1",
"pouchdb-core": "^8.0.1",
"pouchdb-find": "^8.0.1",
"pouchdb-mapreduce": "^8.0.1",
"pouchdb-replication": "^8.0.1",
"pouchdb-utils": "^8.0.1",
"svelte": "^3.55.1",
"svelte-preprocess": "^5.0.1",
"transform-pouch": "^2.0.0",
"tslib": "^2.4.1",
"typescript": "^4.9.3"
"tslib": "^2.5.0",
"typescript": "^4.9.5"
},
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@@ -453,6 +453,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncAfterMerge = false;
this.plugin.settings.syncInternalFiles = false;
this.plugin.settings.usePluginSync = false;
Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
await this.plugin.saveSettings();
applyDisplayEnabled();
@@ -1324,7 +1327,38 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerSyncSettingEl)
.setName("The maximum number of reading chunks online concurrently")
.setDesc("")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.concurrencyOfReadChunksOnline + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10) {
v = 10;
}
this.plugin.settings.concurrencyOfReadChunksOnline = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerSyncSettingEl)
.setName("The minimum interval for reading chunks online")
.setDesc("")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.minimumIntervalOfReadChunksOnline + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10) {
v = 10;
}
this.plugin.settings.minimumIntervalOfReadChunksOnline = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
addScreenElement("30", containerSyncSettingEl);
const containerMiscellaneousEl = containerEl.createDiv();
containerMiscellaneousEl.createEl("h3", { text: "Miscellaneous" });

Submodule src/lib updated: fbb3fcd8b4...8985fa74e9

View File

@@ -64,7 +64,6 @@ function filename2idInternalMetadata(str: string): string {
}
const CHeader = "h:";
const CHeaderEnd = "h;";
// const CHeaderLength = CHeader.length;
function isChunk(str: string): boolean {
return str.startsWith(CHeader);
@@ -217,12 +216,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const notesList = notes.map(e => e.path);
if (notesList.length == 0) {
Logger("There are no conflicted documents", LOG_LEVEL.NOTICE);
return;
return false;
}
const target = await askSelectString(this.app, "File to view History", notesList);
if (target) {
await this.resolveConflicted(target);
return true;
}
return false;
}
async resolveConflicted(target: string) {
if (isInternalMetadata(target)) {
@@ -235,30 +236,18 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async collectDeletedFiles() {
const pageLimit = 1000;
let nextKey = "";
const limitDays = this.settings.automaticallyDeleteMetadataOfDeletedFiles;
if (limitDays <= 0) return;
Logger(`Checking expired file history`);
const limit = Date.now() - (86400 * 1000 * limitDays);
const notes: { path: string, mtime: number, ttl: number, doc: PouchDB.Core.ExistingDocument<EntryDoc & PouchDB.Core.AllDocsMeta> }[] = [];
do {
const docs = await this.localDatabase.localDatabase.allDocs({ limit: pageLimit, startkey: nextKey, conflicts: true, include_docs: true });
nextKey = "";
for (const row of docs.rows) {
const doc = row.doc;
nextKey = `${row.id}\u{10ffff}`;
if (doc.type == "newnote" || doc.type == "plain") {
if (doc.deleted && (doc.mtime - limit) < 0) {
notes.push({ path: id2path(doc._id), mtime: doc.mtime, ttl: (doc.mtime - limit) / 1000 / 86400, doc: doc });
}
}
if (isChunk(nextKey)) {
// skip the chunk zone.
nextKey = CHeaderEnd;
for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
if (doc.type == "newnote" || doc.type == "plain") {
if (doc.deleted && (doc.mtime - limit) < 0) {
notes.push({ path: id2path(doc._id), mtime: doc.mtime, ttl: (doc.mtime - limit) / 1000 / 86400, doc: doc });
}
}
} while (nextKey != "");
}
if (notes.length == 0) {
Logger("There are no old documents");
Logger(`Checking expired file history done`);
@@ -331,7 +320,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (this.settings.suspendFileWatching) {
Logger("'Suspend file watching' turned on. Are you sure this is what you intended? Every modification on the vault will be ignored.", LOG_LEVEL.NOTICE);
}
const isInitialized = await this.initializeDatabase();
const isInitialized = await this.initializeDatabase(false, false);
if (!isInitialized) {
//TODO:stop all sync.
return false;
@@ -435,7 +424,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.settings = newSettingW;
this.usedPassphrase = "";
await this.saveSettings();
await this.resetLocalOldDatabase();
await this.resetLocalDatabase();
await this.localDatabase.initializeDatabase();
await this.markRemoteResolved();
@@ -448,7 +436,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.settings = newSettingW;
this.usedPassphrase = "";
await this.saveSettings();
await this.resetLocalOldDatabase();
await this.resetLocalDatabase();
await this.localDatabase.initializeDatabase();
await this.initializeDatabase(true);
@@ -486,7 +473,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.usedPassphrase = "";
await this.saveSettings();
if (keepLocalDB == "no") {
this.resetLocalOldDatabase();
this.resetLocalDatabase();
this.localDatabase.initializeDatabase();
const rebuild = await askYesNo(this.app, "Rebuild the database?");
@@ -535,6 +521,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const lsKey = "obsidian-live-sync-ver" + this.getVaultName();
const last_version = localStorage.getItem(lsKey);
await this.loadSettings();
const lastVersion = ~~(versionNumberString2Number(manifestVersion) / 1000);
if (lastVersion > this.settings.lastReadUpdates) {
Logger("Self-hosted LiveSync has undergone a major upgrade. Please open the setting dialog, and check the information pane.", LOG_LEVEL.NOTICE);
@@ -729,6 +716,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.pickFileForResolve();
},
})
this.addCommand({
id: "livesync-all-conflictcheck",
name: "Resolve all conflicted files",
callback: async () => {
while (await this.pickFileForResolve());
},
})
this.addCommand({
id: "livesync-runbatch",
name: "Run pended batch processes",
@@ -785,7 +779,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.localDatabase.close();
}
const vaultName = this.getVaultName();
Logger("Open Database...");
Logger("Waiting for ready...");
//@ts-ignore
const isMobile = this.app.isMobile;
this.localDatabase = new LocalPouchDB(this.settings, vaultName, isMobile);
@@ -1586,9 +1580,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const filename = id2path(id2filenameInternalMetadata(queue.entry._id));
// await this.syncInternalFilesAndDatabase("pull", false, false, [filename])
this.procInternalFile(filename);
}
if (isValidPath(id2path(queue.entry._id))) {
} else if (isValidPath(id2path(queue.entry._id))) {
this.handleDBChanged(queue.entry);
} else {
Logger(`Skipped: ${queue.entry._id}`, LOG_LEVEL.VERBOSE);
}
} else if (now > queue.timeout) {
if (!queue.warned) Logger(`Timed out: ${queue.entry._id} could not collect ${queue.missingChildren.length} chunks. plugin keeps watching, but you have to check the file after the replication.`, LOG_LEVEL.NOTICE);
@@ -1925,9 +1920,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
}
async initializeDatabase(showingNotice?: boolean) {
async initializeDatabase(showingNotice?: boolean, reopenDatabase = true) {
this.isReady = false;
if (await this.openDatabase()) {
if ((!reopenDatabase) || await this.openDatabase()) {
if (this.localDatabase.isReady) {
await this.syncAllFiles(showingNotice);
}
@@ -1989,18 +1984,23 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger("Initializing", LOG_LEVEL.NOTICE, "syncAll");
}
Logger("Initialize and checking database files");
Logger("Checking deleted files");
await this.collectDeletedFiles();
Logger("Collecting local files on the storage", LOG_LEVEL.VERBOSE);
const filesStorage = this.app.vault.getFiles().filter(e => this.isTargetFile(e));
const filesStorageName = filesStorage.map((e) => e.path);
const wf = await this.localDatabase.localDatabase.allDocs();
const filesDatabase = wf.rows.filter((e) =>
!isChunk(e.id) &&
!isPluginMetadata(e.id) &&
e.id != "obsydian_livesync_version" &&
e.id != "_design/replicate"
)
.filter(e => isValidPath(e.id)).map((e) => id2path(e.id)).filter(e => this.isTargetFile(e));
Logger("Collecting local files on the DB", LOG_LEVEL.VERBOSE);
const filesDatabase = [] as string[]
for await (const docId of this.localDatabase.findAllDocNames()) {
const path = id2path(docId);
if (isValidPath(docId) && this.isTargetFile(path)) {
filesDatabase.push(path);
}
}
Logger("Opening the key-value database", LOG_LEVEL.VERBOSE);
const isInitialized = await (this.localDatabase.kvDB.get<boolean>("initialized")) || false;
// Make chunk bigger if it is the initial scan. There must be non-active docs.
if (filesDatabase.length == 0 && !isInitialized) {
@@ -2013,33 +2013,18 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const onlyInStorageNames = onlyInStorage.map((e) => e.path);
const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
Logger("Initialize and checking database files");
Logger("Updating database by new files");
this.setStatusBarText(`UPDATE DATABASE`);
const runAll = async<T>(procedureName: string, objects: T[], callback: (arg: T) => Promise<void>) => {
// const count = objects.length;
Logger(procedureName);
// let i = 0;
const semaphore = Semaphore(25);
// Logger(`${procedureName} exec.`);
if (!this.localDatabase.isReady) throw Error("Database is not ready!");
const processes = objects.map(e => (async (v) => {
const releaser = await semaphore.acquire(1, procedureName);
try {
await callback(v);
// i++;
// if (i % 50 == 0) {
// const notify = `${procedureName} : ${i}/${count}`;
// if (showingNotice) {
// Logger(notify, LOG_LEVEL.NOTICE, "syncAll");
// } else {
// Logger(notify);
// }
// this.setStatusBarText(notify);
// }
} catch (ex) {
Logger(`Error while ${procedureName}`, LOG_LEVEL.NOTICE);
Logger(ex);
@@ -2079,7 +2064,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const syncFilesX = syncFiles.splice(0, 100);
const docs = await this.localDatabase.localDatabase.allDocs({ keys: syncFilesX.map(e => path2id(e.path)), include_docs: true })
const syncFilesToSync = syncFilesX.map((e) => ({ file: e, doc: docs.rows.find(ee => ee.id == path2id(e.path)).doc as LoadedEntry }));
await runAll(`CHECK FILE STATUS:${syncFiles.length}/${docsCount}`, syncFilesToSync, async (e) => {
caches = await this.syncFileBetweenDBandStorage(e.file, e.doc, initialScan, caches);
});
@@ -2686,6 +2670,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const dK = `${file.path}-diff`;
const isLastDiff = dK in caches ? caches[dK] : { storageMtime: 0, docMtime: 0 };
if (isLastDiff.docMtime == docMtime && isLastDiff.storageMtime == storageMtime) {
Logger("STORAGE .. DB :" + file.path, LOG_LEVEL.VERBOSE);
caches[dK] = { storageMtime, docMtime };
return caches;
}
@@ -2708,11 +2693,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
caches[dK] = { storageMtime, docMtime };
return caches;
} else {
// Logger("EVEN :" + file.path, LOG_LEVEL.VERBOSE);
// Logger(`${storageMtime} = ${docMtime}`, LOG_LEVEL.VERBOSE);
//eq.case
}
Logger("STORAGE == DB :" + file.path + "", LOG_LEVEL.VERBOSE);
caches[dK] = { storageMtime, docMtime };
return caches;
@@ -2827,11 +2809,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async resetLocalDatabase() {
clearTouched();
await this.localDatabase.resetDatabase();
await this.localDatabase.resetLocalOldDatabase();
}
async resetLocalOldDatabase() {
clearTouched();
await this.localDatabase.resetLocalOldDatabase();
}
async tryResetRemoteDatabase() {

View File

@@ -10,62 +10,7 @@
- Chunk ID numbering rules
#### Minors
- __0.17.1 to 0.17.15 has been moved into `update_old.md`__
- 0.17.16:
- Improved:
- Plugins and their settings no longer need scanning if changes are monitored.
- Now synchronising plugins and their settings are performed parallelly and faster.
- We can place `redflag2.md` to rebuild the database automatically while the boot sequence.
- Experimental:
- We can use a new adapter on PouchDB. This will make us smoother.
- Note: Not compatible with the older version.
- Fixed:
- The default batch size is smaller again.
- Plugins and their setting can be synchronised again.
- Hidden files and plugins are correctly scanned while rebuilding.
- Files with the name started `_` are also being performed conflict-checking.
- 0.17.17
- Fixed: Now we can merge JSON files even if we failed to compare items like null.
- 0.17.18
- Fixed: Fixed lack of error handling.
- 0.17.19
- Fixed: Error reporting has been ensured.
- 0.17.20
- Improved: Changes of hidden files will be notified to Obsidian.
- 0.17.21
- Fixed: Skip patterns now handle capital letters.
- Improved
- New configuration to avoid exceeding throttle capacity.
- We have been grateful to @karasevm!
- The conflicted `data.json` is no longer merged automatically.
- This behaviour is not configurable, unlike the `Use newer file if conflicted` of normal files.
- 0.17.22
- Fixed:
- Now hidden files will not be synchronised while we are not configured.
- Some processes could start without waiting for synchronisation to complete, but now they will wait for.
- Improved
- Now, by placing `redflag3.md`, we can discard the local database and fetch again.
- The document has been updated! Thanks to @hilsonp!
- 0.17.23
- Improved:
- Now we can preserve the logs into the file.
- Note: This option will be enabled automatically also when we flagging a red flag.
- File names can now be made platform-appropriate.
- Refactored:
- Some redundant implementations have been sorted out.
- 0.17.24
- New feature:
- If any conflicted files have been left, they will be reported.
- Fixed:
- Now the name of the conflicting file is shown on the conflict-resolving dialogue.
- Hidden files are now able to be merged again.
- No longer error caused at plug-in being loaded.
- Improved:
- Caching chunks are now limited in total size of cached chunks.
- 0.17.25
- Fixed:
- Now reading error will be reported.
- __0.17.1 to 0.17.25 has been moved into `update_old.md`__
- 0.17.26
- Fixed(Urgent):
- The modified document will be reflected in the storage now.
@@ -75,5 +20,28 @@
- The plugin data can be resolved when conflicted.
- The semaphore status display has been changed to count only.
- Applying to the storage will be concurrent with a few files.
- 0.17.28
-Fixed:
- Some messages have been refined.
- Boot sequence has been speeded up.
- Opening the local database multiple times in a short duration has been suppressed.
- Older migration logic.
- Note: If you have used 0.10.0 or lower and have not upgraded, you will need to run 0.17.27 or earlier once or reinstall Obsidian.
- 0.17.29
- Fixed:
- Requests of reading chunks online are now split into a reasonable(and configurable) size.
- No longer error message will be shown on Linux devices with hidden file synchronisation.
- Improved:
- The interval of reading chunks online is now configurable.
- Boot sequence has been speeded up, more.
- Misc:
- Messages on the boot sequence will now be more detailed. If you want to see them, please enable the verbose log.
- Logs became be kept for 1000 lines while the verbose log is enabled.
- 0.17.30
- Implemented:
- `Resolve all conflicted files` has been implemented.
- Fixed:
- Fixed a problem about reading chunks online when a file has more chunks than the concurrency limit.
- Rollbacked:
- Logs are kept only for 100 lines, again.
... To continue on to `updates_old.md`.

View File

@@ -68,7 +68,60 @@
- Hidden files have been synchronised again.
- Rename of files has been fixed again.
And, minor changes have been included.
- 0.17.16:
- Improved:
- Plugins and their settings no longer need scanning if changes are monitored.
- Now synchronising plugins and their settings are performed parallelly and faster.
- We can place `redflag2.md` to rebuild the database automatically while the boot sequence.
- Experimental:
- We can use a new adapter on PouchDB. This will make us smoother.
- Note: Not compatible with the older version.
- Fixed:
- The default batch size is smaller again.
- Plugins and their setting can be synchronised again.
- Hidden files and plugins are correctly scanned while rebuilding.
- Files with the name started `_` are also being performed conflict-checking.
- 0.17.17
- Fixed: Now we can merge JSON files even if we failed to compare items like null.
- 0.17.18
- Fixed: Fixed lack of error handling.
- 0.17.19
- Fixed: Error reporting has been ensured.
- 0.17.20
- Improved: Changes of hidden files will be notified to Obsidian.
- 0.17.21
- Fixed: Skip patterns now handle capital letters.
- Improved
- New configuration to avoid exceeding throttle capacity.
- We have been grateful to @karasevm!
- The conflicted `data.json` is no longer merged automatically.
- This behaviour is not configurable, unlike the `Use newer file if conflicted` of normal files.
- 0.17.22
- Fixed:
- Now hidden files will not be synchronised while we are not configured.
- Some processes could start without waiting for synchronisation to complete, but now they will wait for.
- Improved
- Now, by placing `redflag3.md`, we can discard the local database and fetch again.
- The document has been updated! Thanks to @hilsonp!
- 0.17.23
- Improved:
- Now we can preserve the logs into the file.
- Note: This option will be enabled automatically also when we flagging a red flag.
- File names can now be made platform-appropriate.
- Refactored:
- Some redundant implementations have been sorted out.
- 0.17.24
- New feature:
- If any conflicted files have been left, they will be reported.
- Fixed:
- Now the name of the conflicting file is shown on the conflict-resolving dialogue.
- Hidden files are now able to be merged again.
- No longer error caused at plug-in being loaded.
- Improved:
- Caching chunks are now limited in total size of cached chunks.
- 0.17.25
- Fixed:
- Now reading error will be reported.
### 0.16.0
- Now hidden files need not be scanned. Changes will be detected automatically.
- If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.