Compare commits

...

4 Commits

Author SHA1 Message Date
vorotamoroz
ef9b19c24b bump 2024-03-04 04:07:51 +00:00
vorotamoroz
4ed9494176 Changed:
- The default settings has been changed.
Improved:
- Default and preferred settings are applied on completion of the wizard.
Fixed:
- Now Initialisation `Fetch` will be performed smoothly and there will be fewer conflicts.
- No longer stuck while Handling transferred or initialised documents.
2024-03-04 04:07:11 +00:00
vorotamoroz
fcd56d59d5 bump 2024-03-01 08:33:37 +00:00
vorotamoroz
1cabfcfd19 Fixed:
- `Verify and repair all files` is no longer broken.
New feature::
- Now `Verify and repair all files` can restore or show history
Improved:
- Performance improved
2024-03-01 08:32:48 +00:00
9 changed files with 214 additions and 136 deletions

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.22.10",
"version": "0.22.12",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.22.10",
"version": "0.22.12",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.22.10",
"version": "0.22.12",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.22.10",
"version": "0.22.12",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",

View File

@@ -134,7 +134,7 @@ export class SetupLiveSync extends LiveSyncCommands {
} else if (setupType == setupAsMerge) {
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.fetchLocalWithKeepLocal();
await this.fetchLocalWithRebuild();
} else if (setupType == setupAgain) {
const confirm = "I know this operation will rebuild all my databases with files on this device, and files that are on the remote database and I didn't synchronize to any other devices will be lost and want to proceed indeed.";
if (await askSelectString(this.app, "Do you really want to do this?", ["Cancel", confirm]) != confirm) {
@@ -377,11 +377,13 @@ Of course, we are able to disable these features.`
await this.plugin.replicateAllFromServer(true);
await delay(1000);
await this.plugin.replicateAllFromServer(true);
await this.fetchRemoteChunks();
// if (!tryLessFetching) {
// await this.fetchRemoteChunks();
// }
await this.resumeReflectingDatabase();
await this.askHiddenFileConfiguration({ enableFetch: true });
}
async fetchLocalWithKeepLocal() {
async fetchLocalWithRebuild() {
return await this.fetchLocal(true);
}
async rebuildRemote() {

View File

@@ -1,13 +1,14 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO } from "./lib/src/types";
import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED } from "./lib/src/types";
import { createBinaryBlob, createTextBlob, delay, isDocContentSame } from "./lib/src/utils";
import { versionNumberString2Number } from "./lib/src/strbin";
import { decodeBinary, versionNumberString2Number } from "./lib/src/strbin";
import { Logger } from "./lib/src/logger";
import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb";
import { testCrypt } from "./lib/src/e2ee_v2";
import ObsidianLiveSyncPlugin from "./main";
import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils";
import { request, type ButtonComponent } from "obsidian";
import { request, type ButtonComponent, TFile } from "obsidian";
import { shouldBeIgnored } from "./lib/src/path";
export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -663,7 +664,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("localOnlyWithChunks");
await rebuildDB("localOnly");
})
)
.addButton((button) =>
@@ -773,9 +774,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
this.plugin.settings.passphrase = "";
}
if (isCloudantURI(this.plugin.settings.couchDB_URI)) {
this.plugin.settings.customChunkSize = 0;
// this.plugin.settings.customChunkSize = 0;
this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_CLOUDANT };
} else {
this.plugin.settings.customChunkSize = 50;
this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_SELF_HOSTED };
}
changeDisplay("30")
})
@@ -1053,7 +1055,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (!this.plugin.settings.isConfigured) {
this.plugin.settings.isConfigured = true;
await this.plugin.saveSettings();
await rebuildDB("localOnlyWithChunks");
await rebuildDB("localOnly");
Logger("All done! Please set up subsequent devices with 'Copy current settings as a new setup URI' and 'Use the copied setup URI'.", LOG_LEVEL_NOTICE);
await this.plugin.addOnSetup.command_copySetupURI();
} else {
@@ -1698,6 +1700,59 @@ ${stringifyYaml(pluginConfig)}`;
const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
hatchWarn.addClass("op-warn-info");
const addResult = (path: string, file: TFile | false, fileOnDB: LoadedEntry | false) => {
resultArea.appendChild(resultArea.createEl("div", {}, el => {
el.appendChild(el.createEl("h6", { text: path }));
el.appendChild(el.createEl("div", {}, infoGroupEl => {
infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Storage : Modified: ${!file ? `Missing:` : `${new Date(file.stat.mtime).toLocaleString()}, Size:${file.stat.size}`}` }))
infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size}`}` }))
}));
if (fileOnDB && file) {
el.appendChild(el.createEl("button", { text: "Show history" }, buttonEl => {
buttonEl.onClickEvent(() => {
this.plugin.showHistory(file, fileOnDB._id);
})
}))
}
if (file) {
el.appendChild(el.createEl("button", { text: "Storage -> Database" }, buttonEl => {
buttonEl.onClickEvent(() => {
this.plugin.updateIntoDB(file, undefined, true);
el.remove();
})
}))
}
if (fileOnDB) {
el.appendChild(el.createEl("button", { text: "Database -> Storage" }, buttonEl => {
buttonEl.onClickEvent(() => {
this.plugin.pullFile(this.plugin.getPath(fileOnDB), [], true, undefined, false);
el.remove();
})
}))
}
return el;
}))
}
const checkBetweenStorageAndDatabase = async (file: TFile, fileOnDB: LoadedEntry) => {
let content: Blob;
let dataContent: Blob;
if (fileOnDB.type == "newnote") {
dataContent = createBinaryBlob(decodeBinary(fileOnDB.data));
content = createBinaryBlob(await this.plugin.vaultAccess.vaultReadBinary(file));
} else {
dataContent = createTextBlob(fileOnDB.data);
content = createTextBlob(await this.plugin.vaultAccess.vaultRead(file));
}
if (await isDocContentSame(content, dataContent)) {
Logger(`Compare: SAME: ${file.path}`)
} else {
Logger(`Compare: CONTENT IS NOT MATCHED! ${file.path}`, LOG_LEVEL_NOTICE);
addResult(file.path, file, fileOnDB)
}
}
new Setting(containerHatchEl)
.setName("Verify and repair all files")
.setDesc("Compare the content of files between on local database and storage. If not matched, you will asked which one want to keep.")
@@ -1708,47 +1763,36 @@ ${stringifyYaml(pluginConfig)}`;
.setWarning()
.onClick(async () => {
const files = this.app.vault.getFiles();
const documents = [] as FilePathWithPrefix[];
const adn = this.plugin.localDatabase.findAllNormalDocs()
for await (const i of adn) documents.push(this.plugin.getPath(i));
const allPaths = [...new Set([...documents, ...files.map(e => e.path as FilePathWithPrefix)])];
let i = 0;
for (const file of files) {
for (const path of allPaths) {
i++;
Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL_NOTICE, "verify");
if (!await this.plugin.isTargetFile(file)) continue;
const fileOnDB = await this.plugin.localDatabase.getDBEntry(file.path as FilePathWithPrefix);
if (!fileOnDB) {
Logger(`Compare: Not found on local database: ${file.path}`, LOG_LEVEL_NOTICE);
Logger(`${i}/${files.length}\n${path}`, LOG_LEVEL_NOTICE, "verify");
if (shouldBeIgnored(path)) continue;
const abstractFile = this.plugin.vaultAccess.getAbstractFileByPath(path);
const fileOnStorage = abstractFile instanceof TFile ? abstractFile : false;
if (!await this.plugin.isTargetFile(path)) continue;
if (fileOnStorage && this.plugin.isFileSizeExceeded(fileOnStorage.stat.size)) continue;
const fileOnDB = await this.plugin.localDatabase.getDBEntry(path);
if (fileOnDB && this.plugin.isFileSizeExceeded(fileOnDB.size)) continue;
if (!fileOnDB && fileOnStorage) {
Logger(`Compare: Not found on the local database: ${path}`, LOG_LEVEL_NOTICE);
addResult(path, fileOnStorage, false)
continue;
}
let content: Blob;
if (fileOnDB.type == "newnote") {
content = createBinaryBlob(await this.plugin.vaultAccess.vaultReadBinary(file));
} else {
content = createTextBlob(await this.plugin.vaultAccess.vaultRead(file));
if (fileOnDB && !fileOnStorage) {
Logger(`Compare: Not found on the storage: ${path}`, LOG_LEVEL_NOTICE);
addResult(path, false, fileOnDB)
continue;
}
if (isDocContentSame(content, fileOnDB.data)) {
Logger(`Compare: SAME: ${file.path}`)
} else {
Logger(`Compare: CONTENT IS NOT MATCHED! ${file.path}`, LOG_LEVEL_NOTICE);
resultArea.appendChild(resultArea.createEl("div", {}, el => {
el.appendChild(el.createEl("h6", { text: file.path }));
el.appendChild(el.createEl("div", {}, infoGroupEl => {
infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Storage : Modified: ${new Date(file.stat.mtime).toLocaleString()}, Size:${file.stat.size}` }))
infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Database: Modified: ${new Date(fileOnDB.mtime).toLocaleString()}, Size:${content.size}` }))
}));
el.appendChild(el.createEl("button", { text: "Storage -> Database" }, buttonEl => {
buttonEl.onClickEvent(() => {
this.plugin.updateIntoDB(file, undefined, true);
el.remove();
})
}))
el.appendChild(el.createEl("button", { text: "Database -> Storage" }, buttonEl => {
buttonEl.onClickEvent(() => {
this.plugin.pullFile(file.path as FilePathWithPrefix, [], true, undefined, false);
el.remove();
})
}))
return el;
}))
if (fileOnStorage && fileOnDB) {
await checkBetweenStorageAndDatabase(fileOnStorage, fileOnDB)
}
}
Logger("done", LOG_LEVEL_NOTICE, "verify");
@@ -1923,7 +1967,7 @@ ${stringifyYaml(pluginConfig)}`;
toggle.setValue(!this.plugin.settings.useIndexedDBAdapter).onChange(async (value) => {
this.plugin.settings.useIndexedDBAdapter = !value;
await this.plugin.saveSettings();
await rebuildDB("localOnlyWithChunks");
await rebuildDB("localOnly");
})
);
@@ -2121,20 +2165,20 @@ ${stringifyYaml(pluginConfig)}`;
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("localOnlyWithChunks");
await rebuildDB("localOnly");
})
)
new Setting(containerMaintenanceEl)
.setName("Fetch rebuilt DB with all remote chunks")
.setDesc("Restore or reconstruct local database from remote database but use remote chunk .")
.setName("Fetch rebuilt DB (Save local documents before)")
.setDesc("Restore or reconstruct local database from remote database but use local chunks.")
.addButton((button) =>
button
.setButtonText("Fetch all")
.setButtonText("Save and Fetch")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await rebuildDB("localOnly");
await rebuildDB("localOnlyWithChunks");
})
)

Submodule src/lib updated: 8a8177c1f0...b9b70535ed

View File

@@ -1631,7 +1631,7 @@ We can perform a command in this file.
}
databaseQueueCount = reactiveSource(0);
databaseQueuedProcessor = new KeyedQueueProcessor(async (docs: EntryBody[]) => {
databaseQueuedProcessor = new QueueProcessor(async (docs: EntryBody[]) => {
const dbDoc = docs[0];
const path = this.getPath(dbDoc);
// If `Read chunks online` is disabled, chunks should be transferred before here.
@@ -1708,7 +1708,7 @@ We can perform a command in this file.
Logger(`Processing ${change.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
return;
}
this.databaseQueuedProcessor.enqueueWithKey(change.path, change);
this.databaseQueuedProcessor.enqueue(change);
}
return;
}, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).startPipeline().onUpdateProgress(() => {
@@ -1902,7 +1902,7 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
const CHOICE_DISMISS = "Dismiss";
const ret = await confirmWithMessage(this, "Cleaned", message, [CHOICE_FETCH, CHOICE_CLEAN, CHOICE_DISMISS], CHOICE_DISMISS, 30);
if (ret == CHOICE_FETCH) {
await performRebuildDB(this, "localOnlyWithChunks");
await performRebuildDB(this, "localOnly");
}
if (ret == CHOICE_CLEAN) {
const remoteDB = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
@@ -1936,7 +1936,7 @@ Or if you are sure know what had been happened, we can unlock the database from
const CHOICE_DISMISS = "Dismiss";
const ret = await confirmWithMessage(this, "Locked", message, [CHOICE_FETCH, CHOICE_DISMISS], CHOICE_DISMISS, 10);
if (ret == CHOICE_FETCH) {
await performRebuildDB(this, "localOnlyWithChunks");
await performRebuildDB(this, "localOnly");
}
}
}

View File

@@ -10,6 +10,23 @@ Note: we got a very performance improvement.
Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
#### Version history
- 0.22.12:
- Changed:
- The default settings has been changed.
- Improved:
- Default and preferred settings are applied on completion of the wizard.
- Fixed:
- Now Initialisation `Fetch` will be performed smoothly and there will be fewer conflicts.
- No longer stuck while Handling transferred or initialised documents.
- 0.22.11:
- Fixed:
- `Verify and repair all files` is no longer broken.
- New feature:
- Now `Verify and repair all files` is able to...
- Restore if the file only in the local database.
- Show the history.
- Improved:
- Performance improved.
- 0.22.10
- Fixed:
- No longer unchanged hidden files and customisations are saved and transferred now.
@@ -64,80 +81,5 @@ Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50
- `Copy setup URI` is now `Copy current settings as a new setup URI`
- `Setup Wizard` is now `Minimal Setup`
- `Check database configuration` is now `Check and Fix database configuration`
- 0.22.5
- Fixed:
- Some description of settings have been refined
- New feature:
- TroubleShooting is now shown in the setting dialogue.
- 0.22.4
- Fixed:
- Now the result of conflict resolution could be surely written into the storage.
- Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
- Some wrong log messages were fixed.
- Change handling now has become more stable.
- Some event handling became to be safer.
- Improved:
- Dumping document information shows conflicts and revisions.
- The timestamp-only differences can be surely cached.
- Timestamp difference detection can be rounded by two seconds.
- Refactored:
- A bit of organisation to write the test.
- 0.22.3
- Fixed:
- No longer detects storage changes which have been caused by Self-hosted LiveSync itself.
- Setting sync file will be detected only if it has been configured now.
- And its log will be shown only while the verbose log is enabled.
- Customisation file enumeration has got less blingy.
- Deletion of files is now reliably synchronised.
- Fixed and improved:
- In-editor-status is now shown in the following areas:
- Note editing pane (Source mode and live-preview mode).
- New tab pane.
- Canvas pane.
- 0.22.2
- Fixed:
- Now the results of resolving conflicts are surely synchronised.
- Modified:
- Some setting items got new clear names. (`Sync Settings` -> `Targets`).
- New feature:
- We can limit the synchronising files by their size. (`Sync Settings` -> `Targets` -> `Maximum file size`).
- It depends on the size of the newer one.
- At Obsidian 1.5.3 on mobile, we should set this to around 50MB to avoid restarting Obsidian.
- Now the settings could be stored in a specific markdown file to synchronise or switch it (`General Setting` -> `Share settings via markdown`).
- [Screwdriver](https://github.com/vrtmrz/obsidian-screwdriver) is quite good, but mostly we only need this.
- Customisation of the obsoleted device is now able to be deleted at once.
- We have to put the maintenance mode in at the Customisation sync dialogue.
- 0.22.1
- New feature:
- We can perform automatic conflict resolution for inactive files, and postpone only manual ones by `Postpone manual resolution of inactive files`.
- Now we can see the image in the document history dialogue.
- We can see the difference of the image, in the document history dialogue.
- And also we can highlight differences.
- Improved:
- Hidden file sync has been stabilised.
- Now automatically reloads the conflict-resolution dialogue when new conflicted revisions have arrived.
- Fixed:
- No longer periodic process runs after unloading the plug-in.
- Now the modification of binary files is surely stored in the storage.
- 0.22.0
- Refined:
- Task scheduling logics has been rewritten.
- Screen updates are also now efficient.
- Possibly many bugs and fragile behaviour has been fixed.
- Status updates and logging have been thinned out to display.
- Fixed:
- Remote-chunk-fetching now works with keeping request intervals
- New feature:
- We can show only the icons in the editor.
- Progress indicators have been more meaningful:
- 📥 Unprocessed transferred items
- 📄 Working database operation
- 💾 Working write storage processes
- ⏳ Working read storage processes
- 🛫 Pending read storage processes
- ⚙️ Working or pending storage processes of hidden files
- 🧩 Waiting chunks
- 🔌 Working Customisation items (Configuration, snippets and plug-ins)
... To continue on to `updates_old.md`.

View File

@@ -1,3 +1,93 @@
### 0.22.0
A few years passed since Self-hosted LiveSync was born, and our codebase had been very complicated. This could be patient now, but it should be a tremendous hurt.
Therefore at v0.22.0, for future maintainability, I refined task scheduling logic totally.
Of course, I think this would be our suffering in some cases. However, I would love to ask you for your cooperation and contribution.
Sorry for being absent so much long. And thank you for your patience!
Note: we got a very performance improvement.
Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
#### Version history
- 0.22.5
- Fixed:
- Some description of settings have been refined
- New feature:
- TroubleShooting is now shown in the setting dialogue.
- 0.22.4
- Fixed:
- Now the result of conflict resolution could be surely written into the storage.
- Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
- Some wrong log messages were fixed.
- Change handling now has become more stable.
- Some event handling became to be safer.
- Improved:
- Dumping document information shows conflicts and revisions.
- The timestamp-only differences can be surely cached.
- Timestamp difference detection can be rounded by two seconds.
- Refactored:
- A bit of organisation to write the test.
- 0.22.3
- Fixed:
- No longer detects storage changes which have been caused by Self-hosted LiveSync itself.
- Setting sync file will be detected only if it has been configured now.
- And its log will be shown only while the verbose log is enabled.
- Customisation file enumeration has got less blingy.
- Deletion of files is now reliably synchronised.
- Fixed and improved:
- In-editor-status is now shown in the following areas:
- Note editing pane (Source mode and live-preview mode).
- New tab pane.
- Canvas pane.
- 0.22.2
- Fixed:
- Now the results of resolving conflicts are surely synchronised.
- Modified:
- Some setting items got new clear names. (`Sync Settings` -> `Targets`).
- New feature:
- We can limit the synchronising files by their size. (`Sync Settings` -> `Targets` -> `Maximum file size`).
- It depends on the size of the newer one.
- At Obsidian 1.5.3 on mobile, we should set this to around 50MB to avoid restarting Obsidian.
- Now the settings could be stored in a specific markdown file to synchronise or switch it (`General Setting` -> `Share settings via markdown`).
- [Screwdriver](https://github.com/vrtmrz/obsidian-screwdriver) is quite good, but mostly we only need this.
- Customisation of the obsoleted device is now able to be deleted at once.
- We have to put the maintenance mode in at the Customisation sync dialogue.
- 0.22.1
- New feature:
- We can perform automatic conflict resolution for inactive files, and postpone only manual ones by `Postpone manual resolution of inactive files`.
- Now we can see the image in the document history dialogue.
- We can see the difference of the image, in the document history dialogue.
- And also we can highlight differences.
- Improved:
- Hidden file sync has been stabilised.
- Now automatically reloads the conflict-resolution dialogue when new conflicted revisions have arrived.
- Fixed:
- No longer periodic process runs after unloading the plug-in.
- Now the modification of binary files is surely stored in the storage.
- 0.22.0
- Refined:
- Task scheduling logics has been rewritten.
- Screen updates are also now efficient.
- Possibly many bugs and fragile behaviour has been fixed.
- Status updates and logging have been thinned out to display.
- Fixed:
- Remote-chunk-fetching now works with keeping request intervals
- New feature:
- We can show only the icons in the editor.
- Progress indicators have been more meaningful:
- 📥 Unprocessed transferred items
- 📄 Working database operation
- 💾 Working write storage processes
- ⏳ Working read storage processes
- 🛫 Pending read storage processes
- ⚙️ Working or pending storage processes of hidden files
- 🧩 Waiting chunks
- 🔌 Working Customisation items (Configuration, snippets and plug-ins)
... To continue on to `updates_old.md`.
### 0.21.0
The E2EE encryption V2 format has been reverted. That was probably the cause of the glitch.
Instead, to maintain efficiency, files are treated with Blob until just before saving. Along with this, the old-fashioned encryption format has also been discontinued.