Compare commits

..

8 Commits

Author SHA1 Message Date
vorotamoroz
f5e28b5e1c bump 2022-12-21 14:51:59 +09:00
vorotamoroz
0c206226b1 New feature
- JSON merging for data and canvas.
2022-12-21 14:51:39 +09:00
vorotamoroz
1ad5dcc1cc bump 2022-12-16 18:56:28 +09:00
vorotamoroz
a512566e5b New feature
- We can merge conflicted documents automatically if sensible.

Fixed:
- Writing to the storage will be pended while they have conflicts after replication.

Minor changes included.
2022-12-16 18:55:04 +09:00
vorotamoroz
02de82af46 bump 2022-12-06 18:03:31 +09:00
vorotamoroz
840e03a2d3 Fixed:
- Now we can verify and repair database again.
2022-12-06 17:59:48 +09:00
vorotamoroz
96b676caf3 bump 2022-12-05 19:53:24 +09:00
vorotamoroz
a8219de375 Improved:
- Splitting markdown
- Saving chunks

Changed:
- Chunk ID numbering rules

Fixed:
- Just weed.
2022-12-05 19:37:24 +09:00
11 changed files with 510 additions and 125 deletions

View File

@@ -13,7 +13,7 @@ if you want to view the source, please visit the github repository of this plugi
const prod = process.argv[2] === "production";
const manifestJson = JSON.parse(fs.readFileSync("./manifest.json"));
const packageJson = JSON.parse(fs.readFileSync("./package.json"));
const updateInfo = JSON.stringify("PATCHED-"+fs.readFileSync("./updates.md") + "");
const updateInfo = JSON.stringify(fs.readFileSync("./updates.md") + "");
esbuild
.build({
banner: {

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.16.8",
"version": "0.17.3",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

52
package-lock.json generated
View File

@@ -1,26 +1,24 @@
{
"name": "obsidian-livesync",
"version": "0.16.8",
"version": "0.17.3",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.16.8",
"version": "0.17.3",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",
"esbuild": "0.15.15",
"esbuild-svelte": "^0.7.3",
"idb": "^7.1.1",
"xxhash-wasm": "^0.4.2",
"xxhashjs": "^0.2.2"
"xxhash-wasm": "^0.4.2"
},
"devDependencies": {
"@types/diff-match-patch": "^1.0.32",
"@types/pouchdb": "^6.4.0",
"@types/pouchdb-browser": "^6.1.3",
"@types/xxhashjs": "^0.2.2",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",
"builtin-modules": "^3.3.0",
@@ -444,15 +442,6 @@
"@types/estree": "*"
}
},
"node_modules/@types/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-+hlk/W1kgnZn0vR22XNhxHk/qIRQYF54i0UTF2MwBAPd0e7xSy+jKOJwSwTdRQrNnOMRVv+vsh8ITV0uyhp2yg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "5.44.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz",
@@ -883,11 +872,6 @@
"node": ">= 8"
}
},
"node_modules/cuint": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
"integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
},
"node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3360,14 +3344,6 @@
"resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
"integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
},
"node_modules/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
"dependencies": {
"cuint": "^0.2.2"
}
},
"node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -3760,15 +3736,6 @@
"@types/estree": "*"
}
},
"@types/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-+hlk/W1kgnZn0vR22XNhxHk/qIRQYF54i0UTF2MwBAPd0e7xSy+jKOJwSwTdRQrNnOMRVv+vsh8ITV0uyhp2yg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@typescript-eslint/eslint-plugin": {
"version": "5.44.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz",
@@ -4044,11 +4011,6 @@
"which": "^2.0.1"
}
},
"cuint": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
"integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
},
"debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -5716,14 +5678,6 @@
"resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
"integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
},
"xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
"requires": {
"cuint": "^0.2.2"
}
},
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.16.8",
"version": "0.17.3",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",
@@ -16,7 +16,6 @@
"@types/diff-match-patch": "^1.0.32",
"@types/pouchdb": "^6.4.0",
"@types/pouchdb-browser": "^6.1.3",
"@types/xxhashjs": "^0.2.2",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",
"builtin-modules": "^3.3.0",
@@ -38,7 +37,6 @@
"esbuild": "0.15.15",
"esbuild-svelte": "^0.7.3",
"idb": "^7.1.1",
"xxhash-wasm": "^0.4.2",
"xxhashjs": "^0.2.2"
"xxhash-wasm": "^0.4.2"
}
}
}

View File

@@ -7,7 +7,6 @@ import { EntryDoc, LOG_LEVEL } from "./lib/src/types.js";
import { enableEncryption } from "./lib/src/utils.js";
import { isCloudantURI, isValidRemoteCouchDBURI } from "./lib/src/utils_couchdb.js";
import { id2path, path2id } from "./utils.js";
import XXH from "xxhashjs";
export class LocalPouchDB extends LocalPouchDBBase {
@@ -34,13 +33,6 @@ export class LocalPouchDB extends LocalPouchDBBase {
await this.kvDB.destroy();
}
async prepareHashFunctions() {
if (this.h32 != null) return;
// const { h32, h32Raw } = await xxhash();
this.h32 = (input: string, seed: number) => (XXH.h32(input, seed).toString(16))// h32;
this.h32Raw = (input: Uint8Array, seed: number) => (XXH.h32(input.buffer, seed).toNumber())// h32;
}
last_successful_post = false;
getLastPostFailedBySize() {
return !this.last_successful_post;

View File

@@ -1,7 +1,7 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "obsidian";
import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, RemoteDBSettings } from "./lib/src/types";
import { path2id, id2path } from "./utils";
import { delay, versionNumberString2Number } from "./lib/src/utils";
import { delay, Semaphore, versionNumberString2Number } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
import { testCrypt } from "./lib/src/e2ee_v2";
@@ -974,6 +974,24 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl)
.setName("Disable sensible auto merging on markdown files")
.setDesc("If this switch is turned on, a merge dialog will be displayed, even if the sensible-merge is possible automatically. (Turn on to previous behavior)")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.disableMarkdownAutoMerge).onChange(async (value) => {
this.plugin.settings.disableMarkdownAutoMerge = value;
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl)
.setName("Write documents after synchronization even if they have conflict")
.setDesc("Turn on to previous behavior")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.writeDocumentsIfConflicted).onChange(async (value) => {
this.plugin.settings.writeDocumentsIfConflicted = value;
await this.plugin.saveSettings();
})
);
new Setting(containerSyncSettingEl)
@@ -1288,7 +1306,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
new Setting(containerHatchEl)
.setName("Make report to inform the issue")
.setDesc("Verify and repair all files and update database without restoring")
.addButton((button) =>
button
.setButtonText("Make report")
@@ -1368,21 +1385,28 @@ ${stringifyYaml(pluginConfig)}`;
.setDisabled(false)
.setWarning()
.onClick(async () => {
const semaphore = Semaphore(10);
const files = this.app.vault.getFiles();
Logger("Verify and repair all files started", LOG_LEVEL.NOTICE, "verify");
// const notice = NewNotice("", 0);
let i = 0;
for (const file of files) {
i++;
Logger(`Update into ${file.path}`);
Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
const processes = files.map(e => (async (file) => {
const releaser = await semaphore.acquire(1, "verifyAndRepair");
try {
await this.plugin.updateIntoDB(file);
Logger(`Update into ${file.path}`);
await this.plugin.updateIntoDB(file, false, null, true);
i++;
Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
} catch (ex) {
Logger("could not update:");
i++;
Logger(`Error while verifyAndRepair`, LOG_LEVEL.NOTICE);
Logger(ex);
} finally {
releaser();
}
}
)(e));
await Promise.all(processes);
Logger("done", LOG_LEVEL.NOTICE, "verify");
})
);

Submodule src/lib updated: 85bb3556ba...bf8ab8883d

View File

@@ -1,5 +1,5 @@
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest, App, } from "obsidian";
import { diff_match_patch } from "diff-match-patch";
import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry } from "./lib/src/types";
import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo } from "./types";
@@ -29,7 +29,7 @@ import { DocumentHistoryModal } from "./DocumentHistoryModal";
import { clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, id2path, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger } from "./utils";
import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
import { decrypt, encrypt } from "./lib/src/e2ee_v2";
const isDebug = false;
@@ -709,7 +709,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
clearAllPeriodic();
clearAllTriggers();
window.removeEventListener("visibilitychange", this.watchWindowVisibility);
window.removeEventListener("online", this.watchOnline)
window.removeEventListener("online", this.watchOnline);
Logger("unloading plugin");
}
@@ -1365,13 +1365,30 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} else if (targetFile instanceof TFile) {
const doc = change;
const file = targetFile;
await this.doc2storage_modify(doc, file);
if (!this.settings.checkConflictOnlyOnOpen) {
this.queueConflictedCheck(file);
} else {
const af = app.workspace.getActiveFile();
if (af && af.path == file.path) {
const queueConflictCheck = () => {
if (!this.settings.checkConflictOnlyOnOpen) {
this.queueConflictedCheck(file);
return true;
} else {
const af = app.workspace.getActiveFile();
if (af && af.path == file.path) {
this.queueConflictedCheck(file);
return true;
}
}
return false;
}
if (this.settings.writeDocumentsIfConflicted) {
await this.doc2storage_modify(doc, file);
queueConflictCheck();
} else {
const d = await this.localDatabase.getDBEntryMeta(id2path(change._id), { conflicts: true })
if (d && !d._conflicts) {
await this.doc2storage_modify(doc, file);
} else {
if (!queueConflictCheck()) {
Logger(`${id2path(change._id)} is conflicted, write to the storage has been pended.`, LOG_LEVEL.NOTICE);
}
}
}
} else {
@@ -1954,6 +1971,193 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
return false;
}
//TODO: TIDY UP
async mergeSensibly(path: string, baseRev: string, currentRev: string, conflictedRev: string): Promise<Diff[] | false> {
const baseLeaf = await this.getConflictedDoc(path, baseRev);
const leftLeaf = await this.getConflictedDoc(path, currentRev);
const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
let autoMerge = false;
if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
return false;
}
// diff between base and each revision
const dmp = new diff_match_patch();
const mapLeft = dmp.diff_linesToChars_(baseLeaf.data, leftLeaf.data);
const diffLeftSrc = dmp.diff_main(mapLeft.chars1, mapLeft.chars2, false);
dmp.diff_charsToLines_(diffLeftSrc, mapLeft.lineArray);
const mapRight = dmp.diff_linesToChars_(baseLeaf.data, rightLeaf.data);
const diffRightSrc = dmp.diff_main(mapRight.chars1, mapRight.chars2, false);
dmp.diff_charsToLines_(diffRightSrc, mapRight.lineArray);
function splitDiffPiece(src: Diff[]): Diff[] {
const ret = [] as Diff[];
do {
const d = src.shift();
const pieces = d[1].split(/([^\n]*\n)/).filter(f => f != "");
if (typeof (d) == "undefined") {
break;
}
if (d[0] != DIFF_DELETE) {
ret.push(...(pieces.map(e => [d[0], e] as Diff)));
}
if (d[0] == DIFF_DELETE) {
const nd = src.shift();
if (typeof (nd) != "undefined") {
const piecesPair = nd[1].split(/([^\n]*\n)/).filter(f => f != "");
if (nd[0] == DIFF_INSERT) {
// it might be pair
for (const pt of pieces) {
ret.push([d[0], pt]);
const pairP = piecesPair.shift();
if (typeof (pairP) != "undefined") ret.push([DIFF_INSERT, pairP]);
}
ret.push(...(piecesPair.map(e => [nd[0], e] as Diff)));
} else {
ret.push(...(pieces.map(e => [d[0], e] as Diff)));
ret.push(...(piecesPair.map(e => [nd[0], e] as Diff)));
}
} else {
ret.push(...(pieces.map(e => [0, e] as Diff)));
}
}
} while (src.length > 0);
return ret;
}
const diffLeft = splitDiffPiece(diffLeftSrc);
const diffRight = splitDiffPiece(diffRightSrc);
let rightIdx = 0;
let leftIdx = 0;
const merged = [] as Diff[];
autoMerge = true;
LOOP_MERGE:
do {
if (leftIdx >= diffLeft.length && rightIdx >= diffRight.length) {
break LOOP_MERGE;
}
const leftItem = diffLeft[leftIdx] ?? [0, ""];
const rightItem = diffRight[rightIdx] ?? [0, ""];
leftIdx++;
rightIdx++;
// when completely same, leave it .
if (leftItem[0] == DIFF_EQUAL && rightItem[0] == DIFF_EQUAL && leftItem[1] == rightItem[1]) {
merged.push(leftItem);
continue;
}
if (leftItem[0] == DIFF_DELETE && rightItem[0] == DIFF_DELETE && leftItem[1] == rightItem[1]) {
// when deleted evenly,
const nextLeftIdx = leftIdx;
const nextRightIdx = rightIdx;
const [nextLeftItem, nextRightItem] = [diffLeft[nextLeftIdx] ?? [0, ""], diffRight[nextRightIdx] ?? [0, ""]];
if ((nextLeftItem[0] == DIFF_INSERT && nextRightItem[0] == DIFF_INSERT) && nextLeftItem[1] != nextRightItem[1]) {
//but next line looks like different
autoMerge = false;
break;
} else {
merged.push(leftItem);
continue;
}
}
// when inserted evenly
if (leftItem[0] == DIFF_INSERT && rightItem[0] == DIFF_INSERT) {
if (leftItem[1] == rightItem[1]) {
merged.push(leftItem);
continue;
} else {
// sort by file date.
if (leftLeaf.mtime <= rightLeaf.mtime) {
merged.push(leftItem);
merged.push(rightItem);
continue;
} else {
merged.push(rightItem);
merged.push(leftItem);
continue;
}
}
}
// when on inserting, index should be fixed again.
if (leftItem[0] == DIFF_INSERT) {
rightIdx--;
merged.push(leftItem);
continue;
}
if (rightItem[0] == DIFF_INSERT) {
leftIdx--;
merged.push(rightItem);
continue;
}
// except insertion, the line should not be different.
if (rightItem[1] != leftItem[1]) {
//TODO: SHOULD BE PANIC.
Logger(`MERGING PANIC:${leftItem[0]},${leftItem[1]} == ${rightItem[0]},${rightItem[1]}`, LOG_LEVEL.VERBOSE);
autoMerge = false;
break LOOP_MERGE;
}
if (leftItem[0] == DIFF_DELETE) {
if (rightItem[0] == DIFF_EQUAL) {
merged.push(leftItem);
continue;
} else {
//we cannot perform auto merge.
autoMerge = false;
break LOOP_MERGE;
}
}
if (rightItem[0] == DIFF_DELETE) {
if (leftItem[0] == DIFF_EQUAL) {
merged.push(rightItem);
continue;
} else {
//we cannot perform auto merge.
autoMerge = false;
break LOOP_MERGE;
}
}
Logger(`Weird condition:${leftItem[0]},${leftItem[1]} == ${rightItem[0]},${rightItem[1]}`, LOG_LEVEL.VERBOSE);
// here is the exception
break LOOP_MERGE;
} while (leftIdx < diffLeft.length || rightIdx < diffRight.length);
if (autoMerge) {
Logger(`Sensibly merge available`, LOG_LEVEL.VERBOSE);
return merged;
} else {
return false;
}
}
async mergeObject(path: string, baseRev: string, currentRev: string, conflictedRev: string): Promise<string | false> {
const baseLeaf = await this.getConflictedDoc(path, baseRev);
const leftLeaf = await this.getConflictedDoc(path, currentRev);
const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
return false;
}
const baseObj = { data: tryParseJSON(baseLeaf.data, {}) } as Record<string | number | symbol, any>;
const leftObj = { data: tryParseJSON(leftLeaf.data, {}) } as Record<string | number | symbol, any>;
const rightObj = { data: tryParseJSON(rightLeaf.data, {}) } as Record<string | number | symbol, any>;
const diffLeft = generatePatchObj(baseObj, leftObj);
const diffRight = generatePatchObj(baseObj, rightObj);
const patches = [
{ mtime: leftLeaf.mtime, patch: diffLeft },
{ mtime: rightLeaf.mtime, patch: diffRight }
].sort((a, b) => a.mtime - b.mtime);
let newObj = { ...baseObj };
try {
for (const patch of patches) {
newObj = applyPatch(newObj, patch.patch);
}
return JSON.stringify(newObj.data);
} catch (ex) {
Logger("Could not merge object");
Logger(ex, LOG_LEVEL.VERBOSE)
return false;
}
}
/**
* Getting file conflicted status.
@@ -1966,9 +2170,56 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (test == null) return false;
if (!test._conflicts) return false;
if (test._conflicts.length == 0) return false;
const conflicts = test._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
if ((isSensibleMargeApplicable(path) || isObjectMargeApplicable(path)) && !this.settings.disableMarkdownAutoMerge) {
const conflictedRev = conflicts[0];
const conflictedRevNo = Number(conflictedRev.split("-")[0]);
//Search
const revFrom = (await this.localDatabase.localDatabase.get(id2path(path), { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first().rev ?? "";
let p = undefined;
if (commonBase) {
if (isSensibleMargeApplicable(path)) {
const result = await this.mergeSensibly(path, commonBase, test._rev, conflictedRev);
if (result) {
p = result.filter(e => e[0] != DIFF_DELETE).map((e) => e[1]).join("");
// can be merged.
Logger(`Sensible merge:${path}`, LOG_LEVEL.INFO);
} else {
Logger(`Sensible merge is not applicable.`, LOG_LEVEL.VERBOSE);
}
} else if (isObjectMargeApplicable(path)) {
// can be merged.
const result = await this.mergeObject(path, commonBase, test._rev, conflictedRev);
if (result) {
Logger(`Object merge:${path}`, LOG_LEVEL.INFO);
p = result;
} else {
Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
}
}
if (p != undefined) {
// remove conflicted revision.
await this.localDatabase.deleteDBEntry(path, { rev: conflictedRev });
const file = getAbstractFileByPath(path) as TFile;
if (file) {
await this.app.vault.modify(file, p);
await this.updateIntoDB(file);
} else {
const newFile = await this.app.vault.create(path, p);
await this.updateIntoDB(newFile);
}
await this.pullFile(path);
Logger(`Automatically merged (sensible) :${path}`, LOG_LEVEL.INFO);
return true;
}
}
}
// should be one or more conflicts;
const leftLeaf = await this.getConflictedDoc(path, test._rev);
const rightLeaf = await this.getConflictedDoc(path, test._conflicts[0]);
const rightLeaf = await this.getConflictedDoc(path, conflicts[0]);
if (leftLeaf == false) {
// what's going on..
Logger(`could not get current revisions:${path}`, LOG_LEVEL.NOTICE);
@@ -1976,7 +2227,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
if (rightLeaf == false) {
// Conflicted item could not load, delete this.
await this.localDatabase.deleteDBEntry(path, { rev: test._conflicts[0] });
await this.localDatabase.deleteDBEntry(path, { rev: conflicts[0] });
await this.pullFile(path, null, true);
Logger(`could not get old revisions, automatically used newer one:${path}`, LOG_LEVEL.NOTICE);
return true;
@@ -2032,11 +2283,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const toDelete = selected;
const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
if (toDelete == "") {
//concat both,
// write data,and delete both old rev.
// concat both,
// delete conflicted revision and write a new file, store it again.
const p = conflictCheckResult.diff.map((e) => e[1]).join("");
await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.left.rev });
await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.right.rev });
await this.localDatabase.deleteDBEntry(filename, { rev: testDoc._conflicts[0] });
const file = getAbstractFileByPath(filename) as TFile;
if (file) {
await this.app.vault.modify(file, p);
@@ -2092,7 +2342,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger(ex);
}
}
}, 1000);
}, 100);
}
async showIfConflicted(filename: string) {
@@ -2192,7 +2442,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData) {
async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData, force?: boolean) {
if (!this.isTargetFile(file)) return;
if (shouldBeIgnored(file.path)) {
return;
@@ -2234,15 +2484,24 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (recentlyTouched(file)) {
return true;
}
const old = await this.localDatabase.getDBEntry(fullPath, null, false, false);
if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted || old.deleted, };
const newData = { data: d.data, deleted: d._deleted || d.deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) {
Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return true;
try {
const old = await this.localDatabase.getDBEntry(fullPath, null, false, false);
if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted || old.deleted, };
const newData = { data: d.data, deleted: d._deleted || d.deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) {
Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return true;
}
// d._rev = old._rev;
}
// d._rev = old._rev;
} catch (ex) {
if (force) {
Logger(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
} else {
Logger(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
}
return !force;
}
return false;
});
@@ -2702,8 +2961,33 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (!("_conflicts" in doc)) return false;
if (doc._conflicts.length == 0) return false;
Logger(`Hidden file conflicted:${id2filenameInternalChunk(id)}`);
const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
const revA = doc._rev;
const revB = doc._conflicts[0];
const revB = conflicts[0];
const conflictedRev = conflicts[0];
const conflictedRevNo = Number(conflictedRev.split("-")[0]);
//Search
const revFrom = (await this.localDatabase.localDatabase.get(id, { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first().rev ?? "";
const result = await this.mergeObject(id, commonBase, doc._rev, conflictedRev);
if (result) {
Logger(`Object merge:${id}`, LOG_LEVEL.INFO);
const filename = id2filenameInternalChunk(id);
const isExists = await this.app.vault.adapter.exists(filename);
if (!isExists) {
await this.ensureDirectoryEx(filename);
}
await this.app.vault.adapter.write(filename, result);
const stat = await this.app.vault.adapter.stat(filename);
await this.storeInternalFileToDatabase({ path: filename, ...stat });
await this.extractInternalFileFromDatabase(filename);
await this.localDatabase.localDatabase.remove(id, revB);
return this.resolveConflictOnInternalFile(id);
} else {
Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
}
const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
// determine which revision should been deleted.

View File

@@ -72,4 +72,129 @@ export function retrieveMemoObject<T>(key: string): T | false {
}
export function disposeMemoObject(key: string) {
delete memos[key];
}
}
export function isSensibleMargeApplicable(path: string) {
if (path.endsWith(".md")) return true;
return false;
}
export function isObjectMargeApplicable(path: string) {
if (path.endsWith(".canvas")) return true;
if (path.endsWith(".json")) return true;
return false;
}
export function tryParseJSON(str: string, fallbackValue?: any) {
try {
return JSON.parse(str);
} catch (ex) {
return fallbackValue;
}
}
const MARK_OPERATOR = `\u{0001}`;
const MARK_DELETED = `${MARK_OPERATOR}__DELETED`;
const MARK_ISARRAY = `${MARK_OPERATOR}__ARRAY`;
const MARK_SWAPPED = `${MARK_OPERATOR}__SWAP`;
function unorderedArrayToObject(obj: Array<any>) {
return obj.map(e => ({ [e.id as string]: e })).reduce((p, c) => ({ ...p, ...c }), {})
}
function objectToUnorderedArray(obj: object) {
const entries = Object.entries(obj);
if (entries.some(e => e[0] != e[1]?.id)) throw new Error("Item looks like not unordered array")
return entries.map(e => e[1]);
}
function generatePatchUnorderedArray(from: Array<any>, to: Array<any>) {
if (from.every(e => typeof (e) == "object" && ("id" in e)) && to.every(e => typeof (e) == "object" && ("id" in e))) {
const fObj = unorderedArrayToObject(from);
const tObj = unorderedArrayToObject(to);
const diff = generatePatchObj(fObj, tObj);
if (Object.keys(diff).length > 0) {
return { [MARK_ISARRAY]: diff };
} else {
return {};
}
}
return { [MARK_SWAPPED]: to };
}
export function generatePatchObj(from: Record<string | number | symbol, any>, to: Record<string | number | symbol, any>) {
const entries = Object.entries(from);
const tempMap = new Map<string | number | symbol, any>(entries);
const ret = {} as Record<string | number | symbol, any>;
const newEntries = Object.entries(to);
for (const [key, value] of newEntries) {
if (!tempMap.has(key)) {
//New
ret[key] = value;
tempMap.delete(key);
} else {
//Exists
const v = tempMap.get(key);
if (typeof (v) !== typeof (value) || (Array.isArray(v) !== Array.isArray(value))) {
//if type is not match, replace completely.
ret[key] = { [MARK_SWAPPED]: value };
} else {
if (typeof (v) == "object" && typeof (value) == "object" && !Array.isArray(v) && !Array.isArray(value)) {
const wk = generatePatchObj(v, value);
if (Object.keys(wk).length > 0) ret[key] = wk;
} else if (typeof (v) == "object" && typeof (value) == "object" && Array.isArray(v) && Array.isArray(value)) {
const wk = generatePatchUnorderedArray(v, value);
if (Object.keys(wk).length > 0) ret[key] = wk;
} else if (typeof (v) != "object" && typeof (value) != "object") {
if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
ret[key] = value;
}
} else {
if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
ret[key] = { [MARK_SWAPPED]: value };
}
}
}
tempMap.delete(key);
}
}
//Not used item, means deleted one
for (const [key,] of tempMap) {
ret[key] = MARK_DELETED
}
return ret;
}
export function applyPatch(from: Record<string | number | symbol, any>, patch: Record<string | number | symbol, any>) {
const ret = from;
const patches = Object.entries(patch);
for (const [key, value] of patches) {
if (value == MARK_DELETED) {
delete ret[key];
continue;
}
if (typeof (value) == "object") {
if (MARK_SWAPPED in value) {
ret[key] = value[MARK_SWAPPED];
continue;
}
if (MARK_ISARRAY in value) {
if (!(key in ret)) ret[key] = [];
if (!Array.isArray(ret[key])) {
throw new Error("Patch target type is mismatched (array to something)");
}
const orgArrayObject = unorderedArrayToObject(ret[key]);
const appliedObject = applyPatch(orgArrayObject, value[MARK_ISARRAY]);
const appliedArray = objectToUnorderedArray(appliedObject);
ret[key] = [...appliedArray];
} else {
if (!(key in ret)) {
ret[key] = value;
continue;
}
ret[key] = applyPatch(ret[key], value);
}
} else {
ret[key] = value;
}
}
return ret;
}

View File

@@ -106,7 +106,8 @@
}
.CodeMirror-wrap::before,
.cm-s-obsidian>.cm-editor::before {
.cm-s-obsidian>.cm-editor::before,
.canvas-wrapper::before {
content: var(--slsmessage);
text-align: right;
white-space: pre-wrap;
@@ -122,6 +123,10 @@
filter: grayscale(100%);
}
.canvas-wrapper::before {
right: 48px;
}
.CodeMirror-wrap::before {
right: 0px;
}

View File

@@ -1,3 +1,28 @@
### 0.17.0
- 0.17.0 has no surfaced changes but the design of saving chunks has been changed. They have compatibility but changing files after upgrading makes different chunks than before 0.16.x.
Please rebuild databases once if you have been worried about storage usage.
- Improved:
- Splitting markdown
- Saving chunks
- Changed:
- Chunk ID numbering rules
#### Minors
- 0.17.1
- Fixed: Now we can verify and repair the database.
- Refactored inside.
- 0.17.2
- New feature
- We can merge conflicted documents automatically if sensible.
- Fixed
- Writing to the storage will be pended while they have conflicts after replication.
- 0.17.3
- Now we supported canvas! And conflicted JSON files are also synchronised with merging its content if they are obvious.
### 0.16.0
- Now hidden files need not be scanned. Changes will be detected automatically.
- If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.
@@ -31,25 +56,3 @@
Note:
Before 0.16.5, LiveSync had some issues making chunks. In this case, synchronisation had became been always failing after a corrupted one should be made. After 0.16.6, the corrupted chunk is automatically detected. Sorry for troubling you but please do `rebuild everything` when this plug-in notified so.
### 0.15.0
- Outdated configuration items have been removed.
- Setup wizard has been implemented!
I appreciate for reviewing and giving me advice @Pouhon158!
#### Minors
- 0.15.1 Missed the stylesheet.
- 0.15.2 The wizard has been improved and documented!
- 0.15.3 Fixed the issue about locking/unlocking remote database while rebuilding in the wizard.
- 0.15.4 Fixed issues about asynchronous processing (e.g., Conflict check or hidden file detection)
- 0.15.5 Add new features for setting Self-hosted LiveSync up more easier.
- 0.15.6 File tracking logic has been refined.
- 0.15.7 Fixed bug about renaming file.
- 0.15.8 Fixed bug about deleting empty directory, weird behaviour on boot-sequence on mobile devices.
- 0.15.9 Improved chunk retrieving, now chunks are retrieved in batch on continuous requests.
- 0.15.10 Fixed:
- The boot sequence has been corrected and now boots smoothly.
- Auto applying of batch save will be processed earlier than before.
... To continue on to `updates_old.md`.