Compare commits

...

4 Commits

Author SHA1 Message Date
vorotamoroz
bb8c7eb043 bump 2023-02-03 18:36:33 +09:00
vorotamoroz
e61bebd3ee - Fixed: Skip patterns now handle capital letters.
- Improved
  - New configuration to avoid exceeding throttle capacity.
  - The conflicted `data.json` is no longer merged automatically.
2023-02-03 18:30:40 +09:00
vorotamoroz
99594fe517 Merge pull request #169 from karasevm/feat-timeout
Allow switching CouchDB long polling from heartbeat to timeout
2023-02-03 17:41:54 +09:00
Maksim Karasev
972d208af4 Expose useTimeouts option in the settings tab 2023-02-02 02:24:32 +03:00
10 changed files with 424 additions and 32 deletions

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.17.20",
"version": "0.17.21",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.17.20",
"version": "0.17.21",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.17.20",
"version": "0.17.21",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.17.20",
"version": "0.17.21",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",

54
src/JsonResolveModal.ts Normal file
View File

@@ -0,0 +1,54 @@
import { App, Modal } from "obsidian";
import { LoadedEntry } from "./lib/src/types";
import JsonResolvePane from "./JsonResolvePane.svelte";
export class JsonResolveModal extends Modal {
// result: Array<[number, string]>;
filename: string;
callback: (keepRev: string, mergedStr?: string) => Promise<void>;
docs: LoadedEntry[];
component: JsonResolvePane;
constructor(app: App, filename: string, docs: LoadedEntry[], callback: (keepRev: string, mergedStr?: string) => Promise<void>) {
super(app);
this.callback = callback;
this.filename = filename;
this.docs = docs;
}
async UICallback(keepRev: string, mergedStr?: string) {
this.close();
await this.callback(keepRev, mergedStr);
this.callback = null;
}
onOpen() {
const { contentEl } = this;
contentEl.empty();
if (this.component == null) {
this.component = new JsonResolvePane({
target: contentEl,
props: {
docs: this.docs,
callback: (keepRev, mergedStr) => this.UICallback(keepRev, mergedStr),
},
});
}
return;
}
onClose() {
const { contentEl } = this;
contentEl.empty();
// contentEl.empty();
if (this.callback != null) {
this.callback(null);
}
if (this.component != null) {
this.component.$destroy();
this.component = null;
}
}
}

162
src/JsonResolvePane.svelte Normal file
View File

@@ -0,0 +1,162 @@
<script lang="ts">
import { Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
import type { LoadedEntry } from "./lib/src/types";
import { base64ToString } from "./lib/src/strbin";
import { getDocData } from "./lib/src/utils";
import { mergeObject } from "./utils";
export let docs: LoadedEntry[] = [];
export let callback: (keepRev: string, mergedStr?: string) => Promise<void> = async (_, __) => {
Promise.resolve();
};
let docA: LoadedEntry = undefined;
let docB: LoadedEntry = undefined;
let docAContent = "";
let docBContent = "";
let objA: any = {};
let objB: any = {};
let objAB: any = {};
let objBA: any = {};
let diffs: Diff[];
const modes = [
["", "Not now"],
["A", "A"],
["B", "B"],
["AB", "A + B"],
["BA", "B + A"],
] as ["" | "A" | "B" | "AB" | "BA", string][];
let mode: "" | "A" | "B" | "AB" | "BA" = "";
function docToString(doc: LoadedEntry) {
return doc.datatype == "plain" ? getDocData(doc.data) : base64ToString(doc.data);
}
function revStringToRevNumber(rev: string) {
return rev.split("-")[0];
}
function getDiff(left: string, right: string) {
const dmp = new diff_match_patch();
const mapLeft = dmp.diff_linesToChars_(left, right);
const diffLeftSrc = dmp.diff_main(mapLeft.chars1, mapLeft.chars2, false);
dmp.diff_charsToLines_(diffLeftSrc, mapLeft.lineArray);
return diffLeftSrc;
}
function getJsonDiff(a: object, b: object) {
return getDiff(JSON.stringify(a, null, 2), JSON.stringify(b, null, 2));
}
function apply() {
if (mode == "A") return callback(docA._rev, null);
if (mode == "B") return callback(docB._rev, null);
if (mode == "BA") return callback(null, JSON.stringify(objBA, null, 2));
if (mode == "AB") return callback(null, JSON.stringify(objAB, null, 2));
callback(null, null);
}
$: {
if (docs && docs.length >= 1) {
if (docs[0].mtime < docs[1].mtime) {
docA = docs[0];
docB = docs[1];
} else {
docA = docs[1];
docB = docs[0];
}
docAContent = docToString(docA);
docBContent = docToString(docB);
try {
objA = false;
objB = false;
objA = JSON.parse(docAContent);
objB = JSON.parse(docBContent);
objAB = mergeObject(objA, objB);
objBA = mergeObject(objB, objA);
if (JSON.stringify(objAB) == JSON.stringify(objBA)) {
objBA = false;
}
} catch (ex) {
objBA = false;
objAB = false;
}
}
}
$: mergedObjs = {
"": false,
A: objA,
B: objB,
AB: objAB,
BA: objBA,
};
$: selectedObj = mode in mergedObjs ? mergedObjs[mode] : {};
$: {
diffs = getJsonDiff(objA, selectedObj);
console.dir(selectedObj);
}
</script>
<h1>File Conflicted</h1>
{#if !docA || !docB}
<div class="message">Just for a minute, please!</div>
<div class="buttons">
<button on:click={apply}>Dismiss</button>
</div>
{:else}
<div class="options">
{#each modes as m}
{#if m[0] == "" || mergedObjs[m[0]] != false}
<label class={`sls-setting-label ${m[0] == mode ? "selected" : ""}`}
><input type="radio" name="disp" bind:group={mode} value={m[0]} class="sls-setting-tab" />
<div class="sls-setting-menu-btn">{m[1]}</div></label
>
{/if}
{/each}
</div>
{#if selectedObj != false}
<div class="op-scrollable json-source">
{#each diffs as diff}
<span class={diff[0] == DIFF_DELETE ? "deleted" : diff[0] == DIFF_INSERT ? "added" : "normal"}>{diff[1]}</span>
{/each}
</div>
{:else}
NO PREVIEW
{/if}
<div>
A Rev:{revStringToRevNumber(docA._rev)} ,{new Date(docA.mtime).toLocaleString()}
{docAContent.length} letters
</div>
<div>
B Rev:{revStringToRevNumber(docB._rev)} ,{new Date(docB.mtime).toLocaleString()}
{docBContent.length} letters
</div>
<div class="buttons">
<button on:click={apply}>Apply</button>
</div>
{/if}
<style>
.deleted {
text-decoration: line-through;
}
* {
box-sizing: border-box;
}
.scroller {
display: flex;
flex-direction: column;
overflow-y: scroll;
max-height: 60vh;
user-select: text;
}
.json-source {
white-space: pre;
height: auto;
overflow: auto;
min-height: var(--font-ui-medium);
flex-grow: 1;
}
</style>

View File

@@ -1279,6 +1279,20 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text.inputEl.setAttribute("type", "number");
});
new Setting(containerSyncSettingEl)
.setName("Use timeouts instead of heartbeats")
.setDesc("If this option is enabled, PouchDB will hold the connection open for 60 seconds, and if no change arrives in that time, close and reopen the socket, instead of holding it open indefinitely. Useful when a proxy limits request duration but can increase resource usage.")
.addToggle((toggle) => {
toggle
.setValue(this.plugin.settings.useTimeouts)
.onChange(async (value) => {
this.plugin.settings.useTimeouts = value;
await this.plugin.saveSettings();
})
return toggle;
}
);
addScreenElement("30", containerSyncSettingEl);
const containerMiscellaneousEl = containerEl.createDiv();
containerMiscellaneousEl.createEl("h3", { text: "Miscellaneous" });

Submodule src/lib updated: 9e993fd984...a765f8eac5

View File

@@ -2,14 +2,14 @@ import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbst
import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2 } from "./lib/src/types";
import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo, queueItem, FileInfo } from "./types";
import { getDocData, isDocContentSame } from "./lib/src/utils";
import { delay, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { LocalPouchDB } from "./LocalPouchDB";
import { LogDisplayModal } from "./LogDisplayModal";
import { ConflictResolveModal } from "./ConflictResolveModal";
import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
import { DocumentHistoryModal } from "./DocumentHistoryModal";
import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, flattenObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
import { decrypt, encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
const isDebug = false;
@@ -23,6 +23,7 @@ import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayB
import { isPlainText, isValidPath, shouldBeIgnored } from "./lib/src/path";
import { runWithLock } from "./lib/src/lock";
import { Semaphore } from "./lib/src/semaphore";
import { JsonResolveModal } from "./JsonResolveModal";
setNoticeClass(Notice);
@@ -1148,9 +1149,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (!this.settings.syncInternalFiles) return;
if (!this.settings.watchInternalFileChanges) return;
if (!path.startsWith(this.app.vault.configDir)) return;
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
if (ignorePatterns.some(e => path.match(e))) return;
this.appendWatchEvent(
[{
@@ -2325,6 +2326,26 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const diffLeft = generatePatchObj(baseObj, leftObj);
const diffRight = generatePatchObj(baseObj, rightObj);
// If each value of the same key has been modified, the automatic merge should be prevented.
//TODO Does it have to be a configurable item?
const diffSetLeft = new Map(flattenObject(diffLeft));
const diffSetRight = new Map(flattenObject(diffRight));
for (const [key, value] of diffSetLeft) {
if (diffSetRight.has(key)) {
if (diffSetRight.get(key) == value) {
// No matter, if changed to the same value.
diffSetRight.delete(key);
}
}
}
for (const [key, value] of diffSetRight) {
if (diffSetLeft.has(key) && diffSetLeft.get(key) != value) {
// Some changes are conflicted
return false;
}
}
const patches = [
{ mtime: leftLeaf.mtime, patch: diffLeft },
{ mtime: rightLeaf.mtime, patch: diffRight }
@@ -2505,7 +2526,60 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}).open();
});
}
showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
return new Promise((res) => {
Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE);
const docs = [docA, docB];
const modal = new JsonResolveModal(this.app, id2path(docA._id), [docA, docB], async (keep, result) => {
// modal.close();
try {
const filename = id2filenameInternalMetadata(docA._id);
let needFlush = false;
if (!result && !keep) {
Logger(`Skipped merging: ${filename}`);
}
//Delete old revisions
if (result || keep) {
for (const doc of docs) {
if (doc._rev != keep) {
if (await this.localDatabase.deleteDBEntry(doc._id, { rev: doc._rev })) {
Logger(`Conflicted revision has been deleted: ${filename}`);
needFlush = true;
}
}
}
}
if (!keep && result) {
const isExists = await this.app.vault.adapter.exists(filename);
if (!isExists) {
await this.ensureDirectoryEx(filename);
}
await this.app.vault.adapter.write(filename, result);
const stat = await this.app.vault.adapter.stat(filename);
await this.storeInternalFileToDatabase({ path: filename, ...stat }, true);
try {
//@ts-ignore internalAPI
await app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
Logger(ex, LOG_LEVEL.VERBOSE);
}
Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`);
}
if (needFlush) {
await this.extractInternalFileFromDatabase(filename, false);
Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`);
}
res(true);
} catch (ex) {
Logger("Could not merge conflicted json");
Logger(ex, LOG_LEVEL.VERBOSE)
res(false);
}
})
modal.open();
});
}
conflictedCheckFiles: string[] = [];
// queueing the conflicted file check
@@ -2976,9 +3050,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async scanInternalFiles(): Promise<InternalFileInfo[]> {
const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
const root = this.app.vault.getRoot();
const findRoot = root.path;
const filenames = (await this.getFiles(findRoot, [], null, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
@@ -3116,8 +3190,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return await runWithLock("file-" + id, false, async () => {
try {
const fileOnDB = await this.localDatabase.getDBEntry(id, null, false, false) as false | LoadedEntry;
// Check conflicted status
//TODO option
const fileOnDB = await this.localDatabase.getDBEntry(id, { conflicts: true }, false, false) as false | LoadedEntry;
if (fileOnDB === false) throw new Error(`File not found on database.:${id}`);
// Prevent overrite for Prevent overwriting while some conflicted revision exists.
if (fileOnDB?._conflicts?.length) {
Logger(`Hidden file ${id} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL.INFO);
return;
}
const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false;
if (deleted) {
if (!isExists) {
@@ -3125,6 +3206,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} else {
Logger(`STORAGE <x- DB:${filename}: deleted (hidden).`);
await this.app.vault.adapter.remove(filename);
try {
//@ts-ignore internalAPI
await app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
Logger(ex, LOG_LEVEL.VERBOSE);
}
}
return true;
}
if (!isExists) {
await this.ensureDirectoryEx(filename);
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await app.vault.adapter.reconcileInternalFile(filename);
@@ -3132,19 +3226,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
Logger(ex, LOG_LEVEL.VERBOSE);
}
}
return true;
}
if (!isExists) {
await this.ensureDirectoryEx(filename);
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
Logger(ex, LOG_LEVEL.VERBOSE);
}
Logger(`STORAGE <-- DB:${filename}: written (hidden,new${force ? ", force" : ""})`);
return true;
} else {
@@ -3175,9 +3256,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
filterTargetFiles(files: InternalFileInfo[], targetFiles: string[] | false = false) {
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
return files.filter(file => !ignorePatterns.some(e => file.path.match(e))).filter(file => !targetFiles || (targetFiles && targetFiles.indexOf(file.path) !== -1))
}
@@ -3200,7 +3281,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async resolveConflictOnInternalFile(id: string): Promise<boolean> {
try {// Retrieve data
try {
// Retrieve data
const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
// If there is no conflict, return with false.
if (!("_conflicts" in doc)) return false;
@@ -3233,6 +3315,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} else {
Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
}
const docAMerge = await this.localDatabase.getDBEntry(id, { rev: revA });
const docBMerge = await this.localDatabase.getDBEntry(id, { rev: revB });
if (docAMerge != false && docBMerge != false) {
if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
await delay(200);
// Again for other conflicted revisions.
return this.resolveConflictOnInternalFile(id);
}
return false;
}
}
const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
// determine which revision should been deleted.
@@ -3258,9 +3351,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.resolveConflictOnInternalFiles();
const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
Logger("Scanning hidden files.", logLevel, "sync_internal");
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e));
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
if (!files) files = await this.scanInternalFiles();
const filesOnDB = ((await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
@@ -3299,7 +3392,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
}
const p = [] as Promise<void>[];
const semaphore = Semaphore(15);
const semaphore = Semaphore(10);
// Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
caches = await this.localDatabase.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches-internal") || {};

View File

@@ -198,3 +198,65 @@ export function applyPatch(from: Record<string | number | symbol, any>, patch: R
}
return ret;
}
export function mergeObject(
objA: Record<string | number | symbol, any>,
objB: Record<string | number | symbol, any>
) {
const newEntries = Object.entries(objB);
const ret: any = { ...objA };
if (
typeof objA !== typeof objB ||
Array.isArray(objA) !== Array.isArray(objB)
) {
return objB;
}
for (const [key, v] of newEntries) {
if (key in ret) {
const value = ret[key];
if (
typeof v !== typeof value ||
Array.isArray(v) !== Array.isArray(value)
) {
//if type is not match, replace completely.
ret[key] = v;
} else {
if (
typeof v == "object" &&
typeof value == "object" &&
!Array.isArray(v) &&
!Array.isArray(value)
) {
ret[key] = mergeObject(v, value);
} else if (
typeof v == "object" &&
typeof value == "object" &&
Array.isArray(v) &&
Array.isArray(value)
) {
ret[key] = [...new Set([...v, ...value])];
} else {
ret[key] = v;
}
}
} else {
ret[key] = v;
}
}
return Object.entries(ret)
.sort()
.reduce((p, [key, value]) => ({ ...p, [key]: value }), {});
}
export function flattenObject(obj: Record<string | number | symbol, any>, path: string[] = []): [string, any][] {
if (typeof (obj) != "object") return [[path.join("."), obj]];
if (Array.isArray(obj)) return [[path.join("."), JSON.stringify(obj)]];
const e = Object.entries(obj);
const ret = []
for (const [key, value] of e) {
const p = flattenObject(value, [...path, key]);
ret.push(...p);
}
return ret;
}

View File

@@ -89,5 +89,12 @@
- Fixed: Error reporting has been ensured.
- 0.17.20
- Improved: Changes of hidden files will be notified to Obsidian.
- 0.17.21
- Fixed: Skip patterns now handle capital letters.
- Improved
- New configuration to avoid exceeding throttle capacity.
- We have been grateful to @karasevm!
- The conflicted `data.json` is no longer merged automatically.
- This behaviour is not configurable, unlike the `Use newer file if conflicted` of normal files.
... To continue on to `updates_old.md`.