diff --git a/README.md b/README.md
index fe16754..77bb4fb 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,6 @@ Runs in Mac, Android, Windows, and iOS.
-
**It's beta. Please make sure back your vault up!**
Limitations: File deletion handling is not completed.
@@ -18,15 +17,30 @@ Limitations: File deletion handling is not completed.
- Self-Hosted data synchronization with conflict detection and resolving in Obsidian.
- Off line sync is also available.
-## How to use the beta build
+## How to use
-1. download this repo and expand `[your-vault]/.obsidian/plugins/` (PC, Mac and Android will work)
-1. enable obsidian livesync in the settings dialog.
-1. If you use your self-hosted CouchDB, set your server's info.
-1. or Use [IBM Cloudant](https://www.ibm.com/cloud/cloudant), take an account and enable **Cloudant** in [Catalog](https://cloud.ibm.com/catalog#services)
- Note please choose "IAM and legacy credentials" for the Authentication method
- Setup details are in Couldant Setup Section.
-1. Setup LiveSync or SyncOnSave or SyncOnStart as you like.
+1. Install from Obsidian, or clone this repo and run `npm run build` ,copy `main.js`, `styles.css` and `manifest.json` into `[your-vault]/.obsidian/plugins/` (PC, Mac and Android will work)
+2. Enable obsidian livesync in the settings dialog.
+3. If you use your self-hosted CouchDB, set your server's info.
+4. or Use [IBM Cloudant](https://www.ibm.com/cloud/cloudant), take an account and enable **Cloudant** in [Catalog](https://cloud.ibm.com/catalog#services)
+ Note please choose "IAM and legacy credentials" for the Authentication method
+ Setup details are in Couldant Setup Section.
+5. Setup LiveSync or SyncOnSave or SyncOnStart as you like.
+
+## When your database looks corrupted
+
+obsidian-livesync changes data treatment of markdown files since 0.1.0
+When you are troubled with synchronization, **Please reset local and remote databases**.
+*Note: Without synchronization, your files won't be deleted.*
+
+1. Disable any synchronizations on all devices.
+2. From the most reliable device(_The device_), back your vault up.
+3. Click "Reset local database" on all devices.
+4. From _The device_ click "Reset remote database".
+5. From _The device_ click "Init Database again".
+6. Enable any sync or Hit Replication button.
+
+And wait for a minute. your data will be uploaded and synchronized with all devices again.
## Cloudant Setup
@@ -107,4 +121,5 @@ example values.
| CouchDB Password | (\*4) | c2c11651d75497fa3d3c486e4c8bdf27 |
# License
+
The source code is licensed MIT.
diff --git a/main.ts b/main.ts
index 281f394..5e47722 100644
--- a/main.ts
+++ b/main.ts
@@ -1,4 +1,4 @@
-import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder, ItemView } from "obsidian";
+import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder } from "obsidian";
import { PouchDB } from "./pouchdb-browser-webpack/dist/pouchdb-browser";
import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
import xxhash from "xxhash-wasm";
@@ -8,7 +8,17 @@ import xxhash from "xxhash-wasm";
// const MAX_DOC_SIZE = 921600;
const MAX_DOC_SIZE = 1000; // for .md file, but if delimiters exists. use that before.
const MAX_DOC_SIZE_BIN = 102400; // 100kb
-const VER = 10
+const VER = 10;
+
+const RECENT_MOFIDIED_DOCS_QTY = 30;
+
+const LOG_LEVEL = {
+ VERBOSE: 1,
+ INFO: 10,
+ NOTICE: 100,
+ URGENT: 1000,
+} as const;
+type LOG_LEVEL = typeof LOG_LEVEL[keyof typeof LOG_LEVEL];
interface ObsidianLiveSyncSettings {
couchDB_URI: string;
@@ -22,7 +32,8 @@ interface ObsidianLiveSyncSettings {
gcDelay: number;
versionUpFlash: string;
minimumChunkSize: number;
- longLineThreshold: number
+ longLineThreshold: number;
+ showVerboseLog: boolean;
}
const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -38,8 +49,8 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
versionUpFlash: "",
minimumChunkSize: 20,
longLineThreshold: 250,
+ showVerboseLog: false,
};
-
interface Entry {
_id: string;
data: string;
@@ -48,6 +59,7 @@ interface Entry {
mtime: number;
size: number;
_deleted?: boolean;
+ _conflicts?: string[];
type?: "notes";
}
interface NewEntry {
@@ -58,6 +70,7 @@ interface NewEntry {
mtime: number;
size: number;
_deleted?: boolean;
+ _conflicts?: string[];
NewNote: true;
type: "newnote";
}
@@ -70,11 +83,12 @@ interface PlainEntry {
size: number;
_deleted?: boolean;
NewNote: true;
+ _conflicts?: string[];
type: "plain";
}
type LoadedEntry = Entry & {
children: string[];
- datatype: "plain" | "newnote"
+ datatype: "plain" | "newnote";
};
interface EntryLeaf {
@@ -106,6 +120,8 @@ type Credential = {
password: string;
};
+type EntryDocResponse = EntryDoc & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta;
+
//-->Functions.
function arrayBufferToBase64(buffer: ArrayBuffer) {
var binary = "";
@@ -183,12 +199,19 @@ class LocalPouchDB {
plugin: ObsidianLiveSyncPlugin;
auth: Credential;
dbname: string;
- addLog: (message: any, isNotify?: boolean) => Promise;
+ addLog: (message: any, levlel?: LOG_LEVEL) => Promise;
localDatabase: PouchDB.Database;
recentModifiedDocs: string[] = [];
h32: (input: string, seed?: number) => string;
h64: (input: string, seedHigh?: number, seedLow?: number) => string;
+ hashCache: {
+ [key: string]: string;
+ } = {};
+ hashCacheRev: {
+ [key: string]: string;
+ } = {};
+
constructor(app: App, plugin: ObsidianLiveSyncPlugin, dbname: string) {
this.plugin = plugin;
this.app = app;
@@ -210,11 +233,18 @@ class LocalPouchDB {
}
return "disabled";
}
- updateRecentModifiedDocs(id: string, rev: string) {
+ disposeHashCache() {
+ this.hashCache = {};
+ this.hashCacheRev = {};
+ }
+
+ updateRecentModifiedDocs(id: string, rev: string, deleted: boolean) {
let idrev = id + rev;
- this.recentModifiedDocs.push(idrev);
- if (this.recentModifiedDocs.length > 10) {
- this.recentModifiedDocs = this.recentModifiedDocs.slice(-30);
+ if (deleted) {
+ this.recentModifiedDocs = this.recentModifiedDocs.filter((e) => e != idrev);
+ } else {
+ this.recentModifiedDocs.push(idrev);
+ this.recentModifiedDocs = this.recentModifiedDocs.slice(0 - RECENT_MOFIDIED_DOCS_QTY);
}
}
isSelfModified(id: string, rev: string): boolean {
@@ -229,17 +259,17 @@ class LocalPouchDB {
revs_limit: 100,
deterministic_revs: true,
});
- await this.prepareHashArg();
+ await this.prepareHashFunctions();
}
- async prepareHashArg() {
+ async prepareHashFunctions() {
if (this.h32 != null) return;
const { h32, h64 } = await xxhash();
this.h32 = h32;
this.h64 = h64;
}
- async getDatabaseDoc(id: string, opt?: any): Promise {
+ async getDBEntry(id: string, opt?: PouchDB.Core.GetOptions): Promise {
try {
- let obj: EntryDoc & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = null;
+ let obj: EntryDocResponse = null;
if (opt) {
obj = await this.localDatabase.get(id, opt);
} else {
@@ -254,7 +284,7 @@ class LocalPouchDB {
//Check it out and fix docs to regular case
if (!obj.type || (obj.type && obj.type == "notes")) {
let note = obj as Entry;
- let doc: LoadedEntry = {
+ let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
data: note.data,
_id: note._id,
ctime: note.ctime,
@@ -262,6 +292,7 @@ class LocalPouchDB {
size: note.size,
_deleted: obj._deleted,
_rev: obj._rev,
+ _conflicts: obj._conflicts,
children: [],
datatype: "newnote",
};
@@ -273,23 +304,27 @@ class LocalPouchDB {
try {
let childrens = [];
for (var v of obj.children) {
- try {
- let elem = await this.localDatabase.get(v);
- if (elem.type && elem.type == "leaf") {
- childrens.push(elem.data);
- } else {
- throw new Error("linked document is not leaf");
+ if (typeof this.hashCacheRev[v] !== "undefined") {
+ childrens.push(this.hashCacheRev[v]);
+ } else {
+ try {
+ let elem = await this.localDatabase.get(v);
+ if (elem.type && elem.type == "leaf") {
+ childrens.push(elem.data);
+ } else {
+ throw new Error("linked document is not leaf");
+ }
+ } catch (ex) {
+ if (ex.status && ex.status == 404) {
+ this.addLog(`Missing document content!, could not read ${v} of ${obj._id}(${obj._rev}) from database.`, LOG_LEVEL.NOTICE);
+ return false;
+ }
+ throw ex;
}
- } catch (ex) {
- if (ex.status && ex.status == 404) {
- this.addLog(`Missing document content!, could not read ${v} of ${obj._id} from database.`, true);
- return false;
- }
- throw ex;
}
}
let data = childrens.join("");
- let doc: LoadedEntry = {
+ let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
data: data,
_id: obj._id,
ctime: obj.ctime,
@@ -298,16 +333,17 @@ class LocalPouchDB {
_deleted: obj._deleted,
_rev: obj._rev,
children: obj.children,
- datatype: obj.type
+ datatype: obj.type,
+ _conflicts: obj._conflicts,
};
return doc;
} catch (ex) {
if (ex.status && ex.status == 404) {
- this.addLog(`Missing document content!, could not read ${obj._id} from database.`, true);
+ this.addLog(`Missing document content!, could not read ${obj._id} from database.`, LOG_LEVEL.NOTICE);
return false;
}
- this.addLog(`Something went wrong on reading ${obj._id} from database.`, true);
+ this.addLog(`Something went wrong on reading ${obj._id} from database.`, LOG_LEVEL.NOTICE);
this.addLog(ex);
}
}
@@ -319,9 +355,9 @@ class LocalPouchDB {
}
return false;
}
- async deleteDBEntry(id: string, opt?: any): Promise {
+ async deleteDBEntry(id: string, opt?: PouchDB.Core.GetOptions): Promise {
try {
- let obj: EntryDoc & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = null;
+ let obj: EntryDocResponse = null;
if (opt) {
obj = await this.localDatabase.get(id, opt);
} else {
@@ -336,15 +372,15 @@ class LocalPouchDB {
if (!obj.type || (obj.type && obj.type == "notes")) {
obj._deleted = true;
let r = await this.localDatabase.put(obj);
- this.updateRecentModifiedDocs(r.id, r.rev);
+ this.updateRecentModifiedDocs(r.id, r.rev, true);
return true;
// simple note
}
if (obj.type == "newnote" || obj.type == "plain") {
obj._deleted = true;
let r = await this.localDatabase.put(obj);
- this.addLog(`entry removed:${obj._id}`);
- this.updateRecentModifiedDocs(r.id, r.rev);
+ this.addLog(`entry removed:${obj._id}-${r.rev}`);
+ this.updateRecentModifiedDocs(r.id, r.rev, true);
return true;
}
} catch (ex) {
@@ -363,6 +399,7 @@ class LocalPouchDB {
let skiped = 0;
let pieceSize = MAX_DOC_SIZE_BIN;
let plainSplit = false;
+ let cacheUsed = 0;
if (note._id.endsWith(".md")) {
pieceSize = MAX_DOC_SIZE;
plainSplit = true;
@@ -370,12 +407,12 @@ class LocalPouchDB {
do {
// To keep low bandwith and database size,
// Dedup pieces on database.
- // from 0.1.10, for best performance. we use markdown delimiters
+ // from 0.1.10, for best performance. we use markdown delimiters
// 1. \n[^\n]{longLineThreshold}[^\n]*\n -> long sentence shuld break.
// 2. \n\n shold break
// 3. \r\n\r\n should break
// 4. \n# should break.
- let cPieceSize = pieceSize
+ let cPieceSize = pieceSize;
let minimumChunkSize = this.plugin.settings.minimumChunkSize;
if (minimumChunkSize < 10) minimumChunkSize = 10;
let longLineThreshold = this.plugin.settings.longLineThreshold;
@@ -406,77 +443,82 @@ class LocalPouchDB {
if (n4 > 0 && cPieceSize < n4) cPieceSize = n4 + 0;
cPieceSize++;
}
- } while (cPieceSize < minimumChunkSize)
- // console.log("and we use:" + cPieceSize)
+ } while (cPieceSize < minimumChunkSize);
}
let piece = leftData.substring(0, cPieceSize);
- // if (plainSplit) {
- // this.addLog(`piece_len:${cPieceSize}`);
- // this.addLog("piece:" + piece);
- // }
leftData = leftData.substring(cPieceSize);
processed++;
+ let leafid = "";
// Get has of piece.
- let hashedPiece = this.h32(piece);
- let leafid = "h:" + hashedPiece;
+ let hashedPiece: string = "";
let hashQ: number = 0; // if hash collided, **IF**, count it up.
let tryNextHash = false;
let needMake = true;
-
- do {
- let nleafid = leafid;
- try {
- nleafid = `${leafid}${hashQ}`;
- // console.log(nleafid);
- let pieceData = await this.localDatabase.get(nleafid);
- if (pieceData.type == "leaf" && pieceData.data == piece) {
- // this.addLog("hash:data exists.");
- leafid = nleafid;
- needMake = false;
- tryNextHash = false;
- } else if (pieceData.type == "leaf") {
- this.addLog("hash:collision!!");
- hashQ++;
- tryNextHash = true;
- } else {
- // this.addLog("hash:no collision, it's not leaf. what's going on..");
- leafid = nleafid;
- tryNextHash = false;
- }
- } catch (ex) {
- if (ex.status && ex.status == 404) {
- //not found, we can use it.
- // this.addLog(`hash:not found.`);
- leafid = nleafid;
- needMake = true;
- } else {
- needMake = false;
- throw ex;
- }
- }
- } while (tryNextHash);
- if (needMake) {
- //have to make
- let d: EntryLeaf = {
- _id: leafid,
- data: piece,
- type: "leaf",
- };
- let result = await this.localDatabase.put(d);
- this.updateRecentModifiedDocs(result.id, result.rev);
- if (result.ok) {
- this.addLog(`ok:saven`);
- made++;
- } else {
- this.addLog("save faild");
- }
- } else {
+ if (typeof this.hashCache[piece] !== "undefined") {
+ hashedPiece = "";
+ leafid = this.hashCache[piece];
+ needMake = false;
skiped++;
+ cacheUsed++;
+ } else {
+ hashedPiece = this.h32(piece);
+ leafid = "h:" + hashedPiece;
+ do {
+ let nleafid = leafid;
+ try {
+ nleafid = `${leafid}${hashQ}`;
+ let pieceData = await this.localDatabase.get(nleafid);
+ if (pieceData.type == "leaf" && pieceData.data == piece) {
+ leafid = nleafid;
+ needMake = false;
+ tryNextHash = false;
+ this.hashCache[piece] = leafid;
+ this.hashCacheRev[leafid] = piece;
+ } else if (pieceData.type == "leaf") {
+ this.addLog("hash:collision!!");
+ hashQ++;
+ tryNextHash = true;
+ } else {
+ leafid = nleafid;
+ tryNextHash = false;
+ }
+ } catch (ex) {
+ if (ex.status && ex.status == 404) {
+ //not found, we can use it.
+ leafid = nleafid;
+ needMake = true;
+ } else {
+ needMake = false;
+ throw ex;
+ }
+ }
+ } while (tryNextHash);
+ if (needMake) {
+ //have to make
+ let d: EntryLeaf = {
+ _id: leafid,
+ data: piece,
+ type: "leaf",
+ };
+ let result = await this.localDatabase.put(d);
+ this.updateRecentModifiedDocs(result.id, result.rev, d._deleted);
+ if (result.ok) {
+ this.addLog(`save ok:id:${result.id} rev:${result.rev}`, LOG_LEVEL.VERBOSE);
+ this.hashCache[piece] = leafid;
+ this.hashCacheRev[leafid] = piece;
+ made++;
+ } else {
+ this.addLog("save faild");
+ }
+ } else {
+ skiped++;
+ }
}
+
savenNotes.push(leafid);
} while (leftData != "");
- this.addLog(`note content saven, pieces:${processed} new:${made}, skip:${skiped}`);
+ this.addLog(`note content saven, pieces:${processed} new:${made}, skip:${skiped}, cache:${cacheUsed}`);
let newDoc: PlainEntry | NewEntry = {
NewNote: true,
children: savenNotes,
@@ -487,7 +529,6 @@ class LocalPouchDB {
type: plainSplit ? "plain" : "newnote",
};
- let deldocs: string[] = [];
// Here for upsert logic,
try {
let old = await this.localDatabase.get(newDoc._id);
@@ -503,8 +544,8 @@ class LocalPouchDB {
}
}
let r = await this.localDatabase.put(newDoc);
- this.updateRecentModifiedDocs(r.id, r.rev);
- this.addLog(`note saven:${newDoc._id}`);
+ this.updateRecentModifiedDocs(r.id, r.rev, newDoc._deleted);
+ this.addLog(`note saven:${newDoc._id}:${r.rev}`);
}
syncHandler: PouchDB.Replication.Sync<{}> = null;
@@ -525,15 +566,19 @@ class LocalPouchDB {
}
let dbret = await connectRemoteCouchDB(uri, auth);
if (dbret === false) {
- this.addLog(`could not connect to ${uri}`, true);
+ this.addLog(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
return;
}
- let syncOption = keepAlive ? { live: true, retry: true } : {};
+ let syncOptionBase: PouchDB.Replication.SyncOptions = {
+ batch_size: 250,
+ batches_limit: 40,
+ };
+ let syncOption: PouchDB.Replication.SyncOptions = keepAlive ? { live: true, retry: true, heartbeat: 30000, ...syncOptionBase } : { ...syncOptionBase };
+
let db = dbret.db;
//replicate once
- let replicate = this.localDatabase.replicate.from(db);
- // console.log("replication start.")
+ let replicate = this.localDatabase.replicate.from(db, syncOptionBase);
replicate
.on("change", async (e) => {
try {
@@ -547,7 +592,7 @@ class LocalPouchDB {
.on("complete", async (info) => {
replicate.removeAllListeners();
replicate.cancel();
- // this.syncHandler = null;
+ this.syncHandler = null;
if (this.syncHandler != null) {
this.syncHandler.removeAllListeners();
}
@@ -566,26 +611,25 @@ class LocalPouchDB {
}
})
.on("complete", (e) => {
- this.addLog("Replication completed", showResult);
+ this.addLog("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
this.syncHandler = null;
})
.on("denied", (e) => {
- this.addLog("Replication denied", true);
+ this.addLog("Replication denied", LOG_LEVEL.NOTICE);
// this.addLog(e);
})
.on("error", (e) => {
- this.addLog("Replication error", true);
+ this.addLog("Replication error", LOG_LEVEL.NOTICE);
// this.addLog(e);
})
.on("paused", (e) => {
- this.addLog("replication paused");
- // console.dir(this.syncHandler);
+ this.addLog("replication paused", LOG_LEVEL.VERBOSE);
// this.addLog(e);
});
- // console.dir();
})
- .on("error", () => {
- this.addLog("Pulling Replication error", true);
+ .on("error", (e) => {
+ this.addLog("Pulling Replication error", LOG_LEVEL.NOTICE);
+ this.addLog(e);
});
}
@@ -604,7 +648,7 @@ class LocalPouchDB {
await this.localDatabase.destroy();
this.localDatabase = null;
await this.initializeDatabase();
- this.addLog("Local Database Reset", true);
+ this.addLog("Local Database Reset", LOG_LEVEL.NOTICE);
}
async tryResetRemoteDatabase(setting: ObsidianLiveSyncSettings) {
await this.closeReplication();
@@ -618,10 +662,10 @@ class LocalPouchDB {
if (con === false) return;
try {
await con.db.destroy();
- this.addLog("Remote Database Destroyed", true);
+ this.addLog("Remote Database Destroyed", LOG_LEVEL.NOTICE);
await this.tryCreateRemoteDatabase(setting);
} catch (ex) {
- this.addLog("something happend on Remote Database Destory", true);
+ this.addLog("something happend on Remote Database Destory", LOG_LEVEL.NOTICE);
}
}
async tryCreateRemoteDatabase(setting: ObsidianLiveSyncSettings) {
@@ -633,7 +677,7 @@ class LocalPouchDB {
};
let con2 = await connectRemoteCouchDB(uri, auth);
if (con2 === false) return;
- this.addLog("Remote Database Created or Connected", true);
+ this.addLog("Remote Database Created or Connected", LOG_LEVEL.NOTICE);
}
async garbageCollect() {
@@ -644,7 +688,7 @@ class LocalPouchDB {
let hashPieces: string[] = [];
let usedPieces: string[] = [];
do {
- let result = await this.localDatabase.allDocs({ include_docs: true, skip: c, limit: 100 });
+ let result = await this.localDatabase.allDocs({ include_docs: true, skip: c, limit: 100, conflicts: true });
readCount = result.rows.length;
if (readCount > 0) {
//there are some result
@@ -653,6 +697,14 @@ class LocalPouchDB {
if (doc.type == "newnote" || doc.type == "plain") {
// used pieces memo.
usedPieces = Array.from(new Set([...usedPieces, ...doc.children]));
+ if (doc._conflicts) {
+ for (let cid of doc._conflicts) {
+ let p = await this.localDatabase.get(doc._id, { rev: cid });
+ if (p.type == "newnote" || p.type == "plain") {
+ usedPieces = Array.from(new Set([...usedPieces, ...p.children]));
+ }
+ }
+ }
}
if (doc.type == "leaf") {
// all pieces.
@@ -743,6 +795,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.watchVaultDelete = debounce(this.watchVaultDelete.bind(this), delay, false);
this.watchVaultRename = debounce(this.watchVaultRename.bind(this), delay, false);
this.watchWorkspaceOpen = debounce(this.watchWorkspaceOpen.bind(this), delay, false);
+
this.registerWatchEvents();
this.parseReplicationResult = this.parseReplicationResult.bind(this);
@@ -767,6 +820,35 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
}, 60 * 1000)
);
+ this.addCommand({
+ id: "livesync-replicate",
+ name: "Replicate now",
+ callback: () => {
+ this.replicate();
+ },
+ });
+ this.addCommand({
+ id: "livesync-gc",
+ name: "garbage collect now",
+ callback: () => {
+ this.garbageCollect();
+ },
+ });
+ this.addCommand({
+ id: "livesync-toggle",
+ name: "Toggle LiveSync",
+ callback: () => {
+ if (this.settings.liveSync) {
+ this.settings.liveSync = false;
+ this.addLog("LiveSync Disabled.", LOG_LEVEL.NOTICE);
+ } else {
+ this.settings.liveSync = true;
+ this.addLog("LiveSync Enabled.", LOG_LEVEL.NOTICE);
+ }
+ this.realizeSettingSyncMode();
+ this.saveSettings();
+ },
+ });
this.watchWindowVisiblity = this.watchWindowVisiblity.bind(this);
window.addEventListener("visibilitychange", this.watchWindowVisiblity);
}
@@ -822,9 +904,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
watchWindowVisiblity() {
- this.addLog("visiblity changed");
let isHidden = document.hidden;
- // this.addLog(isHidden);
if (isHidden) {
this.localDatabase.closeReplication();
} else {
@@ -840,6 +920,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
watchWorkspaceOpen(file: TFile) {
if (file == null) return;
+ this.localDatabase.disposeHashCache();
this.showIfConflicted(file);
this.gcHook();
}
@@ -869,27 +950,26 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
//--> Basic document Functions
- async addLog(message: any, isNotify?: boolean) {
+ async addLog(message: any, level: LOG_LEVEL = LOG_LEVEL.INFO) {
// debugger;
- if (!isNotify && this.settings && this.settings.lessInformationInLog) {
+ if (level < LOG_LEVEL.INFO && this.settings && this.settings.lessInformationInLog) {
return;
}
- // console.log(this.settings);
+ if (this.settings && !this.settings.showVerboseLog && level == LOG_LEVEL.VERBOSE) {
+ return;
+ }
+ let valutName = this.app.vault.getName();
let timestamp = new Date().toLocaleString();
let messagecontent = typeof message == "string" ? message : JSON.stringify(message, null, 2);
let newmessage = timestamp + "->" + messagecontent;
this.logMessage = [].concat(this.logMessage).concat([newmessage]).slice(-100);
- // this.logMessage = [...this.logMessage, timestamp + ":" + newmessage].slice(-100);
- console.log(newmessage);
+ console.log(valutName + ":" + newmessage);
if (this.statusBar2 != null) {
this.statusBar2.setText(newmessage.substring(0, 60));
}
- // if (this.onLogChanged != null) {
- // this.onLogChanged();
- // }
- if (isNotify) {
+ if (level >= LOG_LEVEL.NOTICE) {
new Notice(messagecontent);
}
}
@@ -916,7 +996,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async doc2storage_create(docEntry: Entry, force?: boolean) {
- let doc = await this.localDatabase.getDatabaseDoc(docEntry._id, { _rev: docEntry._rev });
+ let doc = await this.localDatabase.getDBEntry(docEntry._id, { rev: docEntry._rev });
if (doc === false) return;
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
@@ -932,7 +1012,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.addLog("live : write to local (newfile:p) " + doc._id);
await this.app.vault.trigger("create", newfile);
} else {
- this.addLog("live : New data imcoming, but we cound't parse that.1" + doc.datatype, true);
+ this.addLog("live : New data imcoming, but we cound't parse that." + doc.datatype, LOG_LEVEL.NOTICE);
}
}
@@ -950,34 +1030,39 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (docEntry._deleted) {
//basically pass.
//but if there're no docs left, delete file.
- let lastDocs = await this.localDatabase.getDatabaseDoc(docEntry._id);
+ let lastDocs = await this.localDatabase.getDBEntry(docEntry._id);
if (lastDocs === false) {
await this.deleteVaultItem(file);
} else {
+ // it perhaps delete some revisions.
+ // may be we have to reload this
+ await this.pullFile(docEntry._id, null, true);
this.addLog(`delete skipped:${lastDocs._id}`);
}
return;
}
if (file.stat.mtime < docEntry.mtime || force) {
- let doc = await this.localDatabase.getDatabaseDoc(docEntry._id);
+ let doc = await this.localDatabase.getDBEntry(docEntry._id);
+ let msg = "livesync : newer local files so write to local:" + file.path;
+ if (force) msg = "livesync : force write to local:" + file.path;
if (doc === false) return;
- // debugger;
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
+ await this.ensureDirectory(doc._id);
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
- this.addLog("livesync : newer local files so write to local:" + file.path);
+ this.addLog(msg);
await this.app.vault.trigger("modify", file);
}
- } if (doc.datatype == "plain") {
+ }
+ if (doc.datatype == "plain") {
await this.ensureDirectory(doc._id);
await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
- this.addLog("livesync : newer local files so write to local:" + file.path);
+ this.addLog(msg);
await this.app.vault.trigger("modify", file);
-
} else {
- this.addLog("live : New data imcoming, but we cound't parse that.2:" + doc.datatype + "-", true);
+ this.addLog("live : New data imcoming, but we cound't parse that.:" + doc.datatype + "-", LOG_LEVEL.NOTICE);
}
} else if (file.stat.mtime > docEntry.mtime) {
// newer local file.
@@ -987,7 +1072,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
//eq.case
}
}
- async pouchdbChanged(change: Entry) {
+ async handleDBChanged(change: Entry) {
let allfiles = this.app.vault.getFiles();
let targetFiles = allfiles.filter((e) => e.path == change._id);
if (targetFiles.length == 0) {
@@ -1011,8 +1096,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (this.localDatabase.isSelfModified(change._id, change._rev)) {
return;
}
- this.addLog("replication change arrived");
- await this.pouchdbChanged(change);
+ this.addLog("replication change arrived", LOG_LEVEL.VERBOSE);
+ await this.handleDBChanged(change);
this.gcHook();
}
}
@@ -1034,7 +1119,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
}
- //<-- Sync
async initializeDatabase() {
await this.openDatabase();
@@ -1049,19 +1133,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const onlyInStorage = filesStorage.filter((e) => filesDatabase.indexOf(e.path) == -1);
const onlyInDatabase = filesDatabase.filter((e) => filesStorageName.indexOf(e) == -1);
- //simply realize it
+
const onlyInStorageNames = onlyInStorage.map((e) => e.path);
- //have to sync below..
const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
+ // just write to DB from storage.
for (let v of onlyInStorage) {
await this.updateIntoDB(v);
}
+ // simply realize it
for (let v of onlyInDatabase) {
await this.pullFile(v, filesStorage);
}
-
+ // have to sync below..
for (let v of syncFiles) {
await this.syncFileBetweenDBandStorage(v, filesStorage);
}
@@ -1070,9 +1155,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.addLog(`delete folder:${folder.path}`);
for (var v of folder.children) {
let entry = v as TFile & TFolder;
- this.addLog(`->entry:${entry.path}`);
+ this.addLog(`->entry:${entry.path}`, LOG_LEVEL.VERBOSE);
if (entry.children) {
- this.addLog(`->is dir`);
+ this.addLog(`->is dir`, LOG_LEVEL.VERBOSE);
await this.deleteFolderOnDB(entry);
try {
await this.app.vault.delete(entry);
@@ -1080,12 +1165,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (ex.code && ex.code == "ENOENT") {
//NO OP.
} else {
- this.addLog(`error while delete filder:${entry.path}`);
+ this.addLog(`error while delete filder:${entry.path}`, LOG_LEVEL.NOTICE);
this.addLog(ex);
}
}
} else {
- this.addLog(`->is file`);
+ this.addLog(`->is file`, LOG_LEVEL.VERBOSE);
await this.deleteFromDB(entry);
}
}
@@ -1095,7 +1180,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (ex.code && ex.code == "ENOENT") {
//NO OP.
} else {
- this.addLog(`error while delete filder:${folder.path}`);
+ this.addLog(`error while delete filder:${folder.path}`, LOG_LEVEL.NOTICE);
this.addLog(ex);
}
}
@@ -1116,7 +1201,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
// --> conflict resolving
async getConflictedDoc(path: string, rev: string): Promise {
try {
- let doc = await this.localDatabase.getDatabaseDoc(path, { rev: rev });
+ let doc = await this.localDatabase.getDBEntry(path, { rev: rev });
if (doc === false) return false;
let data = doc.data;
if (doc.datatype == "newnote") {
@@ -1137,42 +1222,44 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
return false;
}
+ /**
+ * Getting file conflicted status.
+ * @param path the file location
+ * @returns true -> resolved, false -> nothing to do, or check result.
+ */
async getConflictedStatus(path: string): Promise {
- let test: LoadedEntry & PouchDB.Core.GetMeta = null;
- try {
- let testDoc = await this.localDatabase.getDatabaseDoc(path, { conflicts: true });
- if (testDoc === false) return false;
- if ("_rev" in testDoc) {
- test = testDoc as any;
- }
- } catch (ex) {
- if (ex.status && ex.status == 404) {
- this.addLog(`Getting conflicted status, but there was not ${path}`);
- // NO OP.
- } else {
- throw ex;
- }
- }
+ let test = await this.localDatabase.getDBEntry(path, { conflicts: true });
+ if (test === false) return false;
if (test == null) return false;
if (!test._conflicts) return false;
if (test._conflicts.length == 0) return false;
- // should be two or more conflicts;
+ // should be one or more conflicts;
let leftLeaf = await this.getConflictedDoc(path, test._rev);
let rightLeaf = await this.getConflictedDoc(path, test._conflicts[0]);
- if (leftLeaf === false) return false;
- if (rightLeaf === false) return false;
+ if (leftLeaf == false) {
+ // what's going on..
+ this.addLog(`could not get current revisions:${path}`, LOG_LEVEL.NOTICE);
+ return false;
+ }
+ if (rightLeaf == false) {
+ // Conflicted item could not load, delete this.
+ await this.localDatabase.deleteDBEntry(path, { rev: test._conflicts[0] });
+ await this.pullFile(path, null, true);
+ this.addLog(`could not get old revisions, automaticaly used newer one:${path}`, LOG_LEVEL.NOTICE);
+ return true;
+ }
// first,check for same contents
if (leftLeaf.data == rightLeaf.data) {
let leaf = leftLeaf;
if (leftLeaf.mtime > rightLeaf.mtime) {
leaf = rightLeaf;
}
- await this.localDatabase.deleteDBEntry(path, leaf.rev);
+ await this.localDatabase.deleteDBEntry(path, { rev: leaf.rev });
await this.pullFile(path, null, true);
this.addLog(`automaticaly merged:${path}`);
return true;
- // }
}
+ // make diff.
let dmp = new diff_match_patch();
var diff = dmp.diff_main(leftLeaf.data, rightLeaf.data);
dmp.diff_cleanupSemantic(diff);
@@ -1190,41 +1277,58 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
//auto resolved, but need check again;
setTimeout(() => {
this.showIfConflicted(file);
- }, 50);
+ }, 500);
return;
}
//there conflicts, and have to resolve ;
let leaf = this.app.workspace.activeLeaf;
if (leaf) {
new ConflictResolveModal(this.app, conflictCheckResult, async (selected) => {
+ let testDoc = await this.localDatabase.getDBEntry(file.path, { conflicts: true });
+ if (testDoc === false) return;
+ if (!testDoc._conflicts) {
+ this.addLog("something went wrong on merging.", LOG_LEVEL.NOTICE);
+ return;
+ }
let toDelete = selected;
+ if (toDelete == null) {
+ //concat both,
+ if (conflictCheckResult !== false && conflictCheckResult !== true) {
+ // write data,and delete both old rev.
+ let p = conflictCheckResult.diff.map((e) => e[1]).join("");
+ await this.app.vault.modify(file, p);
+ await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.left.rev });
+ await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.right.rev });
+ }
+ return;
+ }
if (toDelete == "") {
return;
}
this.addLog(`resolved conflict:${file.path}`);
- await this.localDatabase.deleteDBEntry(file.path, toDelete);
+ await this.localDatabase.deleteDBEntry(file.path, { rev: toDelete });
await this.pullFile(file.path, null, true);
setTimeout(() => {
//resolved, check again.
this.showIfConflicted(file);
- }, 50);
+ }, 500);
}).open();
}
}
- async pullFile(filename: string, fileList?: TFile[], force?: boolean) {
+ async pullFile(filename: string, fileList?: TFile[], force?: boolean, rev?: string) {
if (!fileList) {
fileList = this.app.vault.getFiles();
}
let targetFiles = fileList.filter((e) => e.path == filename);
if (targetFiles.length == 0) {
//have to create;
- let doc = await this.localDatabase.getDatabaseDoc(filename);
+ let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null);
if (doc === false) return;
await this.doc2storage_create(doc, force);
} else if (targetFiles.length == 1) {
//normal case
let file = targetFiles[0];
- let doc = await this.localDatabase.getDatabaseDoc(filename);
+ let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null);
if (doc === false) return;
await this.doc2storate_modify(doc, file, force);
} else {
@@ -1234,7 +1338,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
//when to opened file;
}
async syncFileBetweenDBandStorage(file: TFile, fileList?: TFile[]) {
- let doc = await this.localDatabase.getDatabaseDoc(file.path);
+ let doc = await this.localDatabase.getDBEntry(file.path);
if (doc === false) return;
if (file.stat.mtime > doc.mtime) {
//newer local file.
@@ -1268,22 +1372,22 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
mtime: file.stat.mtime,
size: file.stat.size,
children: [],
- datatype: datatype
+ datatype: datatype,
};
//From here
- let old = await this.localDatabase.getDatabaseDoc(fullpath);
+ let old = await this.localDatabase.getDBEntry(fullpath);
if (old !== false) {
let oldData = { data: old.data, deleted: old._deleted };
let newData = { data: d.data, deleted: d._deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) {
- this.addLog("no changed" + fullpath + (d._deleted ? " (deleted)" : ""));
+ this.addLog("not changed:" + fullpath + (d._deleted ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return;
}
// d._rev = old._rev;
}
let ret = await this.localDatabase.putDBEntry(d);
- this.addLog("put database:" + fullpath + "(" + datatype + ")");
+ this.addLog("put database:" + fullpath + "(" + datatype + ") ");
if (this.settings.syncOnSave) {
await this.replicate();
}
@@ -1398,6 +1502,12 @@ class ConflictResolveModal extends Modal {
this.close();
});
});
+ contentEl.createEl("button", { text: "Concat both" }, (e) => {
+ e.addEventListener("click", async () => {
+ await this.callback(null);
+ this.close();
+ });
+ });
contentEl.createEl("button", { text: "Not now" }, (e) => {
e.addEventListener("click", async () => {
this.close();
@@ -1424,10 +1534,10 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
password: this.plugin.settings.couchDB_PASSWORD,
});
if (db === false) {
- this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI}`, true);
+ this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI}`, LOG_LEVEL.NOTICE);
return;
}
- this.plugin.addLog(`Connected to ${db.info.db_name}`, true);
+ this.plugin.addLog(`Connected to ${db.info.db_name}`, LOG_LEVEL.NOTICE);
}
display(): void {
let { containerEl } = this;
@@ -1503,8 +1613,8 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setValue(this.plugin.settings.gcDelay + "")
.onChange(async (value) => {
let v = Number(value);
- if (isNaN(v) || v < 200 || v > 5000) {
- return 30;
+ if (isNaN(v) || v > 5000) {
+ return 0;
//text.inputEl.va;
}
this.plugin.settings.gcDelay = v;
@@ -1521,6 +1631,15 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
+ new Setting(containerEl)
+ .setName("Verbose Log")
+ .setDesc("Show verbose log ")
+ .addToggle((toggle) =>
+ toggle.setValue(this.plugin.settings.showVerboseLog).onChange(async (value) => {
+ this.plugin.settings.showVerboseLog = value;
+ await this.plugin.saveSettings();
+ })
+ );
if (this.plugin.settings.versionUpFlash != "") {
let c = containerEl.createEl("div", { text: this.plugin.settings.versionUpFlash });
c.createEl("button", { text: "I got it and updated." }, (e) => {
@@ -1530,7 +1649,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
c.remove();
});
});
- c.addClass("op-warn")
+ c.addClass("op-warn");
}
// containerEl.createDiv(this.plugin.settings.versionUpFlash);
new Setting(containerEl)
@@ -1593,55 +1712,38 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
});
text.inputEl.setAttribute("type", "number");
});
- new Setting(containerEl)
- .setName("Local Database Operations")
- .addButton((button) =>
- button
- .setButtonText("Reset local database")
- .setDisabled(false)
- .onClick(async () => {
- await this.plugin.resetLocalDatabase();
- //await this.test();
- })
- )
- .addButton((button) =>
- button
- .setButtonText("Reset local files")
- .setDisabled(false)
- .onClick(async () => {
- //await this.test();
- })
- );
- new Setting(containerEl)
- .setName("Re-init")
- .addButton((button) =>
- button
- .setButtonText("Init Database again")
- .setDisabled(false)
- .onClick(async () => {
- await this.plugin.resetLocalDatabase();
- await this.plugin.initializeDatabase();
- })
- );
+ new Setting(containerEl).setName("Local Database Operations").addButton((button) =>
+ button
+ .setButtonText("Reset local database")
+ .setDisabled(false)
+ .onClick(async () => {
+ await this.plugin.resetLocalDatabase();
+ })
+ );
+ new Setting(containerEl).setName("Re-init").addButton((button) =>
+ button
+ .setButtonText("Init Database again")
+ .setDisabled(false)
+ .onClick(async () => {
+ await this.plugin.resetLocalDatabase();
+ await this.plugin.initializeDatabase();
+ })
+ );
- new Setting(containerEl)
- .setName("Garbage Collect")
- .addButton((button) =>
- button
- .setButtonText("Garbage Collection")
- .setDisabled(false)
- .onClick(async () => {
- await this.plugin.garbageCollect();
- //await this.test();
- })
- )
+ new Setting(containerEl).setName("Garbage Collect").addButton((button) =>
+ button
+ .setButtonText("Garbage Collection")
+ .setDisabled(false)
+ .onClick(async () => {
+ await this.plugin.garbageCollect();
+ })
+ );
new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
button
.setButtonText("Reset remote database")
.setDisabled(false)
.onClick(async () => {
await this.plugin.tryResetRemoteDatabase();
- //await this.test();
})
);
new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
@@ -1650,7 +1752,6 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setDisabled(false)
.onClick(async () => {
await this.plugin.tryResetRemoteDatabase();
- //await this.test();
})
);
}
diff --git a/manifest.json b/manifest.json
index 96067df..a53c694 100644
--- a/manifest.json
+++ b/manifest.json
@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Obsidian Live sync",
- "version": "0.1.0",
+ "version": "0.1.1",
"minAppVersion": "0.9.12",
"description": "obsidian Live synchronization plugin.",
"author": "vorotamoroz",
diff --git a/package.json b/package.json
index 1c3fc13..97815ca 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
- "version": "0.10.0",
+ "version": "0.1.1",
"description": "obsidian Live synchronization plugin.",
"main": "main.js",
"scripts": {