Compare commits

...

3 Commits

Author SHA1 Message Date
vorotamoroz
2b11be05ec Add new feature:
- Reread all files
2021-12-06 12:19:05 +09:00
vorotamoroz
0ee73860d1 Fixed:
- Make less file corruption.
- Some notice was not hidden automatically
2021-12-06 11:43:42 +09:00
vorotamoroz
ecec546f13 Improvements:
- Show sync status information inside the editor.

Fixed:
- Reduce the same messages on popup notifications
- show warning message when synchronization
2021-12-03 12:54:18 +09:00
7 changed files with 242 additions and 81 deletions

View File

@@ -143,6 +143,11 @@ You can dump saved note structure to `Dump informations of this doc`. Replace ev
Default values are 20 letters and 250 letters.
## Miscellaneous
### Show status inside editor
Show information inside the editor pane.
It would be useful for mobile.
## Hatch
From here, everything is under the hood. Please handle it with care.
@@ -160,6 +165,9 @@ The remote database indicates that has been unlocked Pattern 1.
When you mark all devices as resolved, you can unlock the database.
But, there's no problem even if you leave it as it is.
### Reread all files
Reread all files in the vault, and update them into the database if there's diff or could not read from the database.
### Drop history
Drop all histories on the local database and the remote database, and initialize When synchronization time has been prolonged to the new device or new vault, or database size became to be much larger. Try this.

View File

@@ -142,6 +142,12 @@ Self-hosted LiveSyncは一つのチャンクのサイズを最低minimum chunk s
改行文字と#を除き、すべて●に置換しても、アルゴリズムは有効に働きます。
デフォルトは20文字と、250文字です。
## Miscellaneous
その他の設定です
### Show status inside editor
同期の情報をエディター内に表示します。
モバイルで便利です。
## Hatch
ここから先は、困ったときに開ける蓋の中身です。注意して使用してください。
@@ -160,6 +166,9 @@ Self-hosted LiveSyncは一つのチャンクのサイズを最低minimum chunk s
ご使用のすべてのデバイスでロックを解除した場合は、データベースのロックを解除することができます。
ただし、このまま放置しても問題はありません。
### Reread all files
Vault内のファイルを全て読み込み直し、もし差分があったり、データベースから正常に読み込めなかったものに関して、データベースに反映します。
### Drop history
データベースに記録されている履歴を削除し、データベースを初期化します。
新しい端末や新しいVaultへの同期にやたらと時間がかかったり、データベースサイズが肥大化したりしてきた際に使用してください。

260
main.ts
View File

@@ -54,6 +54,7 @@ interface ObsidianLiveSyncSettings {
deviceAndVaultName: string;
usePluginSettings: boolean;
showOwnPlugins: boolean;
showStatusOnEditor: boolean;
}
const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -86,6 +87,7 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
deviceAndVaultName: "",
usePluginSettings: false,
showOwnPlugins: false,
showStatusOnEditor: false,
};
interface Entry {
@@ -551,6 +553,13 @@ async function testCrypt() {
}
}
// <-- Encryption
const delay = (ms: number): Promise<void> => {
return new Promise((res) => {
setTimeout(() => {
res();
}, ms);
});
};
//<--Functions
class LocalPouchDB {
auth: Credential;
@@ -705,6 +714,7 @@ class LocalPouchDB {
}
async getDBLeaf(id: string, waitForReady: boolean): Promise<string> {
await this.waitForGCComplete();
// when in cache, use that.
if (this.hashCacheRev[id]) {
return this.hashCacheRev[id];
@@ -764,6 +774,7 @@ class LocalPouchDB {
}
async getDBEntryMeta(path: string, opt?: PouchDB.Core.GetOptions): Promise<false | LoadedEntry> {
await this.waitForGCComplete();
let id = path2id(path);
try {
let obj: EntryDocResponse = null;
@@ -808,6 +819,7 @@ class LocalPouchDB {
return false;
}
async getDBEntry(path: string, opt?: PouchDB.Core.GetOptions, dump = false, waitForReady = true): Promise<false | LoadedEntry> {
await this.waitForGCComplete();
let id = path2id(path);
try {
let obj: EntryDocResponse = null;
@@ -907,6 +919,7 @@ class LocalPouchDB {
return false;
}
async deleteDBEntry(path: string, opt?: PouchDB.Core.GetOptions): Promise<boolean> {
await this.waitForGCComplete();
let id = path2id(path);
try {
let obj: EntryDocResponse = null;
@@ -949,6 +962,7 @@ class LocalPouchDB {
}
}
async deleteDBEntryPrefix(prefixSrc: string): Promise<boolean> {
await this.waitForGCComplete();
// delete database entries by prefix.
// it called from folder deletion.
let c = 0;
@@ -1010,6 +1024,7 @@ class LocalPouchDB {
return false;
}
async putDBEntry(note: LoadedEntry) {
await this.waitForGCComplete();
let leftData = note.data;
let savenNotes = [];
let processed = 0;
@@ -1207,7 +1222,7 @@ class LocalPouchDB {
throw ex;
}
}
let r = await this.localDatabase.put(newDoc);
let r = await this.localDatabase.put(newDoc, { force: true });
this.updateRecentModifiedDocs(r.id, r.rev, newDoc._deleted);
if (typeof this.corruptedEntries[note._id] != "undefined") {
delete this.corruptedEntries[note._id];
@@ -1233,6 +1248,7 @@ class LocalPouchDB {
}
replicateAllToServer(setting: ObsidianLiveSyncSettings, showingNotice?: boolean) {
return new Promise(async (res, rej) => {
await this.waitForGCComplete();
this.closeReplication();
Logger("send all data to server", LOG_LEVEL.NOTICE);
let notice: Notice = null;
@@ -1305,6 +1321,7 @@ class LocalPouchDB {
return false;
}
await this.waitForGCComplete();
if (setting.versionUpFlash != "") {
new Notice("Open settings and check message, please.");
return;
@@ -1481,6 +1498,7 @@ class LocalPouchDB {
}
async resetDatabase() {
await this.waitForGCComplete();
if (this.changeHandler != null) {
this.changeHandler.removeAllListeners();
this.changeHandler.cancel();
@@ -1587,72 +1605,87 @@ class LocalPouchDB {
Logger("Mark this device as 'resolved'.", LOG_LEVEL.NOTICE);
await dbret.db.put(remoteMilestone);
}
gcRunning = false;
async waitForGCComplete() {
while (this.gcRunning) {
Logger("Waiting for Garbage Collection completed.");
await delay(1000);
}
}
async garbageCollect() {
// get all documents of NewEntry2
// we don't use queries , just use allDocs();
let c = 0;
let readCount = 0;
let hashPieces: string[] = [];
let usedPieces: string[] = [];
Logger("Collecting Garbage");
do {
let result = await this.localDatabase.allDocs({ include_docs: true, skip: c, limit: 500, conflicts: true });
readCount = result.rows.length;
Logger("checked:" + readCount);
if (readCount > 0) {
//there are some result
for (let v of result.rows) {
let doc = v.doc;
if (doc.type == "newnote" || doc.type == "plain") {
// used pieces memo.
usedPieces = Array.from(new Set([...usedPieces, ...doc.children]));
if (doc._conflicts) {
for (let cid of doc._conflicts) {
let p = await this.localDatabase.get<EntryDoc>(doc._id, { rev: cid });
if (p.type == "newnote" || p.type == "plain") {
usedPieces = Array.from(new Set([...usedPieces, ...p.children]));
if (this.gcRunning) return;
this.gcRunning = true;
try {
// get all documents of NewEntry2
// we don't use queries , just use allDocs();
this.disposeHashCache();
let c = 0;
let readCount = 0;
let hashPieces: string[] = [];
let usedPieces: string[] = [];
Logger("Collecting Garbage");
do {
let result = await this.localDatabase.allDocs({ include_docs: true, skip: c, limit: 500, conflicts: true });
readCount = result.rows.length;
Logger("checked:" + readCount);
if (readCount > 0) {
//there are some result
for (let v of result.rows) {
let doc = v.doc;
if (doc.type == "newnote" || doc.type == "plain") {
// used pieces memo.
usedPieces = Array.from(new Set([...usedPieces, ...doc.children]));
if (doc._conflicts) {
for (let cid of doc._conflicts) {
let p = await this.localDatabase.get<EntryDoc>(doc._id, { rev: cid });
if (p.type == "newnote" || p.type == "plain") {
usedPieces = Array.from(new Set([...usedPieces, ...p.children]));
}
}
}
}
if (doc.type == "leaf") {
// all pieces.
hashPieces = Array.from(new Set([...hashPieces, doc._id]));
}
}
if (doc.type == "leaf") {
// all pieces.
hashPieces = Array.from(new Set([...hashPieces, doc._id]));
}
c += readCount;
} while (readCount != 0);
// items collected.
Logger("Finding unused pieces");
this.disposeHashCache();
const garbages = hashPieces.filter((e) => usedPieces.indexOf(e) == -1);
let deleteCount = 0;
Logger("we have to delete:" + garbages.length);
let deleteDoc: EntryDoc[] = [];
for (let v of garbages) {
try {
let item = await this.localDatabase.get(v);
item._deleted = true;
deleteDoc.push(item);
if (deleteDoc.length > 50) {
await this.localDatabase.bulkDocs(deleteDoc);
deleteDoc = [];
Logger("delete:" + deleteCount);
}
deleteCount++;
} catch (ex) {
if (ex.status && ex.status == 404) {
// NO OP. It should be timing problem.
} else {
throw ex;
}
}
}
c += readCount;
} while (readCount != 0);
// items collected.
Logger("Finding unused pieces");
const garbages = hashPieces.filter((e) => usedPieces.indexOf(e) == -1);
let deleteCount = 0;
Logger("we have to delete:" + garbages.length);
let deleteDoc: EntryDoc[] = [];
for (let v of garbages) {
try {
let item = await this.localDatabase.get(v);
item._deleted = true;
deleteDoc.push(item);
if (deleteDoc.length > 50) {
await this.localDatabase.bulkDocs(deleteDoc);
deleteDoc = [];
Logger("delete:" + deleteCount);
}
deleteCount++;
} catch (ex) {
if (ex.status && ex.status == 404) {
// NO OP. It should be timing problem.
} else {
throw ex;
}
if (deleteDoc.length > 0) {
await this.localDatabase.bulkDocs(deleteDoc);
}
Logger(`GC:deleted ${deleteCount} items.`);
} finally {
this.gcRunning = false;
}
if (deleteDoc.length > 0) {
await this.localDatabase.bulkDocs(deleteDoc);
}
Logger(`GC:deleted ${deleteCount} items.`);
this.disposeHashCache();
}
}
@@ -2004,6 +2037,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
addLogHook: () => void = null;
//--> Basic document Functions
notifies: { [key: string]: { notice: Notice; timer: NodeJS.Timeout; count: number } } = {};
async addLog(message: any, level: LOG_LEVEL = LOG_LEVEL.INFO) {
if (level < LOG_LEVEL.INFO && this.settings && this.settings.lessInformationInLog) {
return;
@@ -2021,8 +2055,32 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
// if (this.statusBar2 != null) {
// this.statusBar2.setText(newmessage.substring(0, 60));
// }
if (level >= LOG_LEVEL.NOTICE) {
new Notice(messagecontent);
if (messagecontent in this.notifies) {
clearTimeout(this.notifies[messagecontent].timer);
this.notifies[messagecontent].count++;
this.notifies[messagecontent].notice.setMessage(`(${this.notifies[messagecontent].count}):${messagecontent}`);
this.notifies[messagecontent].timer = setTimeout(() => {
const notify = this.notifies[messagecontent].notice;
delete this.notifies[messagecontent];
try {
notify.hide();
} catch (ex) {
// NO OP
}
}, 5000);
} else {
let notify = new Notice(messagecontent, 0);
this.notifies[messagecontent] = {
count: 0,
notice: notify,
timer: setTimeout(() => {
delete this.notifies[messagecontent];
notify.hide();
}, 5000),
};
}
}
if (this.addLogHook != null) this.addLogHook();
}
@@ -2241,6 +2299,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
this.setPeriodicSync();
}
lastMessage = "";
refreshStatusText() {
let sent = this.localDatabase.docSent;
let arrived = this.localDatabase.docArrived;
@@ -2268,9 +2327,23 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
let waiting = "";
if (this.settings.batchSave) {
waiting = " " + this.batchFileChange.map((e) => "🛫").join("");
waiting = waiting.replace(/🛫{10}/g,"🚀");
waiting = waiting.replace(/🛫{10}/g, "🚀");
}
const message = `Sync:${w}${sent}${arrived}${waiting}`;
this.setStatusBarText(message);
}
setStatusBarText(message: string) {
if (this.lastMessage != message) {
this.statusBar.setText(message);
if (this.settings.showStatusOnEditor) {
const root = document.documentElement;
root.style.setProperty("--slsmessage", '"' + message + '"');
} else {
const root = document.documentElement;
root.style.setProperty("--slsmessage", '""');
}
this.lastMessage = message;
}
this.statusBar.setText(`Sync:${w}${sent}${arrived}${waiting}`);
}
async replicate(showMessage?: boolean) {
if (this.settings.versionUpFlash != "") {
@@ -2316,7 +2389,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
Logger("Initialize and checking database files");
Logger("Updating database by new files");
this.statusBar.setText(`UPDATE DATABASE`);
this.setStatusBarText(`UPDATE DATABASE`);
let _this = this;
async function runAll<T>(procedurename: string, objects: T[], callback: (arg: T) => Promise<void>) {
const count = objects.length;
Logger(procedurename);
@@ -2333,7 +2407,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (notice != null) notice.setMessage(notify);
Logger(notify);
// lastTicks = performance.now() + 2000;
// this.statusBar.setText(notify);
_this.setStatusBarText(notify);
}
} catch (ex) {
Logger(`Error while ${procedurename}`, LOG_LEVEL.NOTICE);
@@ -2369,7 +2443,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await runAll("CHECK FILE STATUS", syncFiles, async (e) => {
await this.syncFileBetweenDBandStorage(e, filesStorage);
});
this.statusBar.setText(`NOW TRACKING!`);
this.setStatusBarText(`NOW TRACKING!`);
Logger("Initialized,NOW TRACKING!");
if (showingNotice) {
notice.hide();
@@ -2614,6 +2688,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async updateIntoDB(file: TFile) {
await this.localDatabase.waitForGCComplete();
let content = "";
let datatype: "plain" | "newnote" = "newnote";
if (file.extension != "md") {
@@ -2807,6 +2882,9 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
containerEl.createEl("h2", { text: "Settings for Self-hosted LiveSync." });
containerEl.createEl("h3", { text: "Remote Database configuration" });
let syncWarn = containerEl.createEl("div", { text: "The remote configuration is locked while any synchronization is enabled." });
syncWarn.addClass("op-warn");
syncWarn.addClass("sls-hidden");
const isAnySyncEnabled = (): boolean => {
if (this.plugin.settings.liveSync) return true;
@@ -2820,10 +2898,12 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
dbsettings.forEach((e) => {
e.setDisabled(true).setTooltip("When any sync is enabled, It cound't be changed.");
});
syncWarn.removeClass("sls-hidden");
} else {
dbsettings.forEach((e) => {
e.setDisabled(false).setTooltip("");
});
syncWarn.addClass("sls-hidden");
}
if (this.plugin.settings.liveSync) {
syncNonLive.forEach((e) => {
@@ -3215,6 +3295,17 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text.inputEl.setAttribute("type", "number");
});
containerEl.createEl("h3", { text: "Miscellaneous" });
new Setting(containerEl)
.setName("Show status inside editor")
.setDesc("")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.showStatusOnEditor).onChange(async (value) => {
this.plugin.settings.showStatusOnEditor = value;
await this.plugin.saveSettings();
})
);
containerEl.createEl("h3", { text: "Hatch" });
if (this.plugin.localDatabase.remoteLockedAndDeviceNotAccepted) {
@@ -3265,6 +3356,35 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.replicate(true);
}
};
new Setting(containerEl)
.setName("Reread all files")
.setDesc("Reread all files and update the database without dropping history")
.addButton((button) =>
button
.setButtonText("Reread")
.setDisabled(false)
.setWarning()
.onClick(async () => {
const files = this.app.vault.getFiles();
Logger("Reread all files started", LOG_LEVEL.NOTICE);
let notice = new Notice("", 0);
let i = 0;
for (const file of files) {
i++;
Logger(`Update into ${file.path}`);
notice.setMessage(`${i}/${files.length}\n${file.path}`);
try {
await this.plugin.updateIntoDB(file);
} catch (ex) {
Logger("could not update:");
Logger(ex);
}
}
notice.hide();
Logger("done", LOG_LEVEL.NOTICE);
})
);
new Setting(containerEl)
.setName("Drop History")
.setDesc("Initialize local and remote database, and send all or retrieve all again.")
@@ -3469,7 +3589,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
html += `
<tr>
<th colspan=2>${escapeStringToHTML(vaults)}</th>
</tr>`
</tr>`;
for (let v of plugins[vaults]) {
let mtime = v.mtime == 0 ? "-" : new Date(v.mtime).toLocaleString();
let settingApplyable: boolean | string = "-";
@@ -3480,9 +3600,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (thisDevicePlugins[v.manifest.id].manifest.version == v.manifest.version) {
isSameVersion = true;
}
if (thisDevicePlugins[v.manifest.id].styleCss == v.styleCss &&
thisDevicePlugins[v.manifest.id].mainJs == v.mainJs &&
thisDevicePlugins[v.manifest.id].manifestJson == v.manifestJson) {
if (thisDevicePlugins[v.manifest.id].styleCss == v.styleCss && thisDevicePlugins[v.manifest.id].mainJs == v.mainJs && thisDevicePlugins[v.manifest.id].manifestJson == v.manifestJson) {
isSameContents = true;
}
}
@@ -3515,7 +3633,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
</tr>
<tr>
<th class='sls-table-head'>${escapeStringToHTML(v.manifest.name)}</th>
<td class="sls-table-tail tcenter">${isSameContents?"even":`<button data-key='${v._id}' class='apply-plugin-version mod-cta'>Use (${isSameVersion ? "=" : ""}${v.manifest.version}) </button>`}</td>
<td class="sls-table-tail tcenter">${isSameContents ? "even" : `<button data-key='${v._id}' class='apply-plugin-version mod-cta'>Use (${isSameVersion ? "=" : ""}${v.manifest.version}) </button>`}</td>
</tr>
<tr>
<td class="sls-table-head tcenter">${escapeStringToHTML(mtime)}</td>
@@ -3528,7 +3646,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
<tr class='divider'>
<th colspan=2></th>
</tr>
`
`;
}
html += "</table></div>";
pluginConfig.innerHTML = html;
@@ -3635,7 +3753,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
xx.remove();
});
});
ba.addClass("mod-warning")
ba.addClass("mod-warning");
xx.createEl("button", { text: `Restore from file` }, (e) => {
e.addEventListener("click", async () => {
let f = await this.app.vault.getFiles().filter((e) => path2id(e.path) == k);
@@ -3647,7 +3765,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
xx.remove();
});
});
xx.addClass("mod-warning")
xx.addClass("mod-warning");
}
} else {
let cx = containerEl.createEl("div", { text: "There's no collupted data." });

View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.1.22",
"version": "0.1.24",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.1.22",
"version": "0.1.24",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.1.22",
"version": "0.1.24",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.1.22",
"version": "0.1.24",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"scripts": {

View File

@@ -37,11 +37,11 @@
/* overflow: scroll; */
}
.sls-plugins-tbl {
border:1px solid var(--background-modifier-border);
border: 1px solid var(--background-modifier-border);
width: 100%;
}
.divider th{
border-top:1px solid var(--background-modifier-border);
.divider th {
border-top: 1px solid var(--background-modifier-border);
}
/* .sls-table-head{
width:50%;
@@ -52,8 +52,34 @@
} */
.sls-btn-left {
padding-right:4px;
padding-right: 4px;
}
.sls-btn-right {
padding-left:4px;
}
padding-left: 4px;
}
.sls-hidden {
display: none;
}
:root {
--slsmessage: "";
}
.CodeMirror-wrap::before , .cm-s-obsidian > .cm-editor::before {
content: var(--slsmessage);
position: absolute;
border-radius: 4px;
/* border:1px solid --background-modifier-border; */
display: inline-block;
top: 8px;
color: --text-normal;
opacity: 0.5;
font-size:80%;
-webkit-filter: grayscale(100%);
filter: grayscale(100%);
}
.CodeMirror-wrap::before {
right: 0px;
} .cm-s-obsidian > .cm-editor::before {
right: 16px;
}