Compare commits

..

1 Commits

Author SHA1 Message Date
vorotamoroz
db68bc8e30 WIP 2022-12-06 09:06:44 +09:00
9 changed files with 107 additions and 60 deletions

View File

@@ -13,7 +13,7 @@ if you want to view the source, please visit the github repository of this plugi
const prod = process.argv[2] === "production"; const prod = process.argv[2] === "production";
const manifestJson = JSON.parse(fs.readFileSync("./manifest.json")); const manifestJson = JSON.parse(fs.readFileSync("./manifest.json"));
const packageJson = JSON.parse(fs.readFileSync("./package.json")); const packageJson = JSON.parse(fs.readFileSync("./package.json"));
const updateInfo = JSON.stringify(fs.readFileSync("./updates.md") + ""); const updateInfo = JSON.stringify("PATCHED-"+fs.readFileSync("./updates.md") + "");
esbuild esbuild
.build({ .build({
banner: { banner: {

View File

@@ -1,7 +1,7 @@
{ {
"id": "obsidian-livesync", "id": "obsidian-livesync",
"name": "Self-hosted LiveSync", "name": "Self-hosted LiveSync",
"version": "0.17.1", "version": "0.16.8",
"minAppVersion": "0.9.12", "minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz", "author": "vorotamoroz",

52
package-lock.json generated
View File

@@ -1,24 +1,26 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.17.1", "version": "0.16.8",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.17.1", "version": "0.16.8",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",
"esbuild": "0.15.15", "esbuild": "0.15.15",
"esbuild-svelte": "^0.7.3", "esbuild-svelte": "^0.7.3",
"idb": "^7.1.1", "idb": "^7.1.1",
"xxhash-wasm": "^0.4.2" "xxhash-wasm": "^0.4.2",
"xxhashjs": "^0.2.2"
}, },
"devDependencies": { "devDependencies": {
"@types/diff-match-patch": "^1.0.32", "@types/diff-match-patch": "^1.0.32",
"@types/pouchdb": "^6.4.0", "@types/pouchdb": "^6.4.0",
"@types/pouchdb-browser": "^6.1.3", "@types/pouchdb-browser": "^6.1.3",
"@types/xxhashjs": "^0.2.2",
"@typescript-eslint/eslint-plugin": "^5.44.0", "@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0", "@typescript-eslint/parser": "^5.44.0",
"builtin-modules": "^3.3.0", "builtin-modules": "^3.3.0",
@@ -442,6 +444,15 @@
"@types/estree": "*" "@types/estree": "*"
} }
}, },
"node_modules/@types/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-+hlk/W1kgnZn0vR22XNhxHk/qIRQYF54i0UTF2MwBAPd0e7xSy+jKOJwSwTdRQrNnOMRVv+vsh8ITV0uyhp2yg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@typescript-eslint/eslint-plugin": { "node_modules/@typescript-eslint/eslint-plugin": {
"version": "5.44.0", "version": "5.44.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz",
@@ -872,6 +883,11 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/cuint": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
"integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
},
"node_modules/debug": { "node_modules/debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3344,6 +3360,14 @@
"resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz", "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
"integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA==" "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
}, },
"node_modules/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
"dependencies": {
"cuint": "^0.2.2"
}
},
"node_modules/yallist": { "node_modules/yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -3736,6 +3760,15 @@
"@types/estree": "*" "@types/estree": "*"
} }
}, },
"@types/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-+hlk/W1kgnZn0vR22XNhxHk/qIRQYF54i0UTF2MwBAPd0e7xSy+jKOJwSwTdRQrNnOMRVv+vsh8ITV0uyhp2yg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@typescript-eslint/eslint-plugin": { "@typescript-eslint/eslint-plugin": {
"version": "5.44.0", "version": "5.44.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.44.0.tgz",
@@ -4011,6 +4044,11 @@
"which": "^2.0.1" "which": "^2.0.1"
} }
}, },
"cuint": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
"integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
},
"debug": { "debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -5678,6 +5716,14 @@
"resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz", "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
"integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA==" "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
}, },
"xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
"requires": {
"cuint": "^0.2.2"
}
},
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

View File

@@ -1,6 +1,6 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.17.1", "version": "0.16.8",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js", "main": "main.js",
"type": "module", "type": "module",
@@ -16,6 +16,7 @@
"@types/diff-match-patch": "^1.0.32", "@types/diff-match-patch": "^1.0.32",
"@types/pouchdb": "^6.4.0", "@types/pouchdb": "^6.4.0",
"@types/pouchdb-browser": "^6.1.3", "@types/pouchdb-browser": "^6.1.3",
"@types/xxhashjs": "^0.2.2",
"@typescript-eslint/eslint-plugin": "^5.44.0", "@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0", "@typescript-eslint/parser": "^5.44.0",
"builtin-modules": "^3.3.0", "builtin-modules": "^3.3.0",
@@ -37,6 +38,7 @@
"esbuild": "0.15.15", "esbuild": "0.15.15",
"esbuild-svelte": "^0.7.3", "esbuild-svelte": "^0.7.3",
"idb": "^7.1.1", "idb": "^7.1.1",
"xxhash-wasm": "^0.4.2" "xxhash-wasm": "^0.4.2",
"xxhashjs": "^0.2.2"
} }
} }

View File

@@ -7,6 +7,7 @@ import { EntryDoc, LOG_LEVEL } from "./lib/src/types.js";
import { enableEncryption } from "./lib/src/utils.js"; import { enableEncryption } from "./lib/src/utils.js";
import { isCloudantURI, isValidRemoteCouchDBURI } from "./lib/src/utils_couchdb.js"; import { isCloudantURI, isValidRemoteCouchDBURI } from "./lib/src/utils_couchdb.js";
import { id2path, path2id } from "./utils.js"; import { id2path, path2id } from "./utils.js";
import XXH from "xxhashjs";
export class LocalPouchDB extends LocalPouchDBBase { export class LocalPouchDB extends LocalPouchDBBase {
@@ -33,6 +34,13 @@ export class LocalPouchDB extends LocalPouchDBBase {
await this.kvDB.destroy(); await this.kvDB.destroy();
} }
async prepareHashFunctions() {
if (this.h32 != null) return;
// const { h32, h32Raw } = await xxhash();
this.h32 = (input: string, seed: number) => (XXH.h32(input, seed).toString(16))// h32;
this.h32Raw = (input: Uint8Array, seed: number) => (XXH.h32(input.buffer, seed).toNumber())// h32;
}
last_successful_post = false; last_successful_post = false;
getLastPostFailedBySize() { getLastPostFailedBySize() {
return !this.last_successful_post; return !this.last_successful_post;

View File

@@ -1,7 +1,7 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "obsidian"; import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "obsidian";
import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, RemoteDBSettings } from "./lib/src/types"; import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, RemoteDBSettings } from "./lib/src/types";
import { path2id, id2path } from "./utils"; import { path2id, id2path } from "./utils";
import { delay, Semaphore, versionNumberString2Number } from "./lib/src/utils"; import { delay, versionNumberString2Number } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js"; import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
import { testCrypt } from "./lib/src/e2ee_v2"; import { testCrypt } from "./lib/src/e2ee_v2";
@@ -1288,6 +1288,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
new Setting(containerHatchEl) new Setting(containerHatchEl)
.setName("Make report to inform the issue") .setName("Make report to inform the issue")
.setDesc("Verify and repair all files and update database without restoring")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Make report") .setButtonText("Make report")
@@ -1367,28 +1368,21 @@ ${stringifyYaml(pluginConfig)}`;
.setDisabled(false) .setDisabled(false)
.setWarning() .setWarning()
.onClick(async () => { .onClick(async () => {
const semaphore = Semaphore(10);
const files = this.app.vault.getFiles(); const files = this.app.vault.getFiles();
Logger("Verify and repair all files started", LOG_LEVEL.NOTICE, "verify");
// const notice = NewNotice("", 0);
let i = 0; let i = 0;
const processes = files.map(e => (async (file) => { for (const file of files) {
const releaser = await semaphore.acquire(1, "verifyAndRepair"); i++;
Logger(`Update into ${file.path}`);
Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
try { try {
Logger(`Update into ${file.path}`); await this.plugin.updateIntoDB(file);
await this.plugin.updateIntoDB(file, false, null, true);
i++;
Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
} catch (ex) { } catch (ex) {
i++; Logger("could not update:");
Logger(`Error while verifyAndRepair`, LOG_LEVEL.NOTICE);
Logger(ex); Logger(ex);
} finally {
releaser();
} }
} }
)(e));
await Promise.all(processes);
Logger("done", LOG_LEVEL.NOTICE, "verify"); Logger("done", LOG_LEVEL.NOTICE, "verify");
}) })
); );

Submodule src/lib updated: a13d921fb7...85bb3556ba

View File

@@ -709,7 +709,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
clearAllPeriodic(); clearAllPeriodic();
clearAllTriggers(); clearAllTriggers();
window.removeEventListener("visibilitychange", this.watchWindowVisibility); window.removeEventListener("visibilitychange", this.watchWindowVisibility);
window.removeEventListener("online", this.watchOnline); window.removeEventListener("online", this.watchOnline)
Logger("unloading plugin"); Logger("unloading plugin");
} }
@@ -2192,7 +2192,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData, force?: boolean) { async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData) {
if (!this.isTargetFile(file)) return; if (!this.isTargetFile(file)) return;
if (shouldBeIgnored(file.path)) { if (shouldBeIgnored(file.path)) {
return; return;
@@ -2234,24 +2234,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (recentlyTouched(file)) { if (recentlyTouched(file)) {
return true; return true;
} }
try { const old = await this.localDatabase.getDBEntry(fullPath, null, false, false);
const old = await this.localDatabase.getDBEntry(fullPath, null, false, false); if (old !== false) {
if (old !== false) { const oldData = { data: old.data, deleted: old._deleted || old.deleted, };
const oldData = { data: old.data, deleted: old._deleted || old.deleted, }; const newData = { data: d.data, deleted: d._deleted || d.deleted };
const newData = { data: d.data, deleted: d._deleted || d.deleted }; if (JSON.stringify(oldData) == JSON.stringify(newData)) {
if (JSON.stringify(oldData) == JSON.stringify(newData)) { Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE); return true;
return true;
}
// d._rev = old._rev;
} }
} catch (ex) { // d._rev = old._rev;
if (force) {
Logger(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
} else {
Logger(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
}
return !force;
} }
return false; return false;
}); });

View File

@@ -1,19 +1,3 @@
### 0.17.0
- 0.17.0 has no surfaced changes but the design of saving chunks has been changed. They have compatibility but changing files after upgrading makes different chunks than before 0.16.x.
Please rebuild databases once if you have been worried about storage usage.
- Improved:
- Splitting markdown
- Saving chunks
- Changed:
- Chunk ID numbering rules
#### Minors
- 0.17.1
- Fixed: Now we can verify and repair the database.
- Refactored inside.
### 0.16.0 ### 0.16.0
- Now hidden files need not be scanned. Changes will be detected automatically. - Now hidden files need not be scanned. Changes will be detected automatically.
- If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`. - If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.
@@ -47,3 +31,25 @@
Note: Note:
Before 0.16.5, LiveSync had some issues making chunks. In this case, synchronisation had became been always failing after a corrupted one should be made. After 0.16.6, the corrupted chunk is automatically detected. Sorry for troubling you but please do `rebuild everything` when this plug-in notified so. Before 0.16.5, LiveSync had some issues making chunks. In this case, synchronisation had became been always failing after a corrupted one should be made. After 0.16.6, the corrupted chunk is automatically detected. Sorry for troubling you but please do `rebuild everything` when this plug-in notified so.
### 0.15.0
- Outdated configuration items have been removed.
- Setup wizard has been implemented!
I appreciate for reviewing and giving me advice @Pouhon158!
#### Minors
- 0.15.1 Missed the stylesheet.
- 0.15.2 The wizard has been improved and documented!
- 0.15.3 Fixed the issue about locking/unlocking remote database while rebuilding in the wizard.
- 0.15.4 Fixed issues about asynchronous processing (e.g., Conflict check or hidden file detection)
- 0.15.5 Add new features for setting Self-hosted LiveSync up more easier.
- 0.15.6 File tracking logic has been refined.
- 0.15.7 Fixed bug about renaming file.
- 0.15.8 Fixed bug about deleting empty directory, weird behaviour on boot-sequence on mobile devices.
- 0.15.9 Improved chunk retrieving, now chunks are retrieved in batch on continuous requests.
- 0.15.10 Fixed:
- The boot sequence has been corrected and now boots smoothly.
- Auto applying of batch save will be processed earlier than before.
... To continue on to `updates_old.md`.