public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-02-20 11:08:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(format): no intentional behaviour change - runs pretty

Browse Source
This commit is contained in:
Frank Harrison
2024-11-11 09:39:45 +00:00
parent 6e1eb36f3b
commit 5c97e5b672
71 changed files with 6029 additions and 3740 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
src/modules/core/ModuleDatabaseFileAccess.ts
View File

@@ -1,15 +1,29 @@
import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
import { getPathFromUXFileInfo, isInternalMetadata, markChangesAreSame } from "../../common/utils";
import type { UXFileInfoStub, FilePathWithPrefix, UXFileInfo, MetaEntry, LoadedEntry, FilePath, SavingEntry } from "../../lib/src/common/types";
import type {
UXFileInfoStub,
FilePathWithPrefix,
UXFileInfo,
MetaEntry,
LoadedEntry,
FilePath,
SavingEntry,
} from "../../lib/src/common/types";
import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess";
import { type IObsidianModule } from "../AbstractObsidianModule.ts";
import { isPlainText, shouldBeIgnored } from "../../lib/src/string_and_binary/path";
import { createBlob, createTextBlob, delay, determineTypeFromBlob, isDocContentSame, readContent } from "../../lib/src/common/utils";
import {
createBlob,
createTextBlob,
delay,
determineTypeFromBlob,
isDocContentSame,
readContent,
} from "../../lib/src/common/utils";
import { serialized } from "octagonal-wheels/concurrency/lock";
import { AbstractModule } from "../AbstractModule.ts";
export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidianModule, DatabaseFileAccess {
$everyOnload(): Promise<boolean> {
this.core.databaseFileAccess = this;
@@ -18,7 +32,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
async $everyModuleTest(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
const testString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam nec purus nec nunc"
const testString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam nec purus nec nunc";
// Before test, we need to delete completely.
const conflicts = await this.getConflictedRevs("autoTest.md" as FilePathWithPrefix);
for (const rev of conflicts) {
@@ -27,16 +41,21 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
await this.delete("autoTest.md" as FilePathWithPrefix);
// OK, begin!
await this._test("storeContent", async () => (await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)));
await this._test(
"storeContent",
async () => await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)
);
// For test, we need to clear the caches.
await this.localDatabase.hashCaches.clear();
await this._test("readContent", async () => {
const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
if (!content) return "File not found";
if (content.deleted) return "File is deleted";
return (await content.body.text() == testString) ? true : `Content is not same ${await content.body.text()}`;
return (await content.body.text()) == testString
? true
: `Content is not same ${await content.body.text()}`;
});
await this._test("delete", async () => (await this.delete("autoTest.md" as FilePathWithPrefix)));
await this._test("delete", async () => await this.delete("autoTest.md" as FilePathWithPrefix));
await this._test("read deleted content", async () => {
const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
if (!content) return true;
@@ -49,7 +68,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
async checkIsTargetFile(file: UXFileInfoStub | FilePathWithPrefix): Promise<boolean> {
const path = getPathFromUXFileInfo(file);
if (!await this.core.$$isTargetFile(path)) {
if (!(await this.core.$$isTargetFile(path))) {
this._log(`File is not target`, LOG_LEVEL_VERBOSE);
return false;
}
@@ -61,7 +80,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
}
async delete(file: UXFileInfoStub | FilePathWithPrefix, rev?: string): Promise<boolean> {
if (!await this.checkIsTargetFile(file)) {
if (!(await this.checkIsTargetFile(file))) {
return true;
}
const fullPath = getPathFromUXFileInfo(file);
@@ -95,13 +114,18 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
type: "file",
},
body: blob,
}
};
return await this._store(dummyUXFileInfo, true, false, false);
}
async _store(file: UXFileInfo, force: boolean = false, skipCheck?: boolean, onlyChunks?: boolean): Promise<boolean> {
async _store(
file: UXFileInfo,
force: boolean = false,
skipCheck?: boolean,
onlyChunks?: boolean
): Promise<boolean> {
if (!skipCheck) {
if (!await this.checkIsTargetFile(file)) {
if (!(await this.checkIsTargetFile(file))) {
return true;
}
}
@@ -147,17 +171,33 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
const oldData = { data: old.data, deleted: old._deleted || old.deleted };
const newData = { data: d.data, deleted: d._deleted || d.deleted };
if (oldData.deleted != newData.deleted) return false;
if (!await isDocContentSame(old.data, newData.data)) return false;
this._log(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
if (!(await isDocContentSame(old.data, newData.data))) return false;
this._log(
msg + "Skipped (not changed) " + fullPath + (d._deleted || d.deleted ? " (deleted)" : ""),
LOG_LEVEL_VERBOSE
);
markChangesAreSame(old, d.mtime, old.mtime);
return true;
// d._rev = old._rev;
}
} catch (ex) {
if (force) {
this._log(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
this._log(
msg +
"Error, Could not check the diff for the old one." +
(force ? "force writing." : "") +
fullPath +
(d._deleted || d.deleted ? " (deleted)" : ""),
LOG_LEVEL_VERBOSE
);
} else {
this._log(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
this._log(
msg +
"Error, Could not check the diff for the old one." +
fullPath +
(d._deleted || d.deleted ? " (deleted)" : ""),
LOG_LEVEL_VERBOSE
);
}
this._log(ex, LOG_LEVEL_VERBOSE);
return !force;
@@ -177,7 +217,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
}
async getConflictedRevs(file: UXFileInfoStub | FilePathWithPrefix): Promise<string[]> {
if (!await this.checkIsTargetFile(file)) {
if (!(await this.checkIsTargetFile(file))) {
return [];
}
const filename = getPathFromUXFileInfo(file);
@@ -188,9 +228,13 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
return doc._conflicts || [];
}
async fetch(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady?: boolean, skipCheck = false): Promise<UXFileInfo | false> {
if (skipCheck && !await this.checkIsTargetFile(file)) {
async fetch(
file: UXFileInfoStub | FilePathWithPrefix,
rev?: string,
waitForReady?: boolean,
skipCheck = false
): Promise<UXFileInfo | false> {
if (skipCheck && !(await this.checkIsTargetFile(file))) {
return false;
}
@@ -210,39 +254,55 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
},
body: data,
deleted: entry.deleted || entry._deleted,
}
};
if (isInternalMetadata(entry.path)) {
fileInfo.isInternal = true;
}
return fileInfo;
}
async fetchEntryMeta(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, skipCheck = false): Promise<MetaEntry | false> {
async fetchEntryMeta(
file: UXFileInfoStub | FilePathWithPrefix,
rev?: string,
skipCheck = false
): Promise<MetaEntry | false> {
const filename = getPathFromUXFileInfo(file);
if (skipCheck && !await this.checkIsTargetFile(file)) {
if (skipCheck && !(await this.checkIsTargetFile(file))) {
return false;
}
const doc = await this.localDatabase.getDBEntryMeta(
filename, rev ? { rev: rev } : undefined, true);
const doc = await this.localDatabase.getDBEntryMeta(filename, rev ? { rev: rev } : undefined, true);
if (doc === false) {
return false;
}
return doc as MetaEntry;
}
async fetchEntryFromMeta(meta: MetaEntry, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
if (skipCheck && !await this.checkIsTargetFile(meta.path)) {
async fetchEntryFromMeta(
meta: MetaEntry,
waitForReady: boolean = true,
skipCheck = false
): Promise<LoadedEntry | false> {
if (skipCheck && !(await this.checkIsTargetFile(meta.path))) {
return false;
}
const doc = await this.localDatabase.getDBEntryFromMeta(meta as LoadedEntry, undefined, false, waitForReady, true);
const doc = await this.localDatabase.getDBEntryFromMeta(
meta as LoadedEntry,
undefined,
false,
waitForReady,
true
);
if (doc === false) {
return false;
}
return doc;
}
async fetchEntry(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
if (skipCheck && !await this.checkIsTargetFile(file)) {
async fetchEntry(
file: UXFileInfoStub | FilePathWithPrefix,
rev?: string,
waitForReady: boolean = true,
skipCheck = false
): Promise<LoadedEntry | false> {
if (skipCheck && !(await this.checkIsTargetFile(file))) {
return false;
}
const entry = await this.fetchEntryMeta(file, rev, true);
@@ -253,7 +313,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
return doc;
}
async deleteFromDBbyPath(fullPath: FilePath | FilePathWithPrefix, rev?: string): Promise<boolean> {
if (!await this.checkIsTargetFile(fullPath)) {
if (!(await this.checkIsTargetFile(fullPath))) {
this._log(`storeFromStorage: File is not target: ${fullPath}`);
return true;
}
@@ -262,5 +322,4 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
eventHub.emitEvent(EVENT_FILE_SAVED);
return ret;
}
}
}

118
src/modules/core/ModuleFileHandler.ts
View File

@@ -1,16 +1,29 @@
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { serialized } from "octagonal-wheels/concurrency/lock";
import type { FileEventItem } from "../../common/types";
import type { FilePath, FilePathWithPrefix, MetaEntry, UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../../lib/src/common/types";
import type {
FilePath,
FilePathWithPrefix,
MetaEntry,
UXFileInfo,
UXFileInfoStub,
UXInternalFileInfoStub,
} from "../../lib/src/common/types";
import { AbstractModule } from "../AbstractModule.ts";
import { compareFileFreshness, EVEN, getPath, getPathFromUXFileInfo, getPathWithoutPrefix, markChangesAreSame } from "../../common/utils";
import {
compareFileFreshness,
EVEN,
getPath,
getPathFromUXFileInfo,
getPathWithoutPrefix,
markChangesAreSame,
} from "../../common/utils";
import { getDocDataAsArray, isDocContentSame, readContent } from "../../lib/src/common/utils";
import { shouldBeIgnored } from "../../lib/src/string_and_binary/path";
import type { ICoreModule } from "../ModuleTypes";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
export class ModuleFileHandler extends AbstractModule implements ICoreModule {
get db() {
return this.core.databaseFileAccess;
}
@@ -31,10 +44,14 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
if (!readFile) {
throw new Error(`File ${file.path} is not exist on the storage`);
}
return readFile
return readFile;
}
async storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force: boolean = false, onlyChunks: boolean = false): Promise<boolean | undefined> {
async storeFileToDB(
info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix,
force: boolean = false,
onlyChunks: boolean = false
): Promise<boolean | undefined> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
if (file == null) {
this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
@@ -42,10 +59,13 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
}
// const file = item.args.file;
if (file.isInternal) {
this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
this._log(
`Internal file ${file.path} is not allowed to be processed on processFileEvent`,
LOG_LEVEL_VERBOSE
);
return false;
}
// First, check the file on the database
// First, check the file on the database
const entry = await this.db.fetchEntry(file, undefined, true, true);
if (!entry || entry.deleted || entry._deleted) {
@@ -114,10 +134,13 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
}
// const file = item.args.file;
if (file.isInternal) {
this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
this._log(
`Internal file ${file.path} is not allowed to be processed on processFileEvent`,
LOG_LEVEL_VERBOSE
);
return false;
}
// First, check the file on the database
// First, check the file on the database
const entry = await this.db.fetchEntry(file, undefined, true, true);
if (!entry || entry.deleted || entry._deleted) {
this._log(`File ${file.path} is not exist or already deleted on the database`, LOG_LEVEL_VERBOSE);
@@ -135,24 +158,39 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
return await this.db.delete(file);
}
async deleteRevisionFromDB(info: UXFileInfoStub | FilePath | FilePathWithPrefix, rev: string): Promise<boolean | undefined> {
async deleteRevisionFromDB(
info: UXFileInfoStub | FilePath | FilePathWithPrefix,
rev: string
): Promise<boolean | undefined> {
//TODO: Possibly check the conflicting.
return await this.db.delete(info, rev);
}
async resolveConflictedByDeletingRevision(info: UXFileInfoStub | FilePath, rev: string): Promise<boolean | undefined> {
if (!await this.deleteRevisionFromDB(info, rev)) {
this._log(`Failed to delete the conflicted revision ${rev} of ${getPathFromUXFileInfo(info)}`, LOG_LEVEL_VERBOSE);
async resolveConflictedByDeletingRevision(
info: UXFileInfoStub | FilePath,
rev: string
): Promise<boolean | undefined> {
if (!(await this.deleteRevisionFromDB(info, rev))) {
this._log(
`Failed to delete the conflicted revision ${rev} of ${getPathFromUXFileInfo(info)}`,
LOG_LEVEL_VERBOSE
);
return false;
}
if (!await this.dbToStorageWithSpecificRev(info, rev, true)) {
this._log(`Failed to apply the resolved revision ${rev} of ${getPathFromUXFileInfo(info)} to the storage`, LOG_LEVEL_VERBOSE);
if (!(await this.dbToStorageWithSpecificRev(info, rev, true))) {
this._log(
`Failed to apply the resolved revision ${rev} of ${getPathFromUXFileInfo(info)} to the storage`,
LOG_LEVEL_VERBOSE
);
return false;
}
}
async dbToStorageWithSpecificRev(info: UXFileInfoStub | UXFileInfo | FilePath | null, rev: string, force?: boolean): Promise<boolean> {
async dbToStorageWithSpecificRev(
info: UXFileInfoStub | UXFileInfo | FilePath | null,
rev: string,
force?: boolean
): Promise<boolean> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
if (file == null) {
this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
@@ -166,12 +204,18 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
return await this.dbToStorage(docEntry, file, force);
}
async dbToStorage(entryInfo: MetaEntry | FilePathWithPrefix, info: UXFileInfoStub | UXFileInfo | FilePath | null, force?: boolean): Promise<boolean> {
async dbToStorage(
entryInfo: MetaEntry | FilePathWithPrefix,
info: UXFileInfoStub | UXFileInfo | FilePath | null,
force?: boolean
): Promise<boolean> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
const mode = file == null ? "create" : "modify";
const docEntry = typeof entryInfo === "string" ?
await this.db.fetchEntryMeta(entryInfo, undefined, true) : await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
const docEntry =
typeof entryInfo === "string"
? await this.db.fetchEntryMeta(entryInfo, undefined, true)
: await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
if (!docEntry) {
this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE);
return false;
@@ -255,7 +299,10 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
}
// Let's apply the changes.
} else {
this._log(`File ${docRead.path} ${existOnStorage ? "(new) " : ""} ${force ? " (forced)" : ""}`, LOG_LEVEL_VERBOSE);
this._log(
`File ${docRead.path} ${existOnStorage ? "(new) " : ""} ${force ? " (forced)" : ""}`,
LOG_LEVEL_VERBOSE
);
}
await this.storage.ensureDir(path);
const ret = await this.storage.writeFileAuto(path, docData, { ctime: docRead.ctime, mtime: docRead.mtime });
@@ -268,7 +315,7 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
const eventItem = item.args;
const type = item.type;
const path = eventItem.file.path;
if (!await this.core.$$isTargetFile(path)) {
if (!(await this.core.$$isTargetFile(path))) {
this._log(`File ${path} is not the target file`, LOG_LEVEL_VERBOSE);
return false;
}
@@ -296,7 +343,7 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
async $anyProcessReplicatedDoc(entry: MetaEntry): Promise<boolean | undefined> {
return await serialized(entry.path, async () => {
if (!await this.core.$$isTargetFile(entry.path)) {
if (!(await this.core.$$isTargetFile(entry.path))) {
this._log(`File ${entry.path} is not the target file`, LOG_LEVEL_VERBOSE);
return false;
}
@@ -312,13 +359,15 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
// Nothing to do and other modules should also nothing to do.
return true;
} else {
this._log(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE);
this._log(
`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`,
LOG_LEVEL_VERBOSE
);
const ret = await this.dbToStorage(entry, targetFile);
this._log(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`);
return ret;
}
});
}
async createAllChunks(showingNotice?: boolean): Promise<void> {
@@ -329,11 +378,16 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
const filesStorageSrc = this.storage.getFiles();
const incProcessed = () => {
processed++;
if (processed % 25 == 0) this._log(`Creating missing chunks: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
}
if (processed % 25 == 0)
this._log(
`Creating missing chunks: ${processed} of ${total} files`,
showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO,
"chunkCreation"
);
};
const total = filesStorageSrc.length;
const procAllChunks = filesStorageSrc.map(async (file) => {
if (!await this.core.$$isTargetFile(file)) {
if (!(await this.core.$$isTargetFile(file))) {
incProcessed();
return true;
}
@@ -352,6 +406,10 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
}
});
await Promise.all(procAllChunks);
this._log(`Creating chunks Done: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
this._log(
`Creating chunks Done: ${processed} of ${total} files`,
showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO,
"chunkCreation"
);
}
}
}

4
src/modules/core/ModuleLocalDatabaseObsidian.ts
View File

@@ -5,7 +5,6 @@ import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleLocalDatabaseObsidian extends AbstractModule implements ICoreModule {
$everyOnloadStart(): Promise<boolean> {
return Promise.resolve(true);
}
@@ -23,5 +22,4 @@ export class ModuleLocalDatabaseObsidian extends AbstractModule implements ICore
$$isDatabaseReady(): boolean {
return this.localDatabase != null && this.localDatabase.isReady;
}
}
}

10
src/modules/core/ModulePeriodicProcess.ts
View File

@@ -3,7 +3,6 @@ import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
export class ModulePeriodicProcess extends AbstractModule implements ICoreModule {
periodicSyncProcessor = new PeriodicProcessor(this.core, async () => await this.core.$$replicate());
_disablePeriodic() {
@@ -11,20 +10,19 @@ export class ModulePeriodicProcess extends AbstractModule implements ICoreModule
return Promise.resolve(true);
}
_resumePeriodic() {
this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0);
this.periodicSyncProcessor.enable(
this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0
);
return Promise.resolve(true);
}
$allOnUnload() {
return this._disablePeriodic();
}
$everyBeforeRealizeSetting(): Promise<boolean> {
return this._disablePeriodic();
}
$everyBeforeSuspendProcess(): Promise<boolean> {
return this._disablePeriodic();
}
$everyAfterResumeProcess(): Promise<boolean> {
return this._resumePeriodic();
@@ -32,4 +30,4 @@ export class ModulePeriodicProcess extends AbstractModule implements ICoreModule
$everyAfterRealizeSetting(): Promise<boolean> {
return this._resumePeriodic();
}
}
}

7
src/modules/core/ModulePouchDB.ts
View File

@@ -3,7 +3,10 @@ import type { ICoreModule } from "../ModuleTypes";
import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
export class ModulePouchDB extends AbstractModule implements ICoreModule {
$$createPouchDBInstance<T extends object>(name?: string, options?: PouchDB.Configuration.DatabaseConfiguration): PouchDB.Database<T> {
$$createPouchDBInstance<T extends object>(
name?: string,
options?: PouchDB.Configuration.DatabaseConfiguration
): PouchDB.Database<T> {
const optionPass = options ?? {};
if (this.settings.useIndexedDBAdapter) {
optionPass.adapter = "indexeddb";
@@ -13,4 +16,4 @@ export class ModulePouchDB extends AbstractModule implements ICoreModule {
}
return new PouchDB(name, optionPass);
}
}
}

64
src/modules/core/ModuleRebuilder.ts
View File

@@ -1,5 +1,12 @@
import { delay } from "octagonal-wheels/promises";
import { FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, REMOTE_COUCHDB, REMOTE_MINIO } from "../../lib/src/common/types.ts";
import {
FLAGMD_REDFLAG2_HR,
FLAGMD_REDFLAG3_HR,
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
REMOTE_COUCHDB,
REMOTE_MINIO,
} from "../../lib/src/common/types.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { Rebuilder } from "../interfaces/DatabaseRebuilder.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
@@ -7,12 +14,13 @@ import type { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchd
import { fetchAllUsedChunks } from "../../lib/src/pouchdb/utils_couchdb.ts";
export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebuilder {
$everyOnload(): Promise<boolean> {
this.core.rebuilder = this;
return Promise.resolve(true);
}
async $performRebuildDB(method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"): Promise<void> {
async $performRebuildDB(
method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"
): Promise<void> {
if (method == "localOnly") {
await this.$fetchLocal();
}
@@ -27,11 +35,13 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
}
}
async askUsingOptionalFeature(opt: {
enableFetch?: boolean;
enableOverwrite?: boolean;
}) {
if (await this.core.confirm.askYesNoDialog("Do you want to enable extra features? If you are new to Self-hosted LiveSync, try the core feature first!", { title: "Enable extra features", defaultOption: "No", timeout: 15 }) == "yes") {
async askUsingOptionalFeature(opt: { enableFetch?: boolean; enableOverwrite?: boolean }) {
if (
(await this.core.confirm.askYesNoDialog(
"Do you want to enable extra features? If you are new to Self-hosted LiveSync, try the core feature first!",
{ title: "Enable extra features", defaultOption: "No", timeout: 15 }
)) == "yes"
) {
await this.core.$allAskUsingOptionalSyncFeature(opt);
}
}
@@ -55,7 +65,6 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
return this.rebuildRemote();
}
async rebuildEverything() {
await this.core.$allSuspendExtraSync();
await this.askUseNewAdapter();
@@ -73,7 +82,6 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
await this.core.$$replicateAllToServer(true);
await delay(1000);
await this.core.$$replicateAllToServer(true, true);
}
$rebuildEverything(): Promise<void> {
@@ -116,7 +124,6 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
await this.localDatabase.resetDatabase();
}
async suspendAllSync() {
this.core.settings.liveSync = false;
this.core.settings.periodicReplication = false;
@@ -130,7 +137,10 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
async suspendReflectingDatabase() {
if (this.core.settings.doNotSuspendOnFetching) return;
if (this.core.settings.remoteType == REMOTE_MINIO) return;
this._log(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
this._log(
`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`,
LOG_LEVEL_NOTICE
);
this.core.settings.suspendParseReplicationResult = true;
this.core.settings.suspendFileWatching = true;
await this.core.saveSettings();
@@ -144,7 +154,6 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
await this.core.$$performFullScan(true);
await this.core.$everyBeforeReplicate(false); //TODO: Check actual need of this.
await this.core.saveSettings();
}
async askUseNewAdapter() {
if (!this.core.settings.useIndexedDBAdapter) {
@@ -153,7 +162,13 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
const CHOICE_NO = "No, keep compatibility";
const choices = [CHOICE_YES, CHOICE_NO];
const ret = await this.core.confirm.confirmWithMessage("Database adapter", message, choices, CHOICE_YES, 10);
const ret = await this.core.confirm.confirmWithMessage(
"Database adapter",
message,
choices,
CHOICE_YES,
10
);
if (ret == CHOICE_YES) {
this.core.settings.useIndexedDBAdapter = true;
}
@@ -200,10 +215,18 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
await this.core.$$resetLocalDatabase();
}
async fetchRemoteChunks() {
if (!this.core.settings.doNotSuspendOnFetching && this.core.settings.readChunksOnline && this.core.settings.remoteType == REMOTE_COUCHDB) {
if (
!this.core.settings.doNotSuspendOnFetching &&
this.core.settings.readChunksOnline &&
this.core.settings.remoteType == REMOTE_COUCHDB
) {
this._log(`Fetching chunks`, LOG_LEVEL_NOTICE);
const replicator = this.core.$$getReplicator() as LiveSyncCouchDBReplicator;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.$$isMobile(), true);
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(
this.settings,
this.core.$$isMobile(),
true
);
if (typeof remoteDB == "string") {
this._log(remoteDB, LOG_LEVEL_NOTICE);
} else {
@@ -218,9 +241,14 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
let i = 0;
for (const file of files) {
if (i++ % 10) this._log(`Check and Processing ${i} / ${files.length}`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
if (i++ % 10)
this._log(
`Check and Processing ${i} / ${files.length}`,
LOG_LEVEL_NOTICE,
"resolveAllConflictedFilesByNewerOnes"
);
await this.core.$anyResolveConflictByNewest(file);
}
this._log(`Done!`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
}
}
}

353
src/modules/core/ModuleReplicator.ts
View File

@@ -8,7 +8,14 @@ import { purgeUnreferencedChunks, balanceChunkPurgedDBs } from "../../lib/src/po
import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
import { throttle } from "octagonal-wheels/function";
import { arrayToChunkedArray } from "octagonal-wheels/collection";
import { SYNCINFO_ID, VER, type EntryBody, type EntryDoc, type LoadedEntry, type MetaEntry } from "../../lib/src/common/types";
import {
SYNCINFO_ID,
VER,
type EntryBody,
type EntryDoc,
type LoadedEntry,
type MetaEntry,
} from "../../lib/src/common/types";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import { getPath, isChunk, isValidPath, scheduleTask } from "../../common/utils";
import { sendValue } from "octagonal-wheels/messagepassing/signal";
@@ -17,13 +24,12 @@ import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
export class ModuleReplicator extends AbstractModule implements ICoreModule {
$everyOnloadAfterLoadSettings(): Promise<boolean> {
eventHub.onEvent(EVENT_FILE_SAVED, () => {
if (this.settings.syncOnSave && !this.core.$$isSuspended()) {
scheduleTask("perform-replicate-after-save", 250, () => this.core.$$waitForReplicationOnce());
}
})
});
return Promise.resolve(true);
}
@@ -31,7 +37,7 @@ export class ModuleReplicator extends AbstractModule implements ICoreModule {
const replicator = await this.core.$anyNewReplicator();
if (!replicator) {
this._log("No replicator is available, this is the fatal error.", LOG_LEVEL_NOTICE);
return false
return false;
}
this.core.replicator = replicator;
await yieldMicrotask();
@@ -48,7 +54,6 @@ export class ModuleReplicator extends AbstractModule implements ICoreModule {
$everyOnResetDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.setReplicator();
}
async $everyBeforeReplicate(showMessage: boolean): Promise<boolean> {
@@ -56,7 +61,7 @@ export class ModuleReplicator extends AbstractModule implements ICoreModule {
return true;
}
async $$replicate(showMessage: boolean = false): Promise<boolean | void> {
//--?
//--?
if (!this.core.$$isReady()) return;
if (isLockAcquired("cleanup")) {
Logger("Database cleaning up is in process. replication has been cancelled", LOG_LEVEL_NOTICE);
@@ -66,11 +71,11 @@ export class ModuleReplicator extends AbstractModule implements ICoreModule {
Logger("Open settings and check message, please. replication has been cancelled.", LOG_LEVEL_NOTICE);
return;
}
if (!await this.core.$everyCommitPendingFileEvent()) {
if (!(await this.core.$everyCommitPendingFileEvent())) {
Logger("Some file events are pending. Replication has been cancelled.", LOG_LEVEL_NOTICE);
return false;
}
if (!await this.core.$everyBeforeReplicate(showMessage)) {
if (!(await this.core.$everyBeforeReplicate(showMessage))) {
Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
return false;
}
@@ -80,32 +85,41 @@ export class ModuleReplicator extends AbstractModule implements ICoreModule {
if (!ret) {
if (this.core.replicator.tweakSettingsMismatched) {
await this.core.$$askResolvingMismatchedTweaks();
} else {
if (this.core.replicator?.remoteLockedAndDeviceNotAccepted) {
if (this.core.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) {
Logger(`The remote database has been cleaned.`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
Logger(
`The remote database has been cleaned.`,
showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO
);
await skipIfDuplicated("cleanup", async () => {
const count = await purgeUnreferencedChunks(this.localDatabase.localDatabase, true);
const message = `The remote database has been cleaned up.
To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
However, If there are many chunks to be deleted, maybe fetching again is faster.
We will lose the history of this device if we fetch the remote database again.
Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`
Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`;
const CHOICE_FETCH = "Fetch again";
const CHOICE_CLEAN = "Cleanup";
const CHOICE_DISMISS = "Dismiss";
const ret = await this.core.confirm.confirmWithMessage("Cleaned", message, [
CHOICE_FETCH,
CHOICE_CLEAN,
CHOICE_DISMISS], CHOICE_DISMISS, 30);
const ret = await this.core.confirm.confirmWithMessage(
"Cleaned",
message,
[CHOICE_FETCH, CHOICE_CLEAN, CHOICE_DISMISS],
CHOICE_DISMISS,
30
);
if (ret == CHOICE_FETCH) {
await this.core.rebuilder.$performRebuildDB("localOnly");
}
if (ret == CHOICE_CLEAN) {
const replicator = this.core.$$getReplicator();
if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.$$isMobile(), true);
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(
this.settings,
this.core.$$isMobile(),
true
);
if (typeof remoteDB == "string") {
Logger(remoteDB, LOG_LEVEL_NOTICE);
return false;
@@ -114,16 +128,23 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
// Perform the synchronisation once.
if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) {
if (
await this.core.replicator.openReplication(this.settings, false, showMessage, true)
) {
await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
await this.core.$$getReplicator().markRemoteResolved(this.settings);
Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
Logger(
"The local database has been cleaned up.",
showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO
);
} else {
Logger("Replication has been cancelled. Please try it again.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
Logger(
"Replication has been cancelled. Please try it again.",
showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO
);
}
}
});
} else {
@@ -131,23 +152,30 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
The remote database has been rebuilt.
To synchronize, this device must fetch everything again once.
Or if you are sure know what had been happened, we can unlock the database from the setting dialog.
`
`;
const CHOICE_FETCH = "Fetch again";
const CHOICE_DISMISS = "Dismiss";
const ret = await this.core.confirm.confirmWithMessage("Locked", message, [
CHOICE_FETCH,
CHOICE_DISMISS], CHOICE_DISMISS, 10);
const ret = await this.core.confirm.confirmWithMessage(
"Locked",
message,
[CHOICE_FETCH, CHOICE_DISMISS],
CHOICE_DISMISS,
10
);
if (ret == CHOICE_FETCH) {
const CHOICE_RESTART = "Restart";
const CHOICE_WITHOUT_RESTART = "Without restart";
if (await this.core.confirm.askSelectStringDialogue(
"Self-hosted LiveSync restarts Obsidian to fetch everything safely. However, you can do it without restarting. Please choose one.",
[CHOICE_RESTART, CHOICE_WITHOUT_RESTART], {
title: "Fetch again",
defaultAction: CHOICE_RESTART,
timeout: 30,
}
) == CHOICE_RESTART) {
if (
(await this.core.confirm.askSelectStringDialogue(
"Self-hosted LiveSync restarts Obsidian to fetch everything safely. However, you can do it without restarting. Please choose one.",
[CHOICE_RESTART, CHOICE_WITHOUT_RESTART],
{
title: "Fetch again",
defaultAction: CHOICE_RESTART,
timeout: 30,
}
)) == CHOICE_RESTART
) {
await this.core.rebuilder.scheduleFetch();
// await this.core.$$scheduleAppReload();
return;
@@ -165,16 +193,18 @@ Or if you are sure know what had been happened, we can unlock the database from
$$parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): void {
if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.suspend()
this.replicationResultProcessor.suspend();
}
this.replicationResultProcessor.enqueueAll(docs);
if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume()
this.replicationResultProcessor.resume();
}
}
_saveQueuedFiles = throttle(() => {
const saveData = this.replicationResultProcessor._queue.filter(e => e !== undefined && e !== null).map((e) => e?._id ?? "" as string) as string[];
const kvDBKey = "queued-files"
const saveData = this.replicationResultProcessor._queue
.filter((e) => e !== undefined && e !== null)
.map((e) => e?._id ?? ("" as string)) as string[];
const kvDBKey = "queued-files";
// localStorage.setItem(lsKey, saveData);
fireAndForget(() => this.core.kvDB.set(kvDBKey, saveData));
}, 100);
@@ -184,19 +214,19 @@ Or if you are sure know what had been happened, we can unlock the database from
async loadQueuedFiles() {
if (this.settings.suspendParseReplicationResult) return;
if (!this.settings.isConfigured) return;
const kvDBKey = "queued-files"
const kvDBKey = "queued-files";
// const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
const ids = [...new Set(await this.core.kvDB.get<string[]>(kvDBKey) ?? [])];
const ids = [...new Set((await this.core.kvDB.get<string[]>(kvDBKey)) ?? [])];
const batchSize = 100;
const chunkedIds = arrayToChunkedArray(ids, batchSize);
for await (const idsBatch of chunkedIds) {
const ret = await this.localDatabase.allDocsRaw<EntryDoc>({
keys: idsBatch,
include_docs: true,
limit: 100
limit: 100,
});
const docs = ret.rows.filter(e => e.doc).map(e => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
const errors = ret.rows.filter(e => !e.doc && !e.value.deleted);
const docs = ret.rows.filter((e) => e.doc).map((e) => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
const errors = ret.rows.filter((e) => !e.doc && !e.value.deleted);
if (errors.length > 0) {
Logger("Some queued processes were not resurrected");
Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE);
@@ -204,131 +234,163 @@ Or if you are sure know what had been happened, we can unlock the database from
this.replicationResultProcessor.enqueueAll(docs);
await this.replicationResultProcessor.waitForAllProcessed();
}
}
replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
if (this.settings.suspendParseReplicationResult) return;
const change = docs[0];
if (!change) return;
if (isChunk(change._id)) {
// SendSignal?
// this.parseIncomingChunk(change);
sendValue(`leaf-${change._id}`, change);
return;
}
if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;
// any addon needs this item?
// for (const proc of this.core.addOns) {
// if (await proc.parseReplicationResultItem(change)) {
// return;
// }
// }
if (change.type == "versioninfo") {
if (change.version > VER) {
this.core.replicator.closeReplication();
Logger(`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`, LOG_LEVEL_NOTICE);
}
return;
}
if (change._id == SYNCINFO_ID || // Synchronisation information data
change._id.startsWith("_design") //design document
) {
return;
}
if (isAnyNote(change)) {
const docPath = getPath(change);
if (!await this.core.$$isTargetFile(docPath)) {
Logger(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
replicationResultProcessor = new QueueProcessor(
async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
if (this.settings.suspendParseReplicationResult) return;
const change = docs[0];
if (!change) return;
if (isChunk(change._id)) {
// SendSignal?
// this.parseIncomingChunk(change);
sendValue(`leaf-${change._id}`, change);
return;
}
if (this.databaseQueuedProcessor._isSuspended) {
Logger(`Processing scheduled: ${docPath}`, LOG_LEVEL_INFO);
}
const size = change.size;
if (this.core.$$isFileSizeExceeded(size)) {
Logger(`Processing ${docPath} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;
// any addon needs this item?
// for (const proc of this.core.addOns) {
// if (await proc.parseReplicationResultItem(change)) {
// return;
// }
// }
if (change.type == "versioninfo") {
if (change.version > VER) {
this.core.replicator.closeReplication();
Logger(
`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
LOG_LEVEL_NOTICE
);
}
return;
}
this.databaseQueuedProcessor.enqueue(change);
}
return;
}, {
batchSize: 1,
suspended: true,
concurrentLimit: 100,
delay: 0,
totalRemainingReactiveSource: this.core.replicationResultCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline().onUpdateProgress(() => {
this.saveQueuedFiles();
});
databaseQueuedProcessor = new QueueProcessor(async (docs: EntryBody[]) => {
const dbDoc = docs[0] as LoadedEntry; // It has no `data`
const path = getPath(dbDoc);
// If `Read chunks online` is disabled, chunks should be transferred before here.
// However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
const doc = await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, {}, false, true, true);
if (!doc) {
Logger(`Something went wrong while gathering content of ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `, LOG_LEVEL_NOTICE)
if (
change._id == SYNCINFO_ID || // Synchronisation information data
change._id.startsWith("_design") //design document
) {
return;
}
if (isAnyNote(change)) {
const docPath = getPath(change);
if (!(await this.core.$$isTargetFile(docPath))) {
Logger(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
return;
}
if (this.databaseQueuedProcessor._isSuspended) {
Logger(`Processing scheduled: ${docPath}`, LOG_LEVEL_INFO);
}
const size = change.size;
if (this.core.$$isFileSizeExceeded(size)) {
Logger(
`Processing ${docPath} has been skipped due to file size exceeding the limit`,
LOG_LEVEL_NOTICE
);
return;
}
this.databaseQueuedProcessor.enqueue(change);
}
return;
},
{
batchSize: 1,
suspended: true,
concurrentLimit: 100,
delay: 0,
totalRemainingReactiveSource: this.core.replicationResultCount,
}
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline()
.onUpdateProgress(() => {
this.saveQueuedFiles();
});
if (await this.core.$anyProcessOptionalSyncFiles(dbDoc)) {
// Already processed
} else if (isValidPath(getPath(doc))) {
this.storageApplyingProcessor.enqueue(doc as MetaEntry);
} else {
Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE);
databaseQueuedProcessor = new QueueProcessor(
async (docs: EntryBody[]) => {
const dbDoc = docs[0] as LoadedEntry; // It has no `data`
const path = getPath(dbDoc);
// If `Read chunks online` is disabled, chunks should be transferred before here.
// However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
const doc = await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, {}, false, true, true);
if (!doc) {
Logger(
`Something went wrong while gathering content of ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `,
LOG_LEVEL_NOTICE
);
return;
}
if (await this.core.$anyProcessOptionalSyncFiles(dbDoc)) {
// Already processed
} else if (isValidPath(getPath(doc))) {
this.storageApplyingProcessor.enqueue(doc as MetaEntry);
} else {
Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE);
}
return;
},
{
suspended: true,
batchSize: 1,
concurrentLimit: 10,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.databaseQueueCount,
}
return;
}, {
suspended: true,
batchSize: 1,
concurrentLimit: 10,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.databaseQueueCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline();
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline();
storageApplyingProcessor = new QueueProcessor(async (docs: MetaEntry[]) => {
const entry = docs[0];
await this.core.$anyProcessReplicatedDoc(entry);
return;
}, {
suspended: true,
batchSize: 1,
concurrentLimit: 6,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.storageApplyingCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline()
storageApplyingProcessor = new QueueProcessor(
async (docs: MetaEntry[]) => {
const entry = docs[0];
await this.core.$anyProcessReplicatedDoc(entry);
return;
},
{
suspended: true,
batchSize: 1,
concurrentLimit: 6,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.storageApplyingCount,
}
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline();
$everyBeforeSuspendProcess(): Promise<boolean> {
this.core.replicator.closeReplication();
return Promise.resolve(true);
}
async $$replicateAllToServer(showingNotice: boolean = false, sendChunksInBulkDisabled: boolean = false): Promise<boolean> {
async $$replicateAllToServer(
showingNotice: boolean = false,
sendChunksInBulkDisabled: boolean = false
): Promise<boolean> {
if (!this.core.$$isReady()) return false;
if (!await this.core.$everyBeforeReplicate(showingNotice)) {
if (!(await this.core.$everyBeforeReplicate(showingNotice))) {
Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
return false;
}
if (!sendChunksInBulkDisabled) {
if (this.core.replicator instanceof LiveSyncCouchDBReplicator) {
if (await this.core.confirm.askYesNoDialog("Do you want to send all chunks before replication?", { defaultOption: "No", timeout: 20 }) == "yes") {
if (
(await this.core.confirm.askYesNoDialog("Do you want to send all chunks before replication?", {
defaultOption: "No",
timeout: 20,
})) == "yes"
) {
await this.core.replicator.sendChunks(this.core.settings, undefined, true, 0);
}
}
@@ -351,7 +413,4 @@ Or if you are sure know what had been happened, we can unlock the database from
async $$waitForReplicationOnce(): Promise<boolean | void> {
return await shareRunningResult(`replication`, () => this.core.$$replicate());
}
}
}

5
src/modules/core/ModuleReplicatorCouchDB.ts
View File

@@ -10,10 +10,9 @@ export class ModuleReplicatorCouchDB extends AbstractModule implements ICoreModu
const settings = { ...this.settings, ...settingOverride };
// If new remote types were added, add them here. Do not use `REMOTE_COUCHDB` directly for the safety valve.
if (settings.remoteType == REMOTE_MINIO) {
return undefined!
return undefined!;
}
return Promise.resolve(new LiveSyncCouchDBReplicator(this.core));
}
$everyAfterResumeProcess(): Promise<boolean> {
if (this.settings.remoteType != REMOTE_MINIO) {
@@ -30,4 +29,4 @@ export class ModuleReplicatorCouchDB extends AbstractModule implements ICoreModu
return Promise.resolve(true);
}
}
}

4
src/modules/core/ModuleReplicatorMinIO.ts
View File

@@ -12,6 +12,4 @@ export class ModuleReplicatorMinIO extends AbstractModule implements ICoreModule
}
return undefined!;
}
}
}

104
src/modules/core/ModuleTargetFilter.ts
View File

@@ -1,16 +1,29 @@
import { LRUCache } from "octagonal-wheels/memory/LRUCache";
import { getPathFromUXFileInfo, id2path, isInternalMetadata, path2id, stripInternalMetadataPrefix, useMemo } from "../../common/utils";
import { LOG_LEVEL_VERBOSE, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type ObsidianLiveSyncSettings, type UXFileInfoStub } from "../../lib/src/common/types";
import {
getPathFromUXFileInfo,
id2path,
isInternalMetadata,
path2id,
stripInternalMetadataPrefix,
useMemo,
} from "../../common/utils";
import {
LOG_LEVEL_VERBOSE,
type DocumentID,
type EntryHasPath,
type FilePath,
type FilePathWithPrefix,
type ObsidianLiveSyncSettings,
type UXFileInfoStub,
} from "../../lib/src/common/types";
import { addPrefix, isAcceptedAll, stripAllPrefixes } from "../../lib/src/string_and_binary/path";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
import { EVENT_REQUEST_RELOAD_SETTING_TAB, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import { isDirty } from "../../lib/src/common/utils";
export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
reloadIgnoreFiles() {
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
this.ignoreFiles = this.settings.ignoreFiles.split(",").map((e) => e.trim());
}
$everyOnload(): Promise<boolean> {
eventHub.onEvent(EVENT_SETTING_SAVED, (evt: ObsidianLiveSyncSettings) => {
@@ -32,10 +45,13 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
}
async $$path2id(filename: FilePathWithPrefix | FilePath, prefix?: string): Promise<DocumentID> {
const destPath = addPrefix(filename, prefix ?? "");
return await path2id(destPath, this.settings.usePathObfuscation ? this.settings.passphrase : "", !this.settings.handleFilenameCaseSensitive);
return await path2id(
destPath,
this.settings.usePathObfuscation ? this.settings.passphrase : "",
!this.settings.handleFilenameCaseSensitive
);
}
$$isFileSizeExceeded(size: number) {
if (this.settings.syncMaxSizeInMB > 0 && size > 0) {
if (this.settings.syncMaxSizeInMB * 1024 * 1024 < size) {
@@ -45,7 +61,6 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
return false;
}
$$markFileListPossiblyChanged(): void {
this.totalFileEventCount++;
}
@@ -58,38 +73,39 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
}
async $$isTargetFile(file: string | UXFileInfoStub, keepFileCheckList = false) {
const fileCount = useMemo<Record<string, number>>({
key: "fileCount", // forceUpdate: !keepFileCheckList,
}, (ctx, prev) => {
if (keepFileCheckList && prev) return prev;
if (!keepFileCheckList && prev && !this.fileListPossiblyChanged) {
return prev;
}
const fileList = (ctx.get("fileList") ?? []) as FilePathWithPrefix[];
// const fileNameList = (ctx.get("fileNameList") ?? []) as FilePath[];
// const fileNames =
const vaultFiles = this.core.storageAccess.getFileNames().sort();
if (prev && vaultFiles.length == fileList.length) {
const fl3 = new Set([...fileList, ...vaultFiles]);
if (fileList.length == fl3.size && vaultFiles.length == fl3.size) {
const fileCount = useMemo<Record<string, number>>(
{
key: "fileCount", // forceUpdate: !keepFileCheckList,
},
(ctx, prev) => {
if (keepFileCheckList && prev) return prev;
if (!keepFileCheckList && prev && !this.fileListPossiblyChanged) {
return prev;
}
}
ctx.set("fileList", vaultFiles);
const fileCount: Record<string, number> = {};
for (const file of vaultFiles) {
const lc = file.toLowerCase();
if (!fileCount[lc]) {
fileCount[lc] = 1;
} else {
fileCount[lc]++;
const fileList = (ctx.get("fileList") ?? []) as FilePathWithPrefix[];
// const fileNameList = (ctx.get("fileNameList") ?? []) as FilePath[];
// const fileNames =
const vaultFiles = this.core.storageAccess.getFileNames().sort();
if (prev && vaultFiles.length == fileList.length) {
const fl3 = new Set([...fileList, ...vaultFiles]);
if (fileList.length == fl3.size && vaultFiles.length == fl3.size) {
return prev;
}
}
ctx.set("fileList", vaultFiles);
const fileCount: Record<string, number> = {};
for (const file of vaultFiles) {
const lc = file.toLowerCase();
if (!fileCount[lc]) {
fileCount[lc] = 1;
} else {
fileCount[lc]++;
}
}
return fileCount;
}
return fileCount;
})
);
const filepath = getPathFromUXFileInfo(file);
const lc = filepath.toLowerCase();
@@ -100,7 +116,7 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
}
const fileNameLC = getPathFromUXFileInfo(file).split("/").pop()?.toLowerCase();
if (this.settings.useIgnoreFiles) {
if (this.ignoreFiles.some(e => e.toLowerCase() == fileNameLC)) {
if (this.ignoreFiles.some((e) => e.toLowerCase() == fileNameLC)) {
// We must reload ignore files due to the its change.
await this.readIgnoreFile(filepath);
}
@@ -109,11 +125,11 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
}
}
if (!this.localDatabase?.isTargetFile(filepath)) return false;
return true
return true;
}
ignoreFileCache = new LRUCache<string, string[] | false>(300, 250000, true);
ignoreFiles = [] as string[]
ignoreFiles = [] as string[];
async readIgnoreFile(path: string) {
try {
const file = await this.core.storageAccess.readFileText(path);
@@ -138,14 +154,18 @@ export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
if (!this.settings.useIgnoreFiles) {
return false;
}
const filepath = getPathFromUXFileInfo(file)
const filepath = getPathFromUXFileInfo(file);
if (this.ignoreFileCache.has(filepath)) {
// Renew
await this.readIgnoreFile(filepath);
}
if (!await isAcceptedAll(stripAllPrefixes(filepath), this.ignoreFiles, (filename) => this.getIgnoreFile(filename))) {
if (
!(await isAcceptedAll(stripAllPrefixes(filepath), this.ignoreFiles, (filename) =>
this.getIgnoreFile(filename)
))
) {
return true;
}
return false;
}
}
}