diff --git a/manifest-beta.json b/manifest-beta.json index 83884de..0754a4b 100644 --- a/manifest-beta.json +++ b/manifest-beta.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "0.24.0.dev-rc4", + "version": "0.24.0.dev-rc5", "minAppVersion": "0.9.12", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index d46f8d3..5cdffb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "0.24.0.dev-rc4", + "version": "0.24.0.dev-rc5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "0.24.0.dev-rc4", + "version": "0.24.0.dev-rc5", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.645.0", diff --git a/package.json b/package.json index 30b229a..10a4edf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "0.24.0.dev-rc4", + "version": "0.24.0.dev-rc5", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 010570c..62acee4 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -5,7 +5,6 @@ import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, File import { CANCELLED, LEAVE_TO_SUBSEQUENT, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_SHINY } from "../../lib/src/common/types.ts"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "../../common/types.ts"; import { createBlob, createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, getDocDataAsArray, isDocContentSame, isLoadedEntry, isObjectDifferent } from "../../lib/src/common/utils.ts"; -import { Logger } from "../../lib/src/common/logger.ts"; import { digestHash } from "../../lib/src/string_and_binary/hash.ts"; import { arrayBufferToBase64, decodeBinary, readString } from '../../lib/src/string_and_binary/convert.ts'; import { serialized, shareRunningResult } from "../../lib/src/concurrency/lock.ts"; @@ -419,12 +418,12 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { async $everyOnDatabaseInitialized(showNotice: boolean) { if (!this._isThisModuleEnabled()) return true; try { - Logger("Scanning customizations..."); + this._log("Scanning customizations..."); await this.scanAllConfigFiles(showNotice); - Logger("Scanning customizations : done"); + this._log("Scanning customizations : done"); } catch (ex) { - Logger("Scanning customizations : failed"); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Scanning customizations : failed"); + this._log(ex, LOG_LEVEL_VERBOSE); } return true; } @@ -477,7 +476,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { xFiles.push(work); } if (missingHash) { - Logger(`Digest created for ${path} to improve checking`, LOG_LEVEL_VERBOSE); + this._log(`Digest created for ${path} to improve checking`, LOG_LEVEL_VERBOSE); wx.data = serialize(data); fireAndForget(() => this.localDatabase.putDBEntry(createSavingEntryFromLoadedEntry(wx))); } @@ -512,8 +511,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { return []; } catch (ex) { - Logger(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); } return []; }, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline(); @@ -536,8 +535,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { return []; } catch (ex) { - Logger(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); } return []; }, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline(); @@ -583,11 +582,11 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { if (!loaded) { const d = await this.localDatabase.getDBEntry(unifiedPathV2); if (!d) { - Logger(`The file ${unifiedPathV2} is not found`, LOG_LEVEL_VERBOSE); + this._log(`The file ${unifiedPathV2} is not found`, LOG_LEVEL_VERBOSE); return false; } if (!isLoadedEntry(d)) { - Logger(`The file ${unifiedPathV2} is not a note`, LOG_LEVEL_VERBOSE); + this._log(`The file ${unifiedPathV2} is not a note`, LOG_LEVEL_VERBOSE); return false; } loaded = d; @@ -613,8 +612,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { this.pluginList.filter(e => e instanceof PluginDataExDisplayV2 && e.confKey == confKey).forEach(e => (e as PluginDataExDisplayV2).applyLoadedManifest()); pluginList.set(this.pluginList); } catch (ex) { - Logger(`The file ${loaded.path} seems to manifest, but could not be decoded as JSON`, LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`The file ${loaded.path} seems to manifest, but could not be decoded as JSON`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); } this.loadedManifest_mTime.set(confKey, file.mtime); } else { @@ -687,22 +686,22 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { async migrateV1ToV2(showMessage: boolean, entry: AnyEntry): Promise { const v1Path = entry.path; - Logger(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO); + this._log(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO); if (entry.deleted) { - Logger(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE); + this._log(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE); return; } if (!v1Path.endsWith(".md") && !v1Path.startsWith(ICXHeader)) { - Logger(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE); + this._log(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE); return } if (v1Path.indexOf("%") !== -1) { - Logger(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE); + this._log(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE); return; } const loadedEntry = await this.localDatabase.getDBEntry(v1Path); if (!loadedEntry) { - Logger(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE); + this._log(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE); return; } @@ -723,7 +722,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { const relativeFilename = f.filename.split("/").slice(deletePrefixCount).join("/"); const v2Path = (prefixPath + relativeFilename) as FilePathWithPrefix; // console.warn(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`); - Logger(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE); + this._log(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE); const newId = await this.plugin.$$path2id(v2Path); // const buf = @@ -742,12 +741,12 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } const r = await this.plugin.localDatabase.putDBEntry(saving); if (r && r.ok) { - Logger(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO); + this._log(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO); const delR = await this.deleteConfigOnDatabase(v1Path); if (delR) { - Logger(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO); + this._log(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO); } else { - Logger(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE); + this._log(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE); } } } @@ -802,9 +801,9 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } const fileA = await loadFile(dataA); const fileB = await loadFile(dataB); - Logger(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE); + this._log(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE); if (!fileA || !fileB) { - Logger(`Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, LOG_LEVEL_NOTICE); + this._log(`Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, LOG_LEVEL_NOTICE); return false; } let path = stripAllPrefixes(fileA.path.split("/").slice(-1).join("/") as FilePath); // TODO:adjust @@ -813,15 +812,15 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } if (fileA.path.endsWith(".json")) { return serialized("config:merge-data", () => new Promise((res) => { - Logger("Opening data-merging dialog", LOG_LEVEL_VERBOSE); + this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE); // const docs = [docA, docB]; const modal = new JsonResolveModal(this.app, path, [fileA, fileB], async (keep, result) => { if (result == null) return res(false); try { res(await this.applyData(dataA, result)); } catch (ex) { - Logger("Could not apply merged file"); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Could not apply merged file"); + this._log(ex, LOG_LEVEL_VERBOSE); res(false); } }, "Local", `${dataB.term}`, "B", true, true, "Difference between local and remote"); @@ -847,7 +846,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { right: { rev: "B", ...fileB, data: docBData }, diff: diff } - console.dir(diffResult); + // console.dir(diffResult); const d = new ConflictResolveModal(this.app, path, diffResult, true, dataB.term); d.open(); const ret = await d.waitForResult(); @@ -866,7 +865,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { if (content) { // const dt = createBlob(content); const filename = data.files[0].filename; - Logger(`Applying ${filename} of ${data.displayName || data.name}..`); + this._log(`Applying ${filename} of ${data.displayName || data.name}..`); const path = `${baseDir}/${filename}` as FilePath; await this.plugin.storageAccess.ensureDir(path); // If the content has applied, modified time will be updated to the current time. @@ -879,7 +878,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { // If files have applied, modified time will be updated to the current time. const stat = { mtime: f.mtime, ctime: f.ctime }; const path = `${baseDir}/${f.filename}` as FilePath; - Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`); + this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); // const contentEach = createBlob(f.data); await this.plugin.storageAccess.ensureDir(path); @@ -888,13 +887,13 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { try { oldData = await this.plugin.storageAccess.readHiddenFileBinary(path); } catch (ex) { - Logger(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); oldData = new ArrayBuffer(0); } const content = base64ToArrayBuffer(f.data); if (await isDocContentSame(oldData, content)) { - Logger(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); + this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); continue; } await this.plugin.storageAccess.writeHiddenFileAuto(path, content, stat); @@ -903,30 +902,30 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { try { oldData = await this.plugin.storageAccess.readHiddenFileText(path); } catch (ex) { - Logger(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); oldData = ""; } const content = getDocData(f.data); if (await isDocContentSame(oldData, content)) { - Logger(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); + this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); continue; } await this.plugin.storageAccess.writeHiddenFileAuto(path, content, stat); } - Logger(`Applied ${f.filename} of ${data.displayName || data.name}..`); + this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`); await this.storeCustomisationFileV2(path, this.plugin.deviceAndVaultName); } } } catch (ex) { - Logger(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } return true; } async applyData(data: IPluginDataExDisplay, content?: string): Promise { - Logger(`Applying ${data.displayName || data.name + this._log(`Applying ${data.displayName || data.name }..`); if (data instanceof PluginDataExDisplayV2) { @@ -941,7 +940,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } const loadedData = deserialize(getDocDataAsArray(dx.data), {}) as PluginDataEx; for (const f of loadedData.files) { - Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`); + this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); try { // console.dir(f); const path = `${baseDir}/${f.filename}`; @@ -952,11 +951,11 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } else { await this.plugin.storageAccess.writeHiddenFileAuto(path, content); } - Logger(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`); + this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`); } catch (ex) { - Logger(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); } } @@ -964,7 +963,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { await this.storeCustomizationFiles(uPath); await this.updatePluginList(true, uPath); await delay(100); - Logger(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE); + this._log(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE); if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") { //@ts-ignore const manifests = Object.values(this.app.plugins.manifests) as any as PluginManifest[]; @@ -972,20 +971,20 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { const enabledPlugins = this.app.plugins.enabledPlugins as Set; const pluginManifest = manifests.find((manifest) => enabledPlugins.has(manifest.id) && manifest.dir == `${baseDir}/plugins/${data.name}`); if (pluginManifest) { - Logger(`Unloading plugin: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id); + this._log(`Unloading plugin: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id); // @ts-ignore await this.app.plugins.unloadPlugin(pluginManifest.id); // @ts-ignore await this.app.plugins.loadPlugin(pluginManifest.id); - Logger(`Plugin reloaded: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id); + this._log(`Plugin reloaded: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id); } } else if (data.category == "CONFIG") { this.plugin.$$askReload(); } return true; } catch (ex) { - Logger(`Applying ${data.displayName || data.name}.. Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Applying ${data.displayName || data.name}.. Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } } @@ -1007,12 +1006,12 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { await Promise.allSettled(p); // await this.deleteConfigOnDatabase(data.documentPath); // await this.updatePluginList(false, data.documentPath); - Logger(`Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, LOG_LEVEL_NOTICE); + this._log(`Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, LOG_LEVEL_NOTICE); } return true; } catch (ex) { - Logger(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } @@ -1096,13 +1095,13 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { displayName = `${json.name}`; } } catch (ex) { - Logger(`Configuration sync data: ${path} looks like manifest, but could not read the version`, LOG_LEVEL_INFO); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Configuration sync data: ${path} looks like manifest, but could not read the version`, LOG_LEVEL_INFO); + this._log(ex, LOG_LEVEL_VERBOSE); } } } catch (ex) { - Logger(`The file ${path} could not be encoded`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`The file ${path} could not be encoded`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } const mtime = stat.mtime; @@ -1150,7 +1149,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { }; } else { if (isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN) { - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG); return; } const docXDoc = await this.localDatabase.getDBEntryFromMeta(old, {}, false, false); @@ -1162,7 +1161,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { const oldContent = dataSrc.substring(dataStart + DUMMY_END.length); const oldContentArray = base64ToArrayBuffer(oldContent); if (await isDocContentSame(oldContentArray, content)) { - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE); markChangesAreSame(prefixedFileName, old.mtime, mtime + 1); return true; } @@ -1179,12 +1178,12 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { }; } const ret = await this.localDatabase.putDBEntry(saveData); - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); fireAndForget(() => this.updatePluginListV2(false, this.filenameWithUnifiedKey(path))); return ret; } catch (ex) { - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } }) @@ -1192,7 +1191,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { async storeCustomizationFiles(path: FilePath, termOverRide?: string) { const term = termOverRide || this.plugin.deviceAndVaultName; if (term == "") { - Logger("We have to configure the device name", LOG_LEVEL_NOTICE); + this._log("We have to configure the device name", LOG_LEVEL_NOTICE); return; } if (this.useV2) { @@ -1234,7 +1233,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { for (const target of fileTargets) { const data = await this.makeEntryFromFile(target); if (data == false) { - Logger(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); + this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); continue; } if (data.version) { @@ -1249,9 +1248,9 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } dt.mtime = mtime; - // Logger(`Configuration saving: ${prefixedFileName}`); + // this._log(`Configuration saving: ${prefixedFileName}`); if (dt.files.length == 0) { - Logger(`Nothing left: deleting.. ${path}`); + this._log(`Nothing left: deleting.. ${path}`); await this.deleteConfigOnDatabase(prefixedFileName); await this.updatePluginList(false, prefixedFileName); return @@ -1277,7 +1276,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { }; } else { if (old.mtime == mtime) { - // Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); + // this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); return true; } const oldC = await this.localDatabase.getDBEntryFromMeta(old, {}, false, false); @@ -1289,7 +1288,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { })) const isSame = (await Promise.all(diffs)).every(e => e == true); if (isSame) { - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE); return true; } } @@ -1308,25 +1307,24 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } const ret = await this.localDatabase.putDBEntry(saveData); await this.updatePluginList(false, saveData.path); - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); return ret; } catch (ex) { - Logger(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } }) - } async $anyProcessOptionalFileEvent(path: FilePath): Promise { return await this.watchVaultRawEventsAsync(path); } async watchVaultRawEventsAsync(path: FilePath) { - if (!this._isMainReady) return true; - if (!this._isMainSuspended()) return true; - if (!this._isThisModuleEnabled()) return true; - if (!this.isTargetPath(path)) return false; + if (!this._isMainReady) return false; + if (this._isMainSuspended()) return false; + if (!this._isThisModuleEnabled()) return false; + // if (!this.isTargetPath(path)) return false; const stat = await this.plugin.storageAccess.statHidden(path); // Make sure that target is a file. if (stat && stat.type != "file") @@ -1337,10 +1335,11 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { e.mode != MODE_SELECTIVE && e.mode != MODE_SHINY ).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()); if (synchronisedInConfigSync.some(e => e.startsWith(path.toLowerCase()))) { - Logger(`Customization file skipped: ${path}`, LOG_LEVEL_VERBOSE); + this._log(`Customization file skipped: ${path}`, LOG_LEVEL_VERBOSE); // This file could be handled by the other module. return false; } + // this._log(`Customization file detected: ${path}`, LOG_LEVEL_VERBOSE); const storageMTime = ~~((stat && stat.mtime || 0) / 1000); const key = `${path}-${storageMTime}`; if (this.recentProcessedInternalFiles.contains(key)) { @@ -1359,16 +1358,13 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { return true; } - - - async scanAllConfigFiles(showMessage: boolean) { await shareRunningResult("scanAllConfigFiles", async () => { const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; - Logger("Scanning customizing files.", logLevel, "scan-all-config"); + this._log("Scanning customizing files.", logLevel, "scan-all-config"); const term = this.plugin.deviceAndVaultName; if (term == "") { - Logger("We have to configure the device name", LOG_LEVEL_NOTICE); + this._log("We have to configure the device name", LOG_LEVEL_NOTICE); return; } const filesAll = await this.scanInternalFiles(); @@ -1396,8 +1392,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { await this.deleteConfigOnDatabase(unifiedFilenameWithKey); } } catch (ex) { - Logger(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); } finally { releaser(); } @@ -1412,8 +1408,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { try { await this.storeCustomisationFileV2(filePath, term); } catch (ex) { - Logger(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); } finally { releaser(); @@ -1430,7 +1426,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { for (const vp of virtualPathsOfLocalFiles) { const p = files.find(e => e.key == vp)?.file; if (!p) { - Logger(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE); + this._log(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE); continue; } await this.storeCustomizationFiles(p); @@ -1453,11 +1449,11 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false) as InternalFileEntry | false; let saveData: InternalFileEntry; if (old === false) { - Logger(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`); + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`); return true; } else { if (old.deleted) { - Logger(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`); + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`); return true; } saveData = @@ -1472,11 +1468,11 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { } await this.localDatabase.putRaw(saveData); await this.updatePluginList(false, prefixedFileName); - Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); return true; } catch (ex) { - Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } }); @@ -1530,7 +1526,7 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { $allSuspendExtraSync(): Promise { if (this.plugin.settings.usePluginSync || this.plugin.settings.autoSweepPlugins) { - Logger("Customisation sync have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE) + this._log("Customisation sync have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE) this.plugin.settings.usePluginSync = false; this.plugin.settings.autoSweepPlugins = false; } @@ -1590,8 +1586,8 @@ export class ConfigSync extends LiveSyncCommands implements IObsidianModule { try { w = await this.app.vault.adapter.list(path); } catch (ex) { - Logger(`Could not traverse(ConfigSync):${path}`, LOG_LEVEL_INFO); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Could not traverse(ConfigSync):${path}`, LOG_LEVEL_INFO); + this._log(ex, LOG_LEVEL_VERBOSE); return []; } let files = [ diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index e7ef060..6e24a43 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -1,9 +1,8 @@ import { normalizePath, type PluginManifest, type ListedFiles } from "../../deps.ts"; -import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry, type DocumentID } from "../../lib/src/common/types.ts"; +import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry, type DocumentID, type UXStat, MODE_AUTOMATIC } from "../../lib/src/common/types.ts"; import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "../../common/types.ts"; import { readAsBlob, isDocContentSame, sendSignal, readContent, createBlob, fireAndForget } from "../../lib/src/common/utils.ts"; -import { Logger } from "../../lib/src/common/logger.ts"; -import { getPath, isInternalMetadata, PeriodicProcessor } from "../../common/utils.ts"; +import { BASE_IS_NEW, compareMTime, EVEN, getPath, isInternalMetadata, isMarkedAsSameChanges, markChangesAreSame, PeriodicProcessor, TARGET_IS_NEW } from "../../common/utils.ts"; import { serialized } from "../../lib/src/concurrency/lock.ts"; import { JsonResolveModal } from "../HiddenFileCommon/JsonResolveModal.ts"; import { LiveSyncCommands } from "../LiveSyncCommands.ts"; @@ -41,12 +40,12 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule async $everyOnDatabaseInitialized(showNotice: boolean) { if (this._isThisModuleEnabled()) { try { - Logger("Synchronizing hidden files..."); + this._log("Synchronizing hidden files..."); await this.syncInternalFilesAndDatabase("push", showNotice); - Logger("Synchronizing hidden files done"); + this._log("Synchronizing hidden files done"); } catch (ex) { - Logger("Synchronizing hidden files failed"); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Synchronizing hidden files failed"); + this._log(ex, LOG_LEVEL_VERBOSE); } } return true; @@ -58,6 +57,13 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule return true; } + $everyOnloadAfterLoadSettings(): Promise { + const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns + .replace(/\n| /g, "") + .split(",").filter(e => e).map(e => new RegExp(e, "i")); + this.ignorePatterns = ignorePatterns; + return Promise.resolve(true); + } async $everyOnResumeProcess(): Promise { this.periodicInternalFileScanProcessor?.disable(); @@ -77,6 +83,10 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule if (!this.plugin.isReady) return Promise.resolve(true); this.periodicInternalFileScanProcessor.enable(this._isThisModuleEnabled() && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0); + const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns + .replace(/\n| /g, "") + .split(",").filter(e => e).map(e => new RegExp(e, "i")); + this.ignorePatterns = ignorePatterns; return Promise.resolve(true); } @@ -85,56 +95,56 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule } internalFileProcessor = new QueueProcessor( async (filenames) => { - Logger(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); + this._log(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); await this.syncInternalFilesAndDatabase("pull", false, false, filenames); - Logger(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); + this._log(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); return; }, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 100, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount } ); - recentProcessedInternalFiles = [] as string[]; - async $anyProcessOptionalFileEvent(path: FilePath): Promise { return await this.watchVaultRawEventsAsync(path); } async watchVaultRawEventsAsync(path: FilePath): Promise { + if (!this._isMainReady) return false; + if (this._isMainSuspended()) return false; if (!this._isThisModuleEnabled()) return false; - if (!isInternalMetadata(path)) return false; // Exclude files handled by customization sync const configDir = normalizePath(this.app.vault.configDir); - const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()); + const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : + Object.values(this.settings.pluginSyncExtendedSetting). + filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED). + map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()); if (synchronisedInConfigSync.some(e => e.startsWith(path.toLowerCase()))) { - Logger(`Hidden file skipped: ${path} is synchronized in customization sync.`, LOG_LEVEL_VERBOSE); + this._log(`Hidden file skipped: ${path} is synchronized in customization sync.`, LOG_LEVEL_VERBOSE); return false; } + const stat = await this.plugin.storageAccess.statHidden(path); // sometimes folder is coming. if (stat != null && stat.type != "file") { return false; } - const mtime = stat == null ? 0 : stat?.mtime ?? 0; - const storageMTime = ~~((mtime) / 1000); - const key = `${path}-${storageMTime}`; - if (mtime != 0 && this.recentProcessedInternalFiles.contains(key)) { - //If recently processed, it may caused by self. - // Return true to prevent further processing. + + if (this.isKnownChange(path, stat?.mtime ?? 0)) { + // This could be caused by self. so return true to prevent further processing. return true; } - this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100); - // const id = await this.path2id(path, ICHeader); + const mtime = stat == null ? 0 : stat?.mtime ?? 0; + const storageMTime = ~~((mtime) / 1000); + const prefixedFileName = addPrefix(path, ICHeader); const filesOnDB = await this.localDatabase.getDBEntryMeta(prefixedFileName); const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000); // Skip unchanged file. if (dbMTime == storageMTime) { - // Logger(`STORAGE --> DB:${path}: (hidden) Nothing changed`); + // this._log(`STORAGE --> DB:${path}: (hidden) Nothing changed`); // Handled, but nothing changed. also return true to prevent further processing. return true; } - // Do not compare timestamp. Always local data should be preferred except this plugin wrote one. try { if (storageMTime == 0) { await this.deleteInternalFileOnDatabase(path); @@ -144,8 +154,8 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule // Surely processed. return true; } catch (ex) { - Logger(`Failed to process hidden file:${path}`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Failed to process hidden file:${path}`); + this._log(ex, LOG_LEVEL_VERBOSE); } // Could not be processed. but it was own task. so return true to prevent further processing. return true; @@ -164,8 +174,8 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule } } } catch (ex) { - Logger("something went wrong on resolving all conflicted internal files"); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("something went wrong on resolving all conflicted internal files"); + this._log(ex, LOG_LEVEL_VERBOSE); } await this.conflictResolutionProcessor.startPipeline().waitForAllProcessed(); } @@ -176,12 +186,12 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule // simply check modified time const mtimeCurrent = ("mtime" in currentDoc && currentDoc.mtime) || 0; const mtimeConflicted = ("mtime" in conflictedDoc && conflictedDoc.mtime) || 0; - // Logger(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`); + // this._log(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`); // console.log(`mtime:${mtimeA} - ${mtimeB}`); const delRev = mtimeCurrent < mtimeConflicted ? currentRev : conflictedRev; // delete older one. await this.localDatabase.removeRevision(id, delRev); - Logger(`Older one has been deleted:${path}`); + this._log(`Older one has been deleted:${path}`); const cc = await this.localDatabase.getRaw(id, { conflicts: true }); if (cc._conflicts?.length === 0) { await this.extractInternalFileFromDatabase(stripAllPrefixes(path)) @@ -204,7 +214,7 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule if (doc._conflicts === undefined) return []; if (doc._conflicts.length == 0) return []; - Logger(`Hidden file conflicted:${path}`); + this._log(`Hidden file conflicted:${path}`); const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0])); const revA = doc._rev; const revB = conflicts[0]; @@ -217,7 +227,7 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule const commonBase = revFrom._revs_info?.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? ""; const result = await this.plugin.localDatabase.mergeObject(path, commonBase, doc._rev, conflictedRev); if (result) { - Logger(`Object merge:${path}`, LOG_LEVEL_INFO); + this._log(`Object merge:${path}`, LOG_LEVEL_INFO); const filename = stripAllPrefixes(path); const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(filename); if (!isExists) { @@ -234,7 +244,7 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule this.conflictResolutionProcessor.enqueue(path); return []; } else { - Logger(`Object merge is not applicable.`, LOG_LEVEL_VERBOSE); + this._log(`Object merge is not applicable.`, LOG_LEVEL_VERBOSE); } return [{ path, revA, revB, id, doc }]; } @@ -242,8 +252,8 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule await this.resolveByNewerEntry(id, path, doc, revA, revB); return []; } catch (ex) { - Logger(`Failed to resolve conflict (Hidden): ${path}`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Failed to resolve conflict (Hidden): ${path}`); + this._log(ex, LOG_LEVEL_VERBOSE); return []; } }, { @@ -281,7 +291,7 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule this.procInternalFile(filename); return true; } else { - Logger(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE); + this._log(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE); return false; } } @@ -291,30 +301,46 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule this.conflictResolutionProcessor.enqueue(path); } + knownChanges: { [key: string]: number; } = {}; + markAsKnownChange(path: string, mtime: number) { + this.knownChanges[path] = mtime; + } + isKnownChange(path: string, mtime: number) { + return this.knownChanges[path] == mtime; + } + ignorePatterns: RegExp[] = []; //TODO: Tidy up. Even though it is experimental feature, So dirty... - async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe" | "pullForce" | "pushForce", showMessage: boolean, filesAll: InternalFileInfo[] | false = false, targetFiles: string[] | false = false) { + async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe" | "pullForce" | "pushForce", showMessage: boolean, filesAll: InternalFileInfo[] | false = false, targetFilesSrc: string[] | false = false) { + const targetFiles = targetFilesSrc ? targetFilesSrc.map(e => stripAllPrefixes(e as FilePathWithPrefix)) : false; + // debugger; await this.resolveConflictOnInternalFiles(); const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; - Logger("Scanning hidden files.", logLevel, "sync_internal"); - const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns - .replace(/\n| /g, "") - .split(",").filter(e => e).map(e => new RegExp(e, "i")); + this._log("Scanning hidden files.", logLevel, "sync_internal"); const configDir = normalizePath(this.app.vault.configDir); let files: InternalFileInfo[] = filesAll ? filesAll : (await this.scanInternalFiles()) + const allowedInHiddenFileSync = this.settings.usePluginSync ? Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_AUTOMATIC).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()) : undefined; + if (allowedInHiddenFileSync) { + const systemOrNot = files.reduce((acc, cur) => { + if (cur.path.startsWith(configDir)) { + acc.system.push(cur); + } else { + acc.user.push(cur); + } + return acc; + }, { system: [] as InternalFileInfo[], user: [] as InternalFileInfo[] }); - const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()); - files = files.filter(file => synchronisedInConfigSync.every(filterFile => !file.path.toLowerCase().startsWith(filterFile))) + files = + [...systemOrNot.user, + ...systemOrNot.system.filter(file => allowedInHiddenFileSync.some(filterFile => file.path.toLowerCase().startsWith(filterFile)))]; + } const filesOnDB = ((await this.localDatabase.allDocsRaw({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted); const allFileNamesSrc = [...new Set([...files.map(e => normalizePath(e.path)), ...filesOnDB.map(e => stripAllPrefixes(this.getPath(e)))])]; - const allFileNames = allFileNamesSrc.filter(filename => !targetFiles || (targetFiles && targetFiles.indexOf(filename) !== -1)).filter(path => synchronisedInConfigSync.every(filterFile => !path.toLowerCase().startsWith(filterFile))) - function compareMTime(a: number, b: number) { - const wa = ~~(a / 1000); - const wb = ~~(b / 1000); - const diff = wa - wb; - return diff; + let allFileNames = allFileNamesSrc.filter(filename => !targetFiles || (targetFiles && targetFiles.indexOf(filename) !== -1)); + if (allowedInHiddenFileSync) { + allFileNames = allFileNames.filter(file => allowedInHiddenFileSync.some(filterFile => file.toLowerCase().startsWith(filterFile))); } const fileCount = allFileNames.length; @@ -342,9 +368,7 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule c = pieces.shift(); } }; - // Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content) - let caches: { [key: string]: { storageMtime: number; docMtime: number; }; } = {}; - caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number; }; }>("diff-caches-internal") || {}; + const filesMap = files.reduce((acc, cur) => { acc[cur.path] = cur; return acc; @@ -357,10 +381,10 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule const filename = filenames[0]; processed++; if (processed % 100 == 0) { - Logger(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal"); + this._log(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal"); } if (!filename) return []; - if (ignorePatterns.some(e => filename.match(e))) + if (this.ignorePatterns.some(e => filename.match(e))) return []; if (await this.plugin.$$isIgnoredByIgnoreFiles(filename)) { return []; @@ -384,25 +408,24 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule fileOnDatabase: xFileOnDatabase } = params[0]; if (xFileOnStorage && xFileOnDatabase) { - const cache = filename in caches ? caches[filename] : { storageMtime: 0, docMtime: 0 }; // Both => Synchronize - if ((direction != "pullForce" && direction != "pushForce") && xFileOnDatabase.mtime == cache.docMtime && xFileOnStorage.mtime == cache.storageMtime) { + if ((direction != "pullForce" && direction != "pushForce") && isMarkedAsSameChanges(filename, [xFileOnDatabase.mtime, xFileOnStorage.mtime]) == EVEN) { + this._log(`Hidden file skipped: ${filename} is marked as same`, LOG_LEVEL_VERBOSE); return; } + const nw = compareMTime(xFileOnStorage.mtime, xFileOnDatabase.mtime); - if (nw > 0 || direction == "pushForce") { - await this.storeInternalFileToDatabase(xFileOnStorage); - } - if (nw < 0 || direction == "pullForce") { + if (nw == BASE_IS_NEW || direction == "pushForce") { + if (await this.storeInternalFileToDatabase(xFileOnStorage) !== false) { + // countUpdatedFolder(filename); + } + } else if (nw == TARGET_IS_NEW || direction == "pullForce") { // skip if not extraction performed. - if (!await this.extractInternalFileFromDatabase(filename)) - return; + if (await this.extractInternalFileFromDatabase(filename)) + countUpdatedFolder(filename); + } else { + // Even, or not forced. skip. } - // If process successfully updated or file contents are same, update cache. - cache.docMtime = xFileOnDatabase.mtime; - cache.storageMtime = xFileOnStorage.mtime; - caches[filename] = cache; - countUpdatedFolder(filename); } else if (!xFileOnStorage && xFileOnDatabase) { if (direction == "push" || direction == "pushForce") { if (xFileOnDatabase.deleted) @@ -423,7 +446,9 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule if (direction == "push" || direction == "pushForce" || direction == "safe") { await this.storeInternalFileToDatabase(xFileOnStorage); } else { - await this.extractInternalFileFromDatabase(xFileOnStorage.path); + // if (await this.extractInternalFileFromDatabase(xFileOnStorage.path)) { + // countUpdatedFolder(xFileOnStorage.path); + // } } } else { throw new Error("Invalid state on hidden file sync"); @@ -435,8 +460,6 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule .enqueueAll(allFileNames) .startPipeline().waitForAllDoneAndTerminate(); - await this.kvDB.set("diff-caches-internal", caches); - // When files has been retrieved from the database. they must be reloaded. if ((direction == "pull" || direction == "pullForce") && filesChanged != 0) { // Show notification to restart obsidian when something has been changed in configDir. @@ -459,12 +482,12 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule anchor.text = "HERE"; anchor.addEventListener("click", () => { fireAndForget(async () => { - Logger(`Unloading plugin: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId); + this._log(`Unloading plugin: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId); // @ts-ignore await this.app.plugins.unloadPlugin(updatePluginId); // @ts-ignore await this.app.plugins.loadPlugin(updatePluginId); - Logger(`Plugin reloaded: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId); + this._log(`Plugin reloaded: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId); }); }); } @@ -472,8 +495,8 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule } } } catch (ex) { - Logger("Error on checking plugin status."); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Error on checking plugin status."); + this._log(ex, LOG_LEVEL_VERBOSE); } @@ -491,17 +514,19 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule } } - Logger(`Hidden files scanned: ${filesChanged} files had been modified`, logLevel, "sync_internal"); + this._log(`Hidden files scanned: ${filesChanged} files had been modified`, logLevel, "sync_internal"); } async storeInternalFileToDatabase(file: InternalFileInfo, forceWrite = false) { - if (await this.plugin.$$isIgnoredByIgnoreFiles(file.path)) { - return + const storeFilePath = file.path; + const storageFilePath = file.path; + if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) { + return undefined; } - const id = await this.path2id(file.path, ICHeader); - const prefixedFileName = addPrefix(file.path, ICHeader); - const content = createBlob(await this.plugin.storageAccess.readHiddenFileAuto(file.path)); + const id = await this.path2id(storeFilePath, ICHeader); + const prefixedFileName = addPrefix(storeFilePath, ICHeader); + const content = createBlob(await this.plugin.storageAccess.readHiddenFileAuto(storageFilePath)); const mtime = file.mtime; return await serialized("file-" + prefixedFileName, async () => { try { @@ -523,8 +548,12 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule }; } else { if (await isDocContentSame(readAsBlob(old), content) && !forceWrite) { - // Logger(`STORAGE --> DB:${file.path}: (hidden) Not changed`, LOG_LEVEL_VERBOSE); - return; + // this._log(`STORAGE --> DB:${file.path}: (hidden) Not changed`, LOG_LEVEL_VERBOSE); + const stat = await this.plugin.storageAccess.statHidden(storageFilePath); + if (stat) { + markChangesAreSame(storageFilePath, old.mtime, stat.mtime); + } + return undefined; } saveData = { @@ -539,24 +568,32 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule }; } const ret = await this.localDatabase.putDBEntry(saveData); - Logger(`STORAGE --> DB:${file.path}: (hidden) Done`); - return ret; + if (ret !== false) { + this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Done`); + return true; + } else { + this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`); + return false; + } } catch (ex) { - Logger(`STORAGE --> DB:${file.path}: (hidden) Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } }); } - async deleteInternalFileOnDatabase(filename: FilePath, forceWrite = false) { - const id = await this.path2id(filename, ICHeader); - const prefixedFileName = addPrefix(filename, ICHeader); + async deleteInternalFileOnDatabase(filenameSrc: FilePath, forceWrite = false) { + const storeFilePath = filenameSrc; + const storageFilePath = filenameSrc; + const displayFileName = filenameSrc; + const id = await this.path2id(storeFilePath, ICHeader); + const prefixedFileName = addPrefix(storeFilePath, ICHeader); const mtime = new Date().getTime(); - if (await this.plugin.$$isIgnoredByIgnoreFiles(filename)) { - return + if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) { + return undefined } - await serialized("file-" + prefixedFileName, async () => { + return await serialized("file-" + prefixedFileName, async () => { try { const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, true) as InternalFileEntry | false; let saveData: InternalFileEntry; @@ -578,12 +615,12 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule if (conflicts._conflicts !== undefined) { for (const conflictRev of conflicts._conflicts) { await this.localDatabase.removeRevision(old._id, conflictRev); - Logger(`STORAGE -x> DB:${filename}: (hidden) conflict removed ${old._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE); + this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) conflict removed ${old._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE); } } if (old.deleted) { - Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`); - return; + this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) already deleted`); + return undefined; } saveData = { @@ -595,85 +632,104 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule type: "newnote", }; } - await this.localDatabase.putRaw(saveData); - Logger(`STORAGE -x> DB:${filename}: (hidden) Done`); + const ret = await this.localDatabase.putRaw(saveData); + if (ret && ret.ok) { + this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Done`); + return true; + } else { + this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); + return false; + } } catch (ex) { - Logger(`STORAGE -x> DB:${filename}: (hidden) Failed`); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); return false; } }); } - async extractInternalFileFromDatabase(filename: FilePath, force = false) { - const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(filename); - const prefixedFileName = addPrefix(filename, ICHeader); - if (await this.plugin.$$isIgnoredByIgnoreFiles(filename)) { - return; + async extractInternalFileFromDatabase(filenameSrc: FilePath, force = false) { + const storeFilePath = filenameSrc; + const storageFilePath = filenameSrc; + const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(storageFilePath); + const prefixedFileName = addPrefix(storeFilePath, ICHeader); + const displayFileName = `${storeFilePath}`; + if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) { + return undefined; } return await serialized("file-" + prefixedFileName, async () => { try { // Check conflicted status const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, true, true); if (fileOnDB === false) - throw new Error(`File not found on database.:${filename}`); + throw new Error(`File not found on database.:${displayFileName}`); // Prevent overwrite for Prevent overwriting while some conflicted revision exists. if (fileOnDB?._conflicts?.length) { - Logger(`Hidden file ${filename} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL_INFO); - return; + this._log(`Hidden file ${displayFileName} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL_INFO); + return false; } const deleted = fileOnDB.deleted || fileOnDB._deleted || false; if (deleted) { if (!isExists) { - Logger(`STORAGE { return new Promise((res) => { - Logger("Opening data-merging dialog", LOG_LEVEL_VERBOSE); + this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE); const docs = [docA, docB]; - const path = stripAllPrefixes(docA.path); - const modal = new JsonResolveModal(this.app, path, [docA, docB], async (keep, result) => { + const strippedPath = stripAllPrefixes(docA.path); + const storageFilePath = strippedPath; + const storeFilePath = strippedPath; + const displayFilename = `${storeFilePath}`; + // const path = this.prefixedConfigDir2configDir(stripAllPrefixes(docA.path)) || docA.path; + const modal = new JsonResolveModal(this.app, storageFilePath, [docA, docB], async (keep, result) => { // modal.close(); try { - const filename = path; + // const filename = storeFilePath; let needFlush = false; if (!result && !keep) { - Logger(`Skipped merging: ${filename}`); + this._log(`Skipped merging: ${displayFilename}`); res(false); return; } @@ -701,41 +761,44 @@ export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule for (const doc of docs) { if (doc._rev != keep) { if (await this.localDatabase.deleteDBEntry(this.getPath(doc), { rev: doc._rev })) { - Logger(`Conflicted revision has been deleted: ${filename}`); + this._log(`Conflicted revision has been deleted: ${displayFilename}`); needFlush = true; } } } } if (!keep && result) { - const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(filename); + const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(storageFilePath); if (!isExists) { - await this.plugin.storageAccess.ensureDir(filename); + await this.plugin.storageAccess.ensureDir(storageFilePath); } - await this.plugin.storageAccess.writeHiddenFileAuto(filename, result); - const stat = await this.plugin.storageAccess.statHidden(filename); + await this.plugin.storageAccess.writeHiddenFileAuto(storageFilePath, result); + const stat = await this.plugin.storageAccess.statHidden(storageFilePath); if (!stat) { throw new Error("Stat failed"); } const mtime = stat?.mtime ?? 0; - await this.storeInternalFileToDatabase({ path: filename, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 }, true); + await this.storeInternalFileToDatabase({ path: storageFilePath, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 }, true); try { //@ts-ignore internalAPI - await this.app.vault.adapter.reconcileInternalFile(filename); + await this.app.vault.adapter.reconcileInternalFile(storageFilePath); } catch (ex) { - Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); } - Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`); + this._log(`STORAGE <-- DB:${displayFilename}: written (hidden,merged)`); } if (needFlush) { - await this.extractInternalFileFromDatabase(filename, false); - Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`); + if (await this.extractInternalFileFromDatabase(storeFilePath, false)) { + this._log(`STORAGE --> DB:${displayFilename}: extracted (hidden,merged)`); + } else { + this._log(`STORAGE --> DB:${displayFilename}: extracted (hidden,merged) Failed`); + } } res(true); } catch (ex) { - Logger("Could not merge conflicted json"); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log("Could not merge conflicted json"); + this._log(ex, LOG_LEVEL_VERBOSE); res(false); } }); @@ -789,7 +852,7 @@ ${messageFetch}${messageOverwrite}${messageMerge} $allSuspendExtraSync(): Promise { if (this.plugin.settings.syncInternalFiles) { - Logger("Hidden file synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE) + this._log("Hidden file synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE) this.plugin.settings.syncInternalFiles = false; } return Promise.resolve(true); @@ -810,7 +873,7 @@ ${messageFetch}${messageOverwrite}${messageMerge} await this.plugin.saveSettings(); return; } - Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL_NOTICE); + this._log("Gathering files for enabling Hidden File Sync", LOG_LEVEL_NOTICE); if (mode == "FETCH") { await this.syncInternalFilesAndDatabase("pullForce", true); } else if (mode == "OVERWRITE") { @@ -821,7 +884,7 @@ ${messageFetch}${messageOverwrite}${messageMerge} this.plugin.settings.useAdvancedMode = true; this.plugin.settings.syncInternalFiles = true; await this.plugin.saveSettings(); - Logger(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL_NOTICE); + this._log(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL_NOTICE); } async scanInternalFiles(): Promise { @@ -869,8 +932,8 @@ ${messageFetch}${messageOverwrite}${messageMerge} try { w = await this.app.vault.adapter.list(path); } catch (ex) { - Logger(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO); - Logger(ex, LOG_LEVEL_VERBOSE); + this._log(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO); + this._log(ex, LOG_LEVEL_VERBOSE); return []; } const filesSrc = [ diff --git a/src/features/LiveSyncCommands.ts b/src/features/LiveSyncCommands.ts index d0e3fa3..e21ab57 100644 --- a/src/features/LiveSyncCommands.ts +++ b/src/features/LiveSyncCommands.ts @@ -1,5 +1,6 @@ +import { Logger } from "octagonal-wheels/common/logger"; import { getPath } from "../common/utils.ts"; -import { type AnyEntry, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix } from "../lib/src/common/types.ts"; +import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, type AnyEntry, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type LOG_LEVEL } from "../lib/src/common/types.ts"; import type ObsidianLiveSyncPlugin from "../main.ts"; @@ -40,4 +41,12 @@ export abstract class LiveSyncCommands { _isDatabaseReady() { return this.plugin._isDatabaseReady(); } + + _log = (msg: any, level: LOG_LEVEL = LOG_LEVEL_INFO, key?: string) => { + if (typeof msg === "string" && level !== LOG_LEVEL_NOTICE) { + msg = `[${this.constructor.name}]\u{200A} ${msg}`; + } + // console.log(msg); + Logger(msg, level, key); + }; } diff --git a/src/modules/features/ModuleObsidianSettingAsMarkdown.ts b/src/modules/features/ModuleObsidianSettingAsMarkdown.ts index cfb0380..d52dc01 100644 --- a/src/modules/features/ModuleObsidianSettingAsMarkdown.ts +++ b/src/modules/features/ModuleObsidianSettingAsMarkdown.ts @@ -162,6 +162,7 @@ export class ModuleObsidianSettingsAsMarkdown extends AbstractObsidianModule imp const saveData = { ...(settings ? settings : this.settings) } as Partial; delete saveData.encryptedCouchDBConnection; delete saveData.encryptedPassphrase; + delete saveData.additionalSuffixOfDatabaseName; if (!saveData.writeCredentialsForSettingSync && !keepCredential) { delete saveData.couchDB_USER; delete saveData.couchDB_PASSWORD; diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 8995f5d..36d1865 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -1727,6 +1727,17 @@ However, your report is needed to stabilise this. I appreciate you for your grea const enableOnlyOnPluginSyncIsNotEnabled = enableOnly(() => this.isConfiguredAs("usePluginSync", false)); const visibleOnlyOnPluginSyncEnabled = visibleOnly(() => this.isConfiguredAs("usePluginSync", true)); + this.createEl(paneEl, "div", { + text: "Please set device name to identify this device. This name should be unique among your devices. While not configured, we cannot enable this feature.", + cls: "op-warn" + }, c => { + }, visibleOnly(() => this.isConfiguredAs("deviceAndVaultName", ""))); + this.createEl(paneEl, "div", { + text: "We cannot change the device name while this feature is enabled. Please disable this feature to change the device name.", + cls: "op-warn-info" + }, c => { + }, visibleOnly(() => this.isConfiguredAs("usePluginSync", true))); + new Setting(paneEl) .autoWireText("deviceAndVaultName", { placeHolder: "desktop", onUpdate: enableOnlyOnPluginSyncIsNotEnabled diff --git a/updates.md b/updates.md index 8c7847a..a6003ba 100644 --- a/updates.md +++ b/updates.md @@ -23,6 +23,19 @@ Thank you, and I hope your troubles will be resolved! --- +## 0.24.0.dev-rc5 + +### Improved + +- A note relating to device names has been added to Customisation Sync on the setting dialogue. +- Logs of Hidden File Sync and Customisation Sync have been prefixed with the respective feature names. + +### Fixed + +- Hidden file sync is now working correctly. +- Customisation Sync is now working correctly together with hidden file sync +- No longer database suffix is stored in the setting sharing markdown. + ## 0.24.0.dev-rc4 ### Improved