From 5238dec3f23992710ca5bd402e4b4ec9d8fa08e3 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 4 Nov 2025 11:34:39 +0000 Subject: [PATCH] fix: Now hidden file synchronisation respects the filters correctly (#631, #735) --- .../HiddenFileSync/CmdHiddenFileSync.ts | 226 +++++++++++++----- src/modules/core/ModuleTargetFilter.ts | 12 +- .../storageLib/StorageEventManager.ts | 17 +- 3 files changed, 189 insertions(+), 66 deletions(-) diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 44bb0c2..758a78c 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -1,4 +1,4 @@ -import { normalizePath, type PluginManifest, type ListedFiles } from "../../deps.ts"; +import { type PluginManifest, type ListedFiles } from "../../deps.ts"; import { type LoadedEntry, type FilePathWithPrefix, @@ -10,7 +10,6 @@ import { MODE_PAUSED, type SavingEntry, type DocumentID, - type FilePathWithPrefixLC, type UXFileInfo, type UXStat, LOG_LEVEL_DEBUG, @@ -177,24 +176,10 @@ export class HiddenFileSync extends LiveSyncCommands { this.updateSettingCache(); return Promise.resolve(true); } - updateSettingCache() { - const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); - this.ignorePatterns = ignorePatterns; - const targetFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); - this.targetPatterns = targetFilter; - this.shouldSkipFile = [] as FilePathWithPrefixLC[]; - // Exclude files handled by customization sync - const configDir = normalizePath(this.app.vault.configDir); - const shouldSKip = !this.settings.usePluginSync - ? [] - : Object.values(this.settings.pluginSyncExtendedSetting) - .filter((e) => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED) - .map((e) => e.files) - .flat() - .map((e) => `${configDir}/${e}`.toLowerCase()); - this.shouldSkipFile = shouldSKip as FilePathWithPrefixLC[]; - this._log(`Hidden file will skip ${this.shouldSkipFile.length} files`, LOG_LEVEL_INFO); + updateSettingCache() { + this.cacheCustomisationSyncIgnoredFiles.clear(); + this.cacheFileRegExps.clear(); } isReady() { @@ -203,7 +188,6 @@ export class HiddenFileSync extends LiveSyncCommands { if (!this.isThisModuleEnabled()) return false; return true; } - shouldSkipFile = [] as FilePathWithPrefixLC[]; async performStartupScan(showNotice: boolean) { await this.applyOfflineChanges(showNotice); @@ -232,10 +216,11 @@ export class HiddenFileSync extends LiveSyncCommands { ? this.settings.syncInternalFilesInterval * 1000 : 0 ); - const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); - this.ignorePatterns = ignorePatterns; - const targetFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); - this.targetPatterns = targetFilter; + this.cacheFileRegExps.clear(); + // const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); + // this.ignorePatterns = ignorePatterns; + // const targetFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); + // this.targetPatterns = targetFilter; return Promise.resolve(true); } @@ -558,8 +543,11 @@ Offline Changed files: ${processFiles.length}`; forceWrite = false, includeDeleted = true ): Promise { - if (this.shouldSkipFile.some((e) => e.startsWith(path.toLowerCase()))) { - this._log(`Hidden file skipped: ${path} is synchronized in customization sync.`, LOG_LEVEL_VERBOSE); + if (!(await this.isTargetFile(path))) { + this._log( + `Storage file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, + LOG_LEVEL_VERBOSE + ); return false; } try { @@ -862,6 +850,108 @@ Offline Changed files: ${processFiles.length}`; // --> Database Event Functions + cacheFileRegExps = new Map(); + /** + * Parses the regular expression settings for hidden file synchronization. + * @returns An object containing the ignore and target filters. + */ + parseRegExpSettings() { + const regExpKey = `${this.plugin.settings.syncInternalFilesTargetPatterns}||${this.plugin.settings.syncInternalFilesIgnorePatterns}`; + let ignoreFilter: CustomRegExp[]; + let targetFilter: CustomRegExp[]; + if (this.cacheFileRegExps.has(regExpKey)) { + const cached = this.cacheFileRegExps.get(regExpKey)!; + ignoreFilter = cached[1]; + targetFilter = cached[0]; + } else { + ignoreFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); + targetFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); + this.cacheFileRegExps.clear(); + this.cacheFileRegExps.set(regExpKey, [targetFilter, ignoreFilter]); + } + return { ignoreFilter, targetFilter }; + } + + /** + * Checks if the target file path matches the defined patterns. + */ + isTargetFileInPatterns(path: string): boolean { + const { ignoreFilter, targetFilter } = this.parseRegExpSettings(); + + if (ignoreFilter && ignoreFilter.length > 0) { + for (const pattern of ignoreFilter) { + if (pattern.test(path)) { + return false; + } + } + } + if (targetFilter && targetFilter.length > 0) { + for (const pattern of targetFilter) { + if (pattern.test(path)) { + return true; + } + } + // While having target patterns, it effects as an allow-list. + return false; + } + return true; + } + + cacheCustomisationSyncIgnoredFiles = new Map(); + /** + * Gets the list of files ignored for customization synchronization. + * @returns An array of ignored file paths (lowercase). + */ + getCustomisationSynchronizationIgnoredFiles(): string[] { + const configDir = this.plugin.app.vault.configDir; + const key = + JSON.stringify(this.settings.pluginSyncExtendedSetting) + `||${this.settings.usePluginSync}||${configDir}`; + if (this.cacheCustomisationSyncIgnoredFiles.has(key)) { + return this.cacheCustomisationSyncIgnoredFiles.get(key)!; + } + this.cacheCustomisationSyncIgnoredFiles.clear(); + const synchronisedInConfigSync = !this.settings.usePluginSync + ? [] + : Object.values(this.settings.pluginSyncExtendedSetting) + .filter((e) => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED) + .map((e) => e.files) + .flat() + .map((e) => `${configDir}/${e}`.toLowerCase()); + this.cacheCustomisationSyncIgnoredFiles.set(key, synchronisedInConfigSync); + return synchronisedInConfigSync; + } + /** + * Checks if the given path is not ignored by customization synchronization. + * @param path The file path to check. + * @returns True if the path is not ignored; otherwise, false. + */ + isNotIgnoredByCustomisationSync(path: string): boolean { + const ignoredFiles = this.getCustomisationSynchronizationIgnoredFiles(); + const result = !ignoredFiles.some((e) => path.startsWith(e)); + // console.warn(`Assertion: isNotIgnoredByCustomisationSync(${path}) = ${result}`); + return result; + } + + isHiddenFileSyncHandlingPath(path: FilePath): boolean { + const result = path.startsWith(".") && !path.startsWith(".trash"); + // console.warn(`Assertion: isHiddenFileSyncHandlingPath(${path}) = ${result}`); + return result; + } + + async isTargetFile(path: FilePath): Promise { + const result = + this.isTargetFileInPatterns(path) && + this.isNotIgnoredByCustomisationSync(path) && + this.isHiddenFileSyncHandlingPath(path); + // console.warn(`Assertion: isTargetFile(${path}) : ${result ? "✔️" : "❌"}`); + if (!result) { + return false; + } + const resultByFile = await this.services.vault.isIgnoredByIgnoreFile(path); + // console.warn(`${path} -> isIgnoredByIgnoreFile: ${resultByFile ? "❌" : "✔️"}`); + return !resultByFile; + } + async trackScannedDatabaseChange( processFiles: MetaEntry[], showNotice: boolean = false, @@ -875,14 +965,21 @@ Offline Changed files: ${processFiles.length}`; const processes = processFiles.map(async (file) => { try { const path = stripAllPrefixes(this.getPath(file)); - await this.trackDatabaseFileModification( - path, - "[Hidden file scan]", - !forceWriteAll, - onlyNew, - file, - includeDeletion - ); + if (!(await this.isTargetFile(path))) { + this._log( + `Database file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, + LOG_LEVEL_VERBOSE + ); + } else { + await this.trackDatabaseFileModification( + path, + "[Hidden file scan]", + !forceWriteAll, + onlyNew, + file, + includeDeletion + ); + } notifyProgress(); } catch (ex) { this._log(`Failed to process storage change file:${file}`, logLevel); @@ -1215,7 +1312,13 @@ Offline Changed files: ${files.length}`; ).rows .filter((e) => isInternalMetadata(e.id as DocumentID)) .map((e) => e.doc) as MetaEntry[]; - return allFiles; + const files = [] as MetaEntry[]; + for (const file of allFiles) { + if (await this.isTargetFile(stripAllPrefixes(this.getPath(file)))) { + files.push(file); + } + } + return files; } async rebuildFromDatabase(showNotice: boolean, targetFiles: FilePath[] | false = false, onlyNew = false) { @@ -1696,29 +1799,13 @@ ${messageFetch}${messageOverwrite}${messageMerge} // <-- Configuration handling // --> Local Storage SubFunctions - ignorePatterns: CustomRegExp[] = []; - targetPatterns: CustomRegExp[] = []; async scanInternalFileNames() { - const configDir = normalizePath(this.app.vault.configDir); - const ignoreFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); - const targetFilter = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); - const synchronisedInConfigSync = !this.settings.usePluginSync - ? [] - : Object.values(this.settings.pluginSyncExtendedSetting) - .filter((e) => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED) - .map((e) => e.files) - .flat() - .map((e) => `${configDir}/${e}`.toLowerCase()); const root = this.app.vault.getRoot(); const findRoot = root.path; - const filenames = (await this.getFiles(findRoot, [], targetFilter, ignoreFilter)) - .filter((e) => e.startsWith(".")) - .filter((e) => !e.startsWith(".trash")); - const files = filenames.filter((path) => - synchronisedInConfigSync.every((filterFile) => !path.toLowerCase().startsWith(filterFile)) - ); - return files as FilePath[]; + const filenames = await this.getFiles(findRoot, (path) => this.isTargetFile(path)); + + return filenames as FilePath[]; } async scanInternalFiles(): Promise { @@ -1748,7 +1835,32 @@ ${messageFetch}${messageOverwrite}${messageMerge} return result; } - async getFiles(path: string, ignoreList: string[], filter?: CustomRegExp[], ignoreFilter?: CustomRegExp[]) { + async getFiles(path: string, checkFunction: (path: FilePath) => Promise | boolean) { + let w: ListedFiles; + try { + w = await this.app.vault.adapter.list(path); + } catch (ex) { + this._log(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO); + this._log(ex, LOG_LEVEL_VERBOSE); + return []; + } + let files = [] as string[]; + for (const file of w.files) { + if (!(await checkFunction(file as FilePath))) { + continue; + } + files.push(file); + } + for (const v of w.folders) { + if (!(await checkFunction(v as FilePath))) { + continue; + } + files = files.concat(await this.getFiles(v, checkFunction)); + } + return files; + } + /* + async getFiles_(path: string, ignoreList: string[], filter?: CustomRegExp[], ignoreFilter?: CustomRegExp[]) { let w: ListedFiles; try { w = await this.app.vault.adapter.list(path); @@ -1785,11 +1897,11 @@ ${messageFetch}${messageOverwrite}${messageMerge} if (await this.services.vault.isIgnoredByIgnoreFile(v)) { continue L1; } - files = files.concat(await this.getFiles(v, ignoreList, filter, ignoreFilter)); + files = files.concat(await this.getFiles_(v, ignoreList, filter, ignoreFilter)); } return files; } - + */ // <-- Local Storage SubFunctions onBindFunction(core: LiveSyncCore, services: typeof core.services) { diff --git a/src/modules/core/ModuleTargetFilter.ts b/src/modules/core/ModuleTargetFilter.ts index bffcbc8..06009fc 100644 --- a/src/modules/core/ModuleTargetFilter.ts +++ b/src/modules/core/ModuleTargetFilter.ts @@ -26,6 +26,7 @@ export class ModuleTargetFilter extends AbstractModule { this.ignoreFiles = this.settings.ignoreFiles.split(",").map((e) => e.trim()); } private _everyOnload(): Promise { + this.reloadIgnoreFiles(); eventHub.onEvent(EVENT_SETTING_SAVED, (evt: ObsidianLiveSyncSettings) => { this.reloadIgnoreFiles(); }); @@ -132,12 +133,19 @@ export class ModuleTargetFilter extends AbstractModule { ignoreFiles = [] as string[]; async readIgnoreFile(path: string) { try { - const file = await this.core.storageAccess.readFileText(path); + // this._log(`[ignore]Reading ignore file: ${path}`, LOG_LEVEL_VERBOSE); + if (!(await this.core.storageAccess.isExistsIncludeHidden(path))) { + this.ignoreFileCache.set(path, false); + // this._log(`[ignore]Ignore file not found: ${path}`, LOG_LEVEL_VERBOSE); + return false; + } + const file = await this.core.storageAccess.readHiddenFileText(path); const gitignore = file.split(/\r?\n/g); this.ignoreFileCache.set(path, gitignore); + this._log(`[ignore]Ignore file loaded: ${path}`, LOG_LEVEL_VERBOSE); return gitignore; } catch (ex) { - this._log(`Failed to read ignore file ${path}`); + this._log(`[ignore]Failed to read ignore file ${path}`); this._log(ex, LOG_LEVEL_VERBOSE); this.ignoreFileCache.set(path, false); return false; diff --git a/src/modules/coreObsidian/storageLib/StorageEventManager.ts b/src/modules/coreObsidian/storageLib/StorageEventManager.ts index 5f7f327..480766b 100644 --- a/src/modules/coreObsidian/storageLib/StorageEventManager.ts +++ b/src/modules/coreObsidian/storageLib/StorageEventManager.ts @@ -13,7 +13,7 @@ import { type UXFileInfoStub, type UXInternalFileInfoStub, } from "../../../lib/src/common/types.ts"; -import { delay, fireAndForget, getFileRegExp } from "../../../lib/src/common/utils.ts"; +import { delay, fireAndForget } from "../../../lib/src/common/utils.ts"; import { type FileEventItem } from "../../../common/types.ts"; import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; import { @@ -27,6 +27,7 @@ import type { LiveSyncCore } from "../../../main.ts"; import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts"; import ObsidianLiveSyncPlugin from "../../../main.ts"; import type { StorageAccess } from "../../interfaces/StorageAccess.ts"; +import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts"; // import { InternalFileToUXFileInfo } from "../platforms/obsidian.ts"; export type FileEvent = { @@ -62,11 +63,15 @@ export class StorageEventManagerObsidian extends StorageEventManager { get batchSaveMaximumDelay(): number { return this.core.settings?.batchSaveMaximumDelay ?? DEFAULT_SETTINGS.batchSaveMaximumDelay; } + // Necessary evil. + cmdHiddenFileSync: HiddenFileSync; + constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, storageAccess: StorageAccess) { super(); this.storageAccess = storageAccess; this.plugin = plugin; this.core = core; + this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync; } beginWatch() { const plugin = this.plugin; @@ -181,22 +186,20 @@ export class StorageEventManagerObsidian extends StorageEventManager { // (Calling$$isTargetFile will refresh the cache) void this.services.vault.isTargetFile(path).then(() => this._watchVaultRawEvents(path)); } else { - this._watchVaultRawEvents(path); + void this._watchVaultRawEvents(path); } } - _watchVaultRawEvents(path: FilePath) { + async _watchVaultRawEvents(path: FilePath) { if (!this.plugin.settings.syncInternalFiles && !this.plugin.settings.usePluginSync) return; if (!this.plugin.settings.watchInternalFileChanges) return; if (!path.startsWith(this.plugin.app.vault.configDir)) return; - const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns"); - const targetPatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns"); - if (ignorePatterns.some((e) => e.test(path))) return; - if (!targetPatterns.some((e) => e.test(path))) return; if (path.endsWith("/")) { // Folder return; } + const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path); + if (!isTargetFile) return; void this.appendQueue( [