- Error handling on booting now works fine.
  - Replication is now started automatically in LiveSync mode.
  - Batch database update is now disabled in LiveSync mode.
  - No longer automatically reconnection while off-focused.
  - Status saves are thinned out.
  - Now Self-hosted LiveSync waits for all files between the local database and storage to be surely checked.
- Improved:
  - The job scheduler is now more robust and stable.
  - The status indicator no longer flickers and keeps zero for a while.
  - No longer meaningless frequent updates of status indicators.
  - Now we can configure regular expression filters in handy UI. Thank you so much, @eth-p!
  - `Fetch` or `Rebuild everything` is now more safely performed.
- Minor things
  - Some utility function has been added.
  - Customisation sync now less wrong messages.
  - Digging the weeds for eradication of type errors.
This commit is contained in:
vorotamoroz
2024-04-12 01:30:35 +09:00
parent 6952ef37f5
commit d54b7e2d93
10 changed files with 361 additions and 197 deletions

View File

@@ -4,7 +4,7 @@ import { Notice, type PluginManifest, parseYaml, normalizePath, type ListedFiles
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types"; import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, isDocContentSame } from "./lib/src/utils"; import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, isDocContentSame, throttle } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "./lib/src/strbin"; import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "./lib/src/strbin";
import { serialized } from "./lib/src/lock"; import { serialized } from "./lib/src/lock";
@@ -305,7 +305,8 @@ export class ConfigSync extends LiveSyncCommands {
} }
return false; return false;
} }
createMissingConfigurationEntry() { createMissingConfigurationEntry = throttle(() => this._createMissingConfigurationEntry(), 1000);
_createMissingConfigurationEntry() {
let saveRequired = false; let saveRequired = false;
for (const v of this.pluginList) { for (const v of this.pluginList) {
const key = `${v.category}/${v.name}`; const key = `${v.category}/${v.name}`;
@@ -349,8 +350,7 @@ export class ConfigSync extends LiveSyncCommands {
Logger(ex, LOG_LEVEL_VERBOSE); Logger(ex, LOG_LEVEL_VERBOSE);
} }
return []; return [];
}, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline().root.onIdle(() => { }, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline().root.onUpdateProgress(() => {
// Logger(`All files enumerated`, LOG_LEVEL_INFO, "get-plugins");
this.createMissingConfigurationEntry(); this.createMissingConfigurationEntry();
}); });

View File

@@ -9,7 +9,7 @@ import { serialized } from "./lib/src/lock";
import { JsonResolveModal } from "./JsonResolveModal"; import { JsonResolveModal } from "./JsonResolveModal";
import { LiveSyncCommands } from "./LiveSyncCommands"; import { LiveSyncCommands } from "./LiveSyncCommands";
import { addPrefix, stripAllPrefixes } from "./lib/src/path"; import { addPrefix, stripAllPrefixes } from "./lib/src/path";
import { KeyedQueueProcessor, QueueProcessor } from "./lib/src/processor"; import { QueueProcessor } from "./lib/src/processor";
import { hiddenFilesEventCount, hiddenFilesProcessingCount } from "./lib/src/stores"; import { hiddenFilesEventCount, hiddenFilesProcessingCount } from "./lib/src/stores";
export class HiddenFileSync extends LiveSyncCommands { export class HiddenFileSync extends LiveSyncCommands {
@@ -73,15 +73,15 @@ export class HiddenFileSync extends LiveSyncCommands {
} }
procInternalFile(filename: string) { procInternalFile(filename: string) {
this.internalFileProcessor.enqueueWithKey(filename, filename); this.internalFileProcessor.enqueue(filename);
} }
internalFileProcessor = new KeyedQueueProcessor<string, any>( internalFileProcessor = new QueueProcessor<string, any>(
async (filenames) => { async (filenames) => {
Logger(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); Logger(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
await this.syncInternalFilesAndDatabase("pull", false, false, filenames); await this.syncInternalFilesAndDatabase("pull", false, false, filenames);
Logger(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); Logger(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
return; return;
}, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 10, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount } }, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 100, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount }
); );
recentProcessedInternalFiles = [] as string[]; recentProcessedInternalFiles = [] as string[];

View File

@@ -50,7 +50,7 @@ export class SetupLiveSync extends LiveSyncCommands {
const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "", true); const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "", true);
if (encryptingPassphrase === false) if (encryptingPassphrase === false)
return; return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" }; const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" } as Partial<ObsidianLiveSyncSettings>;
if (stripExtra) { if (stripExtra) {
delete setting.pluginSyncExtendedSetting; delete setting.pluginSyncExtendedSetting;
} }
@@ -377,9 +377,6 @@ Of course, we are able to disable these features.`
await this.plugin.replicateAllFromServer(true); await this.plugin.replicateAllFromServer(true);
await delay(1000); await delay(1000);
await this.plugin.replicateAllFromServer(true); await this.plugin.replicateAllFromServer(true);
// if (!tryLessFetching) {
// await this.fetchRemoteChunks();
// }
await this.resumeReflectingDatabase(); await this.resumeReflectingDatabase();
await this.askHiddenFileConfiguration({ enableFetch: true }); await this.askHiddenFileConfiguration({ enableFetch: true });
} }

View File

@@ -6,7 +6,7 @@ export interface KeyValueDatabase {
clear(): Promise<void>; clear(): Promise<void>;
keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>; keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>;
close(): void; close(): void;
destroy(): void; destroy(): Promise<void>;
} }
const databaseCache: { [key: string]: IDBPDatabase<any> } = {}; const databaseCache: { [key: string]: IDBPDatabase<any> } = {};
export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatabase> => { export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatabase> => {
@@ -20,8 +20,7 @@ export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatab
db.createObjectStore(storeKey); db.createObjectStore(storeKey);
}, },
}); });
let db: IDBPDatabase<any> = null; const db = await dbPromise;
db = await dbPromise;
databaseCache[dbKey] = db; databaseCache[dbKey] = db;
return { return {
get<T>(key: string): Promise<T> { get<T>(key: string): Promise<T> {

View File

@@ -0,0 +1,83 @@
<script lang="ts">
export let patterns = [] as string[];
export let originals = [] as string[];
export let apply: (args: string[]) => Promise<void> = (_: string[]) => Promise.resolve();
function revert() {
patterns = [...originals];
}
const CHECK_OK = "✔";
const CHECK_NG = "⚠";
const MARK_MODIFIED = "✏ ";
function checkRegExp(pattern: string) {
if (pattern.trim() == "") return "";
try {
const _ = new RegExp(pattern);
return CHECK_OK;
} catch (ex) {
return CHECK_NG;
}
}
$: status = patterns.map((e) => checkRegExp(e));
$: modified = patterns.map((e, i) => (e != originals?.[i] ?? "" ? MARK_MODIFIED : ""));
function remove(idx: number) {
patterns[idx] = "";
}
function add() {
patterns = [...patterns, ""];
}
</script>
<ul>
{#each patterns as pattern, idx}
<li><label>{modified[idx]}{status[idx]}</label><input type="text" bind:value={pattern} class={modified[idx]} /><button class="iconbutton" on:click={() => remove(idx)}>🗑</button></li>
{/each}
<li>
<label><button on:click={() => add()}>Add</button></label>
</li>
<li class="buttons">
<button on:click={() => apply(patterns)} disabled={status.some((e) => e == CHECK_NG) || modified.every((e) => e == "")}>Apply</button>
<button on:click={() => revert()} disabled={status.some((e) => e == CHECK_NG) || modified.every((e) => e == "")}>Revert</button>
</li>
</ul>
<style>
label {
min-width: 4em;
width: 4em;
display: inline-flex;
flex-direction: row;
justify-content: flex-end;
}
ul {
flex-grow: 1;
display: inline-flex;
flex-direction: column;
list-style-type: none;
margin-block-start: 0;
margin-block-end: 0;
margin-inline-start: 0px;
margin-inline-end: 0px;
padding-inline-start: 0;
}
li {
padding: var(--size-2-1) var(--size-4-1);
display: inline-flex;
flex-grow: 1;
align-items: center;
justify-content: flex-end;
gap: var(--size-4-2);
}
li input {
min-width: 10em;
}
li.buttons {
}
button.iconbutton {
max-width: 4em;
}
span.spacer {
flex-grow: 1;
}
</style>

View File

@@ -1,5 +1,5 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps"; import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, MarkdownRenderer, stringifyYaml } from "./deps";
import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED } from "./lib/src/types"; import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR } from "./lib/src/types";
import { createBlob, delay, isDocContentSame, readAsBlob } from "./lib/src/utils"; import { createBlob, delay, isDocContentSame, readAsBlob } from "./lib/src/utils";
import { versionNumberString2Number } from "./lib/src/strbin"; import { versionNumberString2Number } from "./lib/src/strbin";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
@@ -9,6 +9,7 @@ import ObsidianLiveSyncPlugin from "./main";
import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils"; import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils";
import { request, type ButtonComponent, TFile } from "obsidian"; import { request, type ButtonComponent, TFile } from "obsidian";
import { shouldBeIgnored } from "./lib/src/path"; import { shouldBeIgnored } from "./lib/src/path";
import MultipleRegExpControl from './MultipleRegExpControl.svelte';
export class ObsidianLiveSyncSettingTab extends PluginSettingTab { export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -46,11 +47,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
let useDynamicIterationCount = this.plugin.settings.useDynamicIterationCount; let useDynamicIterationCount = this.plugin.settings.useDynamicIterationCount;
containerEl.empty(); containerEl.empty();
// const preferred_setting = isCloudantURI(this.plugin.settings.couchDB_URI) ? PREFERRED_SETTING_CLOUDANT : PREFERRED_SETTING_SELF_HOSTED;
// const default_setting = { ...DEFAULT_SETTINGS };
containerEl.createEl("h2", { text: "Settings for Self-hosted LiveSync." }); containerEl.createEl("h2", { text: "Settings for Self-hosted LiveSync." });
containerEl.addClass("sls-setting"); containerEl.addClass("sls-setting");
containerEl.removeClass("isWizard"); containerEl.removeClass("isWizard");
@@ -1342,43 +1338,48 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}); });
text.inputEl.setAttribute("type", "number"); text.inputEl.setAttribute("type", "number");
}); });
let skipPatternTextArea: TextAreaComponent; const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, ^\\.git\\/, \\/obsidian-livesync\\/";
const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, \\/obsidian-livesync\\/";
const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$,\\/workspace-mobile.json$"; const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$,\\/workspace-mobile.json$";
new Setting(containerSyncSettingEl)
.setName("Folders and files to ignore") const pat = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != "");
.setDesc( const patSetting = new Setting(containerSyncSettingEl)
"Regular expression, If you use hidden file sync between desktop and mobile, adding `workspace$` is recommended." .setName("Hidden files ignore patterns")
) .setDesc("");
.setClass("wizardHidden")
.addTextArea((text) => { new MultipleRegExpControl(
text {
.setValue(this.plugin.settings.syncInternalFilesIgnorePatterns) target: patSetting.controlEl,
.setPlaceholder("\\/node_modules\\/, \\/\\.git\\/") props: {
.onChange(async (value) => { patterns: pat, originals: [...pat], apply: async (newPatterns) => {
this.plugin.settings.syncInternalFilesIgnorePatterns = value; this.plugin.settings.syncInternalFilesIgnorePatterns = newPatterns.map(e => e.trim()).filter(e => e != "").join(", ");
await this.plugin.saveSettings(); await this.plugin.saveSettings();
}) this.display();
skipPatternTextArea = text; }
return text; }
} }
); )
const addDefaultPatterns = async (patterns: string) => {
const oldList = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != "");
const newList = patterns.split(",").map(x => x.trim()).filter(x => x != "");
const allSet = new Set([...oldList, ...newList]);
this.plugin.settings.syncInternalFilesIgnorePatterns = [...allSet].join(", ");
await this.plugin.saveSettings();
this.display();
}
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Restore the skip pattern to default") .setName("Add default patterns")
.setClass("wizardHidden") .setClass("wizardHidden")
.addButton((button) => { .addButton((button) => {
button.setButtonText("Default") button.setButtonText("Default")
.onClick(async () => { .onClick(async () => {
skipPatternTextArea.setValue(defaultSkipPattern); await addDefaultPatterns(defaultSkipPattern);
this.plugin.settings.syncInternalFilesIgnorePatterns = defaultSkipPattern;
await this.plugin.saveSettings();
}) })
}).addButton((button) => { }).addButton((button) => {
button.setButtonText("Cross-platform") button.setButtonText("Cross-platform")
.onClick(async () => { .onClick(async () => {
skipPatternTextArea.setValue(defaultSkipPatternXPlat); await addDefaultPatterns(defaultSkipPatternXPlat);
this.plugin.settings.syncInternalFilesIgnorePatterns = defaultSkipPatternXPlat;
await this.plugin.saveSettings();
}) })
}) })
@@ -1430,54 +1431,41 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
containerSyncSettingEl.createEl("h4", { containerSyncSettingEl.createEl("h4", {
text: sanitizeHTMLToDom(`Targets`), text: sanitizeHTMLToDom(`Targets`),
}).addClass("wizardHidden"); }).addClass("wizardHidden");
new Setting(containerSyncSettingEl)
const syncFilesSetting = new Setting(containerSyncSettingEl)
.setName("Synchronising files") .setName("Synchronising files")
.setDesc("(RegExp) Empty to sync all files. set filter as a regular expression to limit synchronising files.") .setDesc("(RegExp) Empty to sync all files. set filter as a regular expression to limit synchronising files.")
.setClass("wizardHidden") .setClass("wizardHidden")
.addTextArea((text) => { new MultipleRegExpControl(
text {
.setValue(this.plugin.settings.syncOnlyRegEx) target: syncFilesSetting.controlEl,
.setPlaceholder("\\.md$|\\.txt") props: {
.onChange(async (value) => { patterns: this.plugin.settings.syncOnlyRegEx.split("|[]|"), originals: [...this.plugin.settings.syncOnlyRegEx.split("|[]|")], apply: async (newPatterns) => {
let isValidRegExp = false; this.plugin.settings.syncOnlyRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|");
try { await this.plugin.saveSettings();
new RegExp(value); this.display();
isValidRegExp = true; }
} catch (_) { }
// NO OP.
}
if (isValidRegExp || value.trim() == "") {
this.plugin.settings.syncOnlyRegEx = value;
await this.plugin.saveSettings();
}
})
return text;
} }
); )
new Setting(containerSyncSettingEl)
const nonSyncFilesSetting = new Setting(containerSyncSettingEl)
.setName("Non-Synchronising files") .setName("Non-Synchronising files")
.setDesc("(RegExp) If this is set, any changes to local and remote files that match this will be skipped.") .setDesc("(RegExp) If this is set, any changes to local and remote files that match this will be skipped.")
.setClass("wizardHidden") .setClass("wizardHidden");
.addTextArea((text) => {
text new MultipleRegExpControl(
.setValue(this.plugin.settings.syncIgnoreRegEx) {
.setPlaceholder("\\.pdf$") target: nonSyncFilesSetting.controlEl,
.onChange(async (value) => { props: {
let isValidRegExp = false; patterns: this.plugin.settings.syncIgnoreRegEx.split("|[]|"), originals: [...this.plugin.settings.syncIgnoreRegEx.split("|[]|")], apply: async (newPatterns) => {
try { this.plugin.settings.syncIgnoreRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|");
new RegExp(value); await this.plugin.saveSettings();
isValidRegExp = true; this.display();
} catch (_) { }
// NO OP. }
}
if (isValidRegExp || value.trim() == "") {
this.plugin.settings.syncIgnoreRegEx = value;
await this.plugin.saveSettings();
}
})
return text;
} }
); )
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Maximum file size") .setName("Maximum file size")
.setDesc("(MB) If this is set, changes to local and remote files that are larger than this will be skipped. If the file becomes smaller again, a newer one will be used.") .setDesc("(MB) If this is set, changes to local and remote files that are larger than this will be skipped. If the file becomes smaller again, a newer one will be used.")
@@ -2173,6 +2161,15 @@ ${stringifyYaml(pluginConfig)}`;
.setButtonText("Fetch") .setButtonText("Fetch")
.setWarning() .setWarning()
.setDisabled(false) .setDisabled(false)
.onClick(async () => {
await this.plugin.vaultAccess.vaultCreate(FLAGMD_REDFLAG3_HR, "");
this.plugin.performAppReload();
})
).addButton((button) =>
button
.setButtonText("Fetch w/o restarting")
.setWarning()
.setDisabled(false)
.onClick(async () => { .onClick(async () => {
await rebuildDB("localOnly"); await rebuildDB("localOnly");
}) })
@@ -2232,10 +2229,21 @@ ${stringifyYaml(pluginConfig)}`;
.setButtonText("Rebuild") .setButtonText("Rebuild")
.setWarning() .setWarning()
.setDisabled(false) .setDisabled(false)
.onClick(async () => {
await this.plugin.vaultAccess.vaultCreate(FLAGMD_REDFLAG2_HR, "");
this.plugin.performAppReload();
})
)
.addButton((button) =>
button
.setButtonText("Rebuild w/o restarting")
.setWarning()
.setDisabled(false)
.onClick(async () => { .onClick(async () => {
await rebuildDB("rebuildBothByThisDevice"); await rebuildDB("rebuildBothByThisDevice");
}) })
) )
applyDisplayEnabled(); applyDisplayEnabled();
addScreenElement("70", containerMaintenanceEl); addScreenElement("70", containerMaintenanceEl);

View File

@@ -2,7 +2,7 @@ import type { SerializedFileAccess } from "./SerializedFileAccess";
import { Plugin, TAbstractFile, TFile, TFolder } from "./deps"; import { Plugin, TAbstractFile, TFile, TFolder } from "./deps";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { shouldBeIgnored } from "./lib/src/path"; import { shouldBeIgnored } from "./lib/src/path";
import type { KeyedQueueProcessor } from "./lib/src/processor"; import type { QueueProcessor } from "./lib/src/processor";
import { LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "./lib/src/types"; import { LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "./lib/src/types";
import { delay } from "./lib/src/utils"; import { delay } from "./lib/src/utils";
import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "./types"; import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "./types";
@@ -19,7 +19,7 @@ type LiveSyncForStorageEventManager = Plugin &
vaultAccess: SerializedFileAccess vaultAccess: SerializedFileAccess
} & { } & {
isTargetFile: (file: string | TAbstractFile) => Promise<boolean>, isTargetFile: (file: string | TAbstractFile) => Promise<boolean>,
fileEventQueue: KeyedQueueProcessor<FileEventItem, any>, fileEventQueue: QueueProcessor<FileEventItem, any>,
isFileSizeExceeded: (size: number) => boolean; isFileSizeExceeded: (size: number) => boolean;
}; };
@@ -133,8 +133,7 @@ export class StorageEventManagerObsidian extends StorageEventManager {
path: file.path, path: file.path,
size: file.stat.size size: file.stat.size
} as FileInfo : file as InternalFileInfo; } as FileInfo : file as InternalFileInfo;
this.plugin.fileEventQueue.enqueue({
this.plugin.fileEventQueue.enqueueWithKey(`file-${fileInfo.path}`, {
type, type,
args: { args: {
file: fileInfo, file: fileInfo,

Submodule src/lib updated: 98809f37df...0d217242a8

View File

@@ -4,7 +4,7 @@ import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stri
import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types"; import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types";
import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types"; import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
import { arrayToChunkedArray, createBlob, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue } from "./lib/src/utils"; import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils";
import { Logger, setGlobalLogFunction } from "./lib/src/logger"; import { Logger, setGlobalLogFunction } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { ConflictResolveModal } from "./ConflictResolveModal"; import { ConflictResolveModal } from "./ConflictResolveModal";
@@ -31,7 +31,7 @@ import { GlobalHistoryView, VIEW_TYPE_GLOBAL_HISTORY } from "./GlobalHistoryView
import { LogPaneView, VIEW_TYPE_LOG } from "./LogPaneView"; import { LogPaneView, VIEW_TYPE_LOG } from "./LogPaneView";
import { LRUCache } from "./lib/src/LRUCache"; import { LRUCache } from "./lib/src/LRUCache";
import { SerializedFileAccess } from "./SerializedFileAccess.js"; import { SerializedFileAccess } from "./SerializedFileAccess.js";
import { KeyedQueueProcessor, QueueProcessor, type QueueItemWithKey } from "./lib/src/processor.js"; import { QueueProcessor } from "./lib/src/processor.js";
import { reactive, reactiveSource } from "./lib/src/reactive.js"; import { reactive, reactiveSource } from "./lib/src/reactive.js";
import { initializeStores } from "./stores.js"; import { initializeStores } from "./stores.js";
@@ -312,8 +312,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
this.replicator = new LiveSyncDBReplicator(this); this.replicator = new LiveSyncDBReplicator(this);
} }
async onResetDatabase(db: LiveSyncLocalDB): Promise<void> { async onResetDatabase(db: LiveSyncLocalDB): Promise<void> {
const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName(); const kvDBKey = "queued-files"
localStorage.removeItem(lsKey); this.kvDB.del(kvDBKey);
// localStorage.removeItem(lsKey);
await this.kvDB.destroy(); await this.kvDB.destroy();
this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv");
this.replicator = new LiveSyncDBReplicator(this); this.replicator = new LiveSyncDBReplicator(this);
@@ -535,7 +536,7 @@ Click anywhere to stop counting down.
this.registerWatchEvents(); this.registerWatchEvents();
await this.realizeSettingSyncMode(); await this.realizeSettingSyncMode();
this.swapSaveCommand(); this.swapSaveCommand();
if (this.settings.syncOnStart) { if (!this.settings.liveSync && this.settings.syncOnStart) {
this.replicator.openReplication(this.settings, false, false); this.replicator.openReplication(this.settings, false, false);
} }
this.scanStat(); this.scanStat();
@@ -1007,7 +1008,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
} }
this.deviceAndVaultName = localStorage.getItem(lsKey) || ""; this.deviceAndVaultName = localStorage.getItem(lsKey) || "";
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
this.fileEventQueue.delay = this.settings.batchSave ? 5000 : 100; this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100;
} }
async saveSettingData() { async saveSettingData() {
@@ -1039,7 +1040,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
} }
await this.saveData(settings); await this.saveData(settings);
this.localDatabase.settings = this.settings; this.localDatabase.settings = this.settings;
this.fileEventQueue.delay = this.settings.batchSave ? 5000 : 100; this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100;
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
if (this.settings.settingSyncFile != "") { if (this.settings.settingSyncFile != "") {
fireAndForget(() => this.saveSettingToMarkdown(this.settings.settingSyncFile)); fireAndForget(() => this.saveSettingToMarkdown(this.settings.settingSyncFile));
@@ -1237,9 +1238,13 @@ We can perform a command in this file.
_this.performCommand('editor:save-file'); _this.performCommand('editor:save-file');
}; };
} }
hasFocus = true;
isLastHidden = false;
registerWatchEvents() { registerWatchEvents() {
this.registerEvent(this.app.workspace.on("file-open", this.watchWorkspaceOpen)); this.registerEvent(this.app.workspace.on("file-open", this.watchWorkspaceOpen));
this.registerDomEvent(document, "visibilitychange", this.watchWindowVisibility); this.registerDomEvent(document, "visibilitychange", this.watchWindowVisibility);
this.registerDomEvent(window, "focus", () => this.setHasFocus(true));
this.registerDomEvent(window, "blur", () => this.setHasFocus(false));
this.registerDomEvent(window, "online", this.watchOnline); this.registerDomEvent(window, "online", this.watchOnline);
this.registerDomEvent(window, "offline", this.watchOnline); this.registerDomEvent(window, "offline", this.watchOnline);
} }
@@ -1255,15 +1260,30 @@ We can perform a command in this file.
await this.syncAllFiles(); await this.syncAllFiles();
} }
} }
setHasFocus(hasFocus: boolean) {
this.hasFocus = hasFocus;
this.watchWindowVisibility();
}
watchWindowVisibility() { watchWindowVisibility() {
scheduleTask("watch-window-visibility", 500, () => fireAndForget(() => this.watchWindowVisibilityAsync())); scheduleTask("watch-window-visibility", 100, () => fireAndForget(() => this.watchWindowVisibilityAsync()));
} }
async watchWindowVisibilityAsync() { async watchWindowVisibilityAsync() {
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return; if (!this.settings.isConfigured) return;
if (!this.isReady) return; if (!this.isReady) return;
if (this.isLastHidden && !this.hasFocus) {
// NO OP while non-focused after made hidden;
return;
}
const isHidden = document.hidden; const isHidden = document.hidden;
if (this.isLastHidden === isHidden) {
return;
}
this.isLastHidden = isHidden;
await this.applyBatchChange(); await this.applyBatchChange();
if (isHidden) { if (isHidden) {
this.replicator.closeReplication(); this.replicator.closeReplication();
@@ -1283,12 +1303,12 @@ We can perform a command in this file.
} }
cancelRelativeEvent(item: FileEventItem) { cancelRelativeEvent(item: FileEventItem) {
this.fileEventQueue.modifyQueue((items) => [...items.filter(e => e.entity.key != item.key)]) this.fileEventQueue.modifyQueue((items) => [...items.filter(e => e.key != item.key)])
} }
queueNextFileEvent(items: QueueItemWithKey<FileEventItem>[], newItem: QueueItemWithKey<FileEventItem>): QueueItemWithKey<FileEventItem>[] { queueNextFileEvent(items: FileEventItem[], newItem: FileEventItem): FileEventItem[] {
if (this.settings.batchSave && !this.settings.liveSync) { if (this.settings.batchSave && !this.settings.liveSync) {
const file = newItem.entity.args.file; const file = newItem.args.file;
// if the latest event is the same type, omit that // if the latest event is the same type, omit that
// a.md MODIFY <- this should be cancelled when a.md MODIFIED // a.md MODIFY <- this should be cancelled when a.md MODIFIED
// b.md MODIFY <- this should be cancelled when b.md MODIFIED // b.md MODIFY <- this should be cancelled when b.md MODIFIED
@@ -1300,16 +1320,16 @@ We can perform a command in this file.
while (i >= 0) { while (i >= 0) {
i--; i--;
if (i < 0) break L1; if (i < 0) break L1;
if (items[i].entity.args.file.path != file.path) { if (items[i].args.file.path != file.path) {
continue L1; continue L1;
} }
if (items[i].entity.type != newItem.entity.type) break L1; if (items[i].type != newItem.type) break L1;
items.remove(items[i]); items.remove(items[i]);
} }
} }
items.push(newItem); items.push(newItem);
// When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition. // When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition.
if (newItem.entity.type == "DELETE" || newItem.entity.type == "RENAME") { if (newItem.type == "DELETE" || newItem.type == "RENAME") {
this.fileEventQueue.requestNextFlush(); this.fileEventQueue.requestNextFlush();
} }
return items; return items;
@@ -1363,7 +1383,7 @@ We can perform a command in this file.
pendingFileEventCount = reactiveSource(0); pendingFileEventCount = reactiveSource(0);
processingFileEventCount = reactiveSource(0); processingFileEventCount = reactiveSource(0);
fileEventQueue = fileEventQueue =
new KeyedQueueProcessor( new QueueProcessor(
(items: FileEventItem[]) => this.handleFileEvent(items[0]), (items: FileEventItem[]) => this.handleFileEvent(items[0]),
{ suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount } { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount }
).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem)); ).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem));
@@ -1622,21 +1642,32 @@ We can perform a command in this file.
this.conflictCheckQueue.enqueue(path); this.conflictCheckQueue.enqueue(path);
} }
_saveQueuedFiles = throttle(() => {
const saveData = this.replicationResultProcessor._queue.filter(e => e !== undefined && e !== null).map((e) => e?._id ?? "" as string) as string[];
const kvDBKey = "queued-files"
// localStorage.setItem(lsKey, saveData);
fireAndForget(() => this.kvDB.set(kvDBKey, saveData));
}, 100);
saveQueuedFiles() { saveQueuedFiles() {
const saveData = JSON.stringify(this.replicationResultProcessor._queue.map((e) => e._id)); this._saveQueuedFiles();
const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName();
localStorage.setItem(lsKey, saveData);
} }
async loadQueuedFiles() { async loadQueuedFiles() {
if (this.settings.suspendParseReplicationResult) return; if (this.settings.suspendParseReplicationResult) return;
if (!this.settings.isConfigured) return; if (!this.settings.isConfigured) return;
const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName(); const kvDBKey = "queued-files"
const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[]; // const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
const ids = [...new Set(await this.kvDB.get<string[]>(kvDBKey) ?? [])];
const batchSize = 100; const batchSize = 100;
const chunkedIds = arrayToChunkedArray(ids, batchSize); const chunkedIds = arrayToChunkedArray(ids, batchSize);
for await (const idsBatch of chunkedIds) { for await (const idsBatch of chunkedIds) {
const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: idsBatch, include_docs: true, limit: 100 }); const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: idsBatch, include_docs: true, limit: 100 });
this.replicationResultProcessor.enqueueAll(ret.rows.map(doc => doc.doc!)); const docs = ret.rows.filter(e => e.doc).map(e => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
const errors = ret.rows.filter(e => !e.doc && !e.value.deleted);
if (errors.length > 0) {
Logger("Some queued processes were not resurrected");
Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE);
}
this.replicationResultProcessor.enqueueAll(docs);
await this.replicationResultProcessor.waitForPipeline(); await this.replicationResultProcessor.waitForPipeline();
} }
@@ -1658,34 +1689,43 @@ We can perform a command in this file.
const filename = this.getPathWithoutPrefix(doc); const filename = this.getPathWithoutPrefix(doc);
this.isTargetFile(filename).then((ret) => ret ? this.addOnHiddenFileSync.procInternalFile(filename) : Logger(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE)); this.isTargetFile(filename).then((ret) => ret ? this.addOnHiddenFileSync.procInternalFile(filename) : Logger(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE));
} else if (isValidPath(this.getPath(doc))) { } else if (isValidPath(this.getPath(doc))) {
this.storageApplyingProcessor.enqueueWithKey(doc.path, doc); this.storageApplyingProcessor.enqueue(doc);
} else { } else {
Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE); Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE);
} }
return; return;
}, { suspended: true, batchSize: 1, concurrentLimit: 10, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.databaseQueueCount }).startPipeline(); }, { suspended: true, batchSize: 1, concurrentLimit: 10, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.databaseQueueCount }).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline();
storageApplyingCount = reactiveSource(0); storageApplyingCount = reactiveSource(0);
storageApplyingProcessor = new KeyedQueueProcessor(async (docs: LoadedEntry[]) => { storageApplyingProcessor = new QueueProcessor(async (docs: LoadedEntry[]) => {
const entry = docs[0]; const entry = docs[0];
const path = this.getPath(entry); await serialized(entry.path, async () => {
Logger(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE); const path = this.getPath(entry);
const targetFile = this.vaultAccess.getAbstractFileByPath(this.getPathWithoutPrefix(entry)); Logger(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE);
if (targetFile instanceof TFolder) { const targetFile = this.vaultAccess.getAbstractFileByPath(this.getPathWithoutPrefix(entry));
Logger(`${this.getPath(entry)} is already exist as the folder`); if (targetFile instanceof TFolder) {
} else { Logger(`${this.getPath(entry)} is already exist as the folder`);
await this.processEntryDoc(entry, targetFile instanceof TFile ? targetFile : undefined); } else {
Logger(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`); await this.processEntryDoc(entry, targetFile instanceof TFile ? targetFile : undefined);
} Logger(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`);
}
});
return; return;
}, { suspended: true, batchSize: 1, concurrentLimit: 2, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.storageApplyingCount }).startPipeline() }, { suspended: true, batchSize: 1, concurrentLimit: 6, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.storageApplyingCount }).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline()
replicationResultCount = reactiveSource(0); replicationResultCount = reactiveSource(0);
replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => { replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
if (this.settings.suspendParseReplicationResult) return; if (this.settings.suspendParseReplicationResult) return;
const change = docs[0]; const change = docs[0];
if (!change) return;
if (isChunk(change._id)) { if (isChunk(change._id)) {
// SendSignal? // SendSignal?
// this.parseIncomingChunk(change); // this.parseIncomingChunk(change);
@@ -1722,16 +1762,19 @@ We can perform a command in this file.
this.databaseQueuedProcessor.enqueue(change); this.databaseQueuedProcessor.enqueue(change);
} }
return; return;
}, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).startPipeline().onUpdateProgress(() => { }, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline().onUpdateProgress(() => {
this.saveQueuedFiles(); this.saveQueuedFiles();
}); });
//---> Sync //---> Sync
parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>) { parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>) {
if (this.settings.suspendParseReplicationResult) { if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.suspend() this.replicationResultProcessor.suspend()
} }
this.replicationResultProcessor.enqueueAll(docs); this.replicationResultProcessor.enqueueAll(docs);
if (!this.settings.suspendParseReplicationResult) { if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume() this.replicationResultProcessor.resume()
} }
} }
@@ -1761,8 +1804,33 @@ We can perform a command in this file.
lastMessage = ""; lastMessage = "";
observeForLogs() { observeForLogs() {
const padSpaces = `\u{2007}`.repeat(10);
// const emptyMark = `\u{2003}`;
const rerenderTimer = new Map<string, [ReturnType<typeof setTimeout>, number]>;
const tick = reactiveSource(0);
function padLeftSp(num: number, mark: string) {
const numLen = `${num}`.length + 1;
const [timer, len] = rerenderTimer.get(mark) ?? [undefined, numLen];
if (num || timer) {
if (num) {
if (timer) clearTimeout(timer);
rerenderTimer.set(mark, [setTimeout(async () => {
rerenderTimer.delete(mark);
await delay(100);
tick.value = tick.value + 1;
}, 3000), Math.max(len, numLen)]);
}
return ` ${mark}${`${padSpaces}${num}`.slice(-(len))}`;
} else {
return "";
}
}
// const logStore // const logStore
const queueCountLabel = reactive(() => { const queueCountLabel = reactive(() => {
// For invalidating
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const _ = tick.value;
const dbCount = this.databaseQueueCount.value; const dbCount = this.databaseQueueCount.value;
const replicationCount = this.replicationResultCount.value; const replicationCount = this.replicationResultCount.value;
const storageApplyingCount = this.storageApplyingCount.value; const storageApplyingCount = this.storageApplyingCount.value;
@@ -1770,13 +1838,13 @@ We can perform a command in this file.
const pluginScanCount = pluginScanningCount.value; const pluginScanCount = pluginScanningCount.value;
const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value; const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value;
const conflictProcessCount = this.conflictProcessQueueCount.value; const conflictProcessCount = this.conflictProcessQueueCount.value;
const labelReplication = replicationCount ? `📥 ${replicationCount} ` : ""; const labelReplication = padLeftSp(replicationCount, `📥`);
const labelDBCount = dbCount ? `📄 ${dbCount} ` : ""; const labelDBCount = padLeftSp(dbCount, `📄`);
const labelStorageCount = storageApplyingCount ? `💾 ${storageApplyingCount}` : ""; const labelStorageCount = padLeftSp(storageApplyingCount, `💾`);
const labelChunkCount = chunkCount ? `🧩${chunkCount} ` : ""; const labelChunkCount = padLeftSp(chunkCount, `🧩`);
const labelPluginScanCount = pluginScanCount ? `🔌${pluginScanCount} ` : ""; const labelPluginScanCount = padLeftSp(pluginScanCount, `🔌`);
const labelHiddenFilesCount = hiddenFilesCount ? `⚙️${hiddenFilesCount} ` : ""; const labelHiddenFilesCount = padLeftSp(hiddenFilesCount, `⚙️`)
const labelConflictProcessCount = conflictProcessCount ? `🔩${conflictProcessCount} ` : ""; const labelConflictProcessCount = padLeftSp(conflictProcessCount, `🔩`);
return `${labelReplication}${labelDBCount}${labelStorageCount}${labelChunkCount}${labelPluginScanCount}${labelHiddenFilesCount}${labelConflictProcessCount}`; return `${labelReplication}${labelDBCount}${labelStorageCount}${labelChunkCount}${labelPluginScanCount}${labelHiddenFilesCount}${labelConflictProcessCount}`;
}) })
const requestingStatLabel = reactive(() => { const requestingStatLabel = reactive(() => {
@@ -1821,11 +1889,15 @@ We can perform a command in this file.
return { w, sent, pushLast, arrived, pullLast }; return { w, sent, pushLast, arrived, pullLast };
}) })
const waitingLabel = reactive(() => { const waitingLabel = reactive(() => {
// For invalidating
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const _ = tick.value;
const e = this.pendingFileEventCount.value; const e = this.pendingFileEventCount.value;
const proc = this.processingFileEventCount.value; const proc = this.processingFileEventCount.value;
const pend = e - proc; const pend = e - proc;
const labelProc = proc != 0 ? `${proc} ` : ""; const labelProc = padLeftSp(proc, ``);
const labelPend = pend != 0 ? ` 🛫${pend}` : ""; const labelPend = padLeftSp(pend, `🛫`);
return `${labelProc}${labelPend}`; return `${labelProc}${labelPend}`;
}) })
const statusLineLabel = reactive(() => { const statusLineLabel = reactive(() => {
@@ -1834,7 +1906,7 @@ We can perform a command in this file.
const waiting = waitingLabel.value; const waiting = waitingLabel.value;
const networkActivity = requestingStatLabel.value; const networkActivity = requestingStatLabel.value;
return { return {
message: `${networkActivity}Sync: ${w}${sent}${pushLast}${arrived}${pullLast}${waiting} ${queued}`, message: `${networkActivity}Sync: ${w} ${sent}${pushLast} ${arrived}${pullLast}${waiting}${queued}`,
}; };
}) })
const statusBarLabels = reactive(() => { const statusBarLabels = reactive(() => {
@@ -1845,31 +1917,20 @@ We can perform a command in this file.
message, status message, status
} }
}) })
let last = 0;
const applyToDisplay = () => { const applyToDisplay = throttle(() => {
const v = statusBarLabels.value; const v = statusBarLabels.value;
const now = Date.now();
if (now - last < 10) {
scheduleTask("applyToDisplay", 20, () => applyToDisplay());
return;
}
this.applyStatusBarText(v.message, v.status); this.applyStatusBarText(v.message, v.status);
last = now;
} }, 20);
statusBarLabels.onChanged(applyToDisplay); statusBarLabels.onChanged(applyToDisplay);
} }
applyStatusBarText(message: string, log: string) { applyStatusBarText(message: string, log: string) {
const newMsg = message; const newMsg = message.replace(/\n/g, "\\A ");
const newLog = log; const newLog = log.replace(/\n/g, "\\A ");
// scheduleTask("update-display", 50, () => {
this.statusBar?.setText(newMsg.split("\n")[0]); this.statusBar?.setText(newMsg.split("\n")[0]);
// const selector = `.CodeMirror-wrap,` +
// `.markdown-preview-view.cm-s-obsidian,` +
// `.markdown-source-view.cm-s-obsidian,` +
// `.canvas-wrapper,` +
// `.empty-state`
// ;
if (this.settings.showStatusOnEditor) { if (this.settings.showStatusOnEditor) {
const root = activeDocument.documentElement; const root = activeDocument.documentElement;
root.style.setProperty("--sls-log-text", "'" + (newMsg + "\\A " + newLog) + "'"); root.style.setProperty("--sls-log-text", "'" + (newMsg + "\\A " + newLog) + "'");
@@ -1877,7 +1938,6 @@ We can perform a command in this file.
// const root = activeDocument.documentElement; // const root = activeDocument.documentElement;
// root.style.setProperty("--log-text", "'" + (newMsg + "\\A " + newLog) + "'"); // root.style.setProperty("--log-text", "'" + (newMsg + "\\A " + newLog) + "'");
} }
// }, true);
scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" }); scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" });
@@ -2053,9 +2113,15 @@ Or if you are sure know what had been happened, we can unlock the database from
const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1); const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
Logger("Updating database by new files"); Logger("Updating database by new files");
const processStatus = {} as Record<string, string>;
const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
const updateLog = throttle((key: string, msg: string) => {
processStatus[key] = msg;
const log = Object.values(processStatus).join("\n");
Logger(log, logLevel, "syncAll");
}, 25);
const initProcess = []; const initProcess = [];
const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
const runAll = async<T>(procedureName: string, objects: T[], callback: (arg: T) => Promise<void>) => { const runAll = async<T>(procedureName: string, objects: T[], callback: (arg: T) => Promise<void>) => {
if (objects.length == 0) { if (objects.length == 0) {
Logger(`${procedureName}: Nothing to do`); Logger(`${procedureName}: Nothing to do`);
@@ -2077,12 +2143,14 @@ Or if you are sure know what had been happened, we can unlock the database from
failed++; failed++;
} }
if ((success + failed) % step == 0) { if ((success + failed) % step == 0) {
Logger(`${procedureName}: DONE:${success}, FAILED:${failed}, LAST:${processor._queue.length}`, logLevel, `log-${procedureName}`); const msg = `${procedureName}: DONE:${success}, FAILED:${failed}, LAST:${processor._queue.length}`;
updateLog(procedureName, msg);
} }
return; return;
}, { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, objects) }, { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, objects)
await processor.waitForPipeline(); await processor.waitForPipeline();
Logger(`${procedureName} All done: DONE:${success}, FAILED:${failed}`, logLevel, `log-${procedureName}`); const msg = `${procedureName} All done: DONE:${success}, FAILED:${failed}`;
updateLog(procedureName, msg)
} }
initProcess.push(runAll("UPDATE DATABASE", onlyInStorage, async (e) => { initProcess.push(runAll("UPDATE DATABASE", onlyInStorage, async (e) => {
if (!this.isFileSizeExceeded(e.stat.size)) { if (!this.isFileSizeExceeded(e.stat.size)) {
@@ -2116,7 +2184,6 @@ Or if you are sure know what had been happened, we can unlock the database from
const id = await this.path2id(getPathFromTFile(file)); const id = await this.path2id(getPathFromTFile(file));
const pair: FileDocPair = { file, id }; const pair: FileDocPair = { file, id };
return [pair]; return [pair];
// processSyncFile.enqueue(pair);
} }
, { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, syncFiles); , { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, syncFiles);
processPrepareSyncFile processPrepareSyncFile
@@ -2138,10 +2205,18 @@ Or if you are sure know what had been happened, we can unlock the database from
}, { batchSize: 1, concurrentLimit: 5, delay: 10, suspended: false } }, { batchSize: 1, concurrentLimit: 5, delay: 10, suspended: false }
)) ))
processPrepareSyncFile.startPipeline(); const allSyncFiles = syncFiles.length;
initProcess.push(async () => { let lastRemain = allSyncFiles;
await processPrepareSyncFile.waitForPipeline(); const step = 25;
}) const remainLog = (remain: number) => {
if (lastRemain - remain > step) {
const msg = ` CHECK AND SYNC: ${remain} / ${allSyncFiles}`;
updateLog("sync", msg);
lastRemain = remain;
}
}
processPrepareSyncFile.startPipeline().onUpdateProgress(() => remainLog(processPrepareSyncFile.totalRemaining + processPrepareSyncFile.nowProcessing))
initProcess.push(processPrepareSyncFile.waitForPipeline());
await Promise.all(initProcess); await Promise.all(initProcess);
// this.setStatusBarText(`NOW TRACKING!`); // this.setStatusBarText(`NOW TRACKING!`);
@@ -2501,38 +2576,39 @@ Or if you are sure know what had been happened, we can unlock the database from
conflictProcessQueueCount = reactiveSource(0); conflictProcessQueueCount = reactiveSource(0);
conflictResolveQueue = conflictResolveQueue =
new KeyedQueueProcessor(async (entries: { filename: FilePathWithPrefix }[]) => { new QueueProcessor(async (filenames: FilePathWithPrefix[]) => {
const entry = entries[0]; const filename = filenames[0];
const filename = entry.filename; await serialized(`conflict-resolve:${filename}`, async () => {
const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename); const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename);
if (conflictCheckResult === MISSING_OR_ERROR || conflictCheckResult === NOT_CONFLICTED || conflictCheckResult === CANCELLED) { if (conflictCheckResult === MISSING_OR_ERROR || conflictCheckResult === NOT_CONFLICTED || conflictCheckResult === CANCELLED) {
// nothing to do. // nothing to do.
return;
}
if (conflictCheckResult === AUTO_MERGED) {
//auto resolved, but need check again;
if (this.settings.syncAfterMerge && !this.suspended) {
//Wait for the running replication, if not running replication, run it once.
await shareRunningResult(`replication`, () => this.replicate());
}
Logger("conflict:Automatically merged, but we have to check it again");
this.conflictCheckQueue.enqueue(filename);
return;
}
if (this.settings.showMergeDialogOnlyOnActive) {
const af = this.getActiveFile();
if (af && af.path != filename) {
Logger(`${filename} is conflicted. Merging process has been postponed to the file have got opened.`, LOG_LEVEL_NOTICE);
return; return;
} }
} if (conflictCheckResult === AUTO_MERGED) {
Logger("conflict:Manual merge required!"); //auto resolved, but need check again;
await this.resolveConflictByUI(filename, conflictCheckResult); if (this.settings.syncAfterMerge && !this.suspended) {
//Wait for the running replication, if not running replication, run it once.
await shareRunningResult(`replication`, () => this.replicate());
}
Logger("conflict:Automatically merged, but we have to check it again");
this.conflictCheckQueue.enqueue(filename);
return;
}
if (this.settings.showMergeDialogOnlyOnActive) {
const af = this.getActiveFile();
if (af && af.path != filename) {
Logger(`${filename} is conflicted. Merging process has been postponed to the file have got opened.`, LOG_LEVEL_NOTICE);
return;
}
}
Logger("conflict:Manual merge required!");
await this.resolveConflictByUI(filename, conflictCheckResult);
});
}, { suspended: false, batchSize: 1, concurrentLimit: 1, delay: 10, keepResultUntilDownstreamConnected: false }).replaceEnqueueProcessor( }, { suspended: false, batchSize: 1, concurrentLimit: 1, delay: 10, keepResultUntilDownstreamConnected: false }).replaceEnqueueProcessor(
(queue, newEntity) => { (queue, newEntity) => {
const filename = newEntity.entity.filename; const filename = newEntity;
sendValue("cancel-resolve-conflict:" + filename, true); sendValue("cancel-resolve-conflict:" + filename, true);
const newQueue = [...queue].filter(e => e.key != newEntity.key); const newQueue = [...queue].filter(e => e != newEntity);
return [...newQueue, newEntity]; return [...newQueue, newEntity];
}); });
@@ -2544,10 +2620,9 @@ Or if you are sure know what had been happened, we can unlock the database from
const file = this.vaultAccess.getAbstractFileByPath(filename); const file = this.vaultAccess.getAbstractFileByPath(filename);
// if (!file) return; // if (!file) return;
// if (!(file instanceof TFile)) return; // if (!(file instanceof TFile)) return;
if ((file instanceof TFolder)) return; if ((file instanceof TFolder)) return [];
// Check again? // Check again?
return [filename];
return [{ key: filename, entity: { filename } }];
// this.conflictResolveQueue.enqueueWithKey(filename, { filename, file }); // this.conflictResolveQueue.enqueueWithKey(filename, { filename, file });
}, { }, {
suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, pipeTo: this.conflictResolveQueue, totalRemainingReactiveSource: this.conflictProcessQueueCount suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, pipeTo: this.conflictResolveQueue, totalRemainingReactiveSource: this.conflictProcessQueueCount

View File

@@ -103,6 +103,9 @@
.canvas-wrapper::before, .canvas-wrapper::before,
.empty-state::before { .empty-state::before {
content: var(--sls-log-text, ""); content: var(--sls-log-text, "");
font-variant-numeric: tabular-nums;
font-variant-emoji: emoji;
tab-size: 4;
text-align: right; text-align: right;
white-space: pre-wrap; white-space: pre-wrap;
position: absolute; position: absolute;