Merge pull request #804 from A-wry/reduce-fetch-error-visibility

Add connection warning style logic and settings UI dropdown

docs/datastructure.md

@@ -0,0 +1,168 @@
+# Data Structures of Self-Hosted LiveSync
+## Overview
+
+Self-hosted LiveSync uses the following types of documents:
+
+- Metadata
+    - Legacy Metadata
+    - Binary Metadata
+    - Plain Metadata
+- Chunk
+- Versioning
+- Synchronise Information
+- Synchronise Parameters
+- Milestone Information
+
+## Description of Each Data Structure
+
+All documents inherit from the `DatabaseEntry` interface. This is necessary for conflict resolution and deletion flags.
+
+```ts
+export interface DatabaseEntry {
+    _id: DocumentID;
+    _rev?: string;
+    _deleted?: boolean;
+}
+```
+
+### Versioning Document
+
+This document stores version information for Self-hosted LiveSync.
+The ID is fixed as `obsydian_livesync_version` [VERSIONING_DOCID]. Yes, the typo has become a curse.
+When Self-hosted LiveSync detects changes to this document via Replication, it reads the version information and checks compatibility.
+In that case, if there are major changes, synchronisation may be stopped.
+Please refer to negotiation.ts.
+
+### Synchronise Information Document
+
+This document stores information that should be verified in synchronisation settings.
+The ID is fixed as `syncinfo` [SYNCINFO_ID].
+The information stored in this document is only the conditions necessary for synchronisation to succeed, and as of v0.25.43, only a random string is stored.
+This document is only used during rebuilds from the settings screen for CouchDB-based synchronisation, making it like an appendix. It may be removed in the future.
+
+### Synchronise Parameters Document
+
+This document stores synchronisation parameters.
+Synchronisation parameters include the protocol version and salt used for encryption, but do not include chunking settings.
+
+The ID is fixed as `_local/obsidian_livesync_sync_parameters` [DOCID_SYNC_PARAMETERS] or `_obsidian_livesync_journal_sync_parameters.json` [DOCID_JOURNAL_SYNC_PARAMETERS].
+
+This document exists only on the remote and not locally.
+This document stores the following information.
+It is read each time before connecting and is used to verify that E2EE settings match.
+This mismatch cannot be ignored and synchronisation will be stopped.
+
+```ts
+export interface SyncParameters extends DatabaseEntry {
+    _id: typeof DOCID_SYNC_PARAMETERS;
+    type: (typeof EntryTypes)["SYNC_PARAMETERS"];
+    protocolVersion: ProtocolVersion;
+    pbkdf2salt: string;
+}
+```
+
+#### protocolVersion
+
+This field indicates the protocol version used by the remote. Mostly, this value should be `2` (ProtocolVersions.ADVANCED_E2EE), which indicates safer E2EE support.
+
+#### pbkdf2salt
+
+This field stores the salt used for PBKDF2 key derivation on the remote. This salt and the passphrase provides E2EE encryption keys.
+
+### Milestone Information Document
+
+This document stores information about how the remote accepts and recognises clients.
+The ID is fixed as `_local/obsidian_livesync_milestone` [MILESTONE_DOCID].
+This document exists only on the remote and not locally.
+This document is used to indicate synchronisation progress and includes the version range of accepted chunks for each node and adjustment values for each node.
+Tweak Mismatched is determined based on the information in this document.
+
+For details, please refer to LiveSyncReplicator.ts, LiveSyncJournalReplicator.ts, and LiveSyncDBFunctions.ts.
+
+```ts
+export interface EntryMilestoneInfo extends DatabaseEntry {
+    _id: typeof MILESTONE_DOCID;
+    type: EntryTypes["MILESTONE_INFO"];
+    created: number;
+    accepted_nodes: string[];
+    node_info: { [key: NodeKey]: NodeData };
+    locked: boolean;
+    cleaned?: boolean;
+    node_chunk_info: { [key: NodeKey]: ChunkVersionRange };
+    tweak_values: { [key: NodeKey]: TweakValues };
+}
+```
+
+### locked
+
+If the remote has been requested to lock out from any client, this is set to true.
+When set to true, clients will stop synchronisation unless they are included in accepted_nodes.
+
+### cleaned
+
+If the remote has been cleaned up from any client, this is set to true.
+In this case, clients will stop synchronisation as they need to rebuild again.
+
+### Metadata Document
+
+Metadata documents store metadata for Obsidian notes.
+
+```ts
+export interface MetadataDocument extends DatabaseEntry {
+    _id: DocumentID;
+    ctime: number;
+    mtime: number;
+    size: number;
+    deleted?: boolean;
+    eden: Record<string, EdenChunk>; // Obsolete
+    path: FilePathWithPrefix;
+    children: string[];
+    type: EntryTypes["NOTE_LEGACY" | "NOTE_BINARY" | "NOTE_PLAIN"];
+}
+```
+
+### type
+
+This field indicates the type of Metadata document.
+By convention, Self-hosted LiveSync does not save the mime type of the file, but distinguishes them with this field. Please note this.
+Possible values are as follows:
+
+- NOTE_LEGACY: Legacy metadata document
+    - Please do not use
+- NOTE_BINARY: Binary metadata document (newnote)
+- NOTE_PLAIN: Plain metadata document (plain)
+
+#### children
+
+This field stores an array of Chunk Document IDs.
+
+#### \_id, path
+
+\_id is generated based on the path of the Obsidian note.
+
+- If the path starts with `_`, it is converted to `/_` for convenience.
+- If Case Sensitive is disabled, it is converted to lowercase.
+
+When Obfuscation is enabled, the path field contains `f:{obfuscated path}`.
+The path field stores the path as is. However, when Obfuscation is enabled, the obfuscated path is stored.
+
+When Property Encryption is enabled, the path field stores all properties including children, mtime, ctime, and size in an encrypted state. Please refer to encryption.ts.
+
+### Chunk Document
+
+```ts
+export type EntryLeaf = DatabaseEntry & {
+    _id: DocumentID;
+    type: EntryTypes["CHUNK"];
+    data: string;
+};
+```
+
+Chunk documents store parts of note content.
+
+- The type field is always `[CHUNK]`, `leaf`.
+- The data field stores the chunk content.
+- The \_id field is generated based on a hash of the content and the passphrase.
+
+Hash functions used include xxHash and SHA-1, depending on settings.
+Chunking methods used include Contextual Chunking and Rabin-Karp Chunking, depending on settings.

src/modules/core/ModuleReplicator.ts

@@ -116,7 +116,8 @@ export class ModuleReplicator extends AbstractModule {
         }
         // Showing message is false: that because be shown here. (And it is a fatal error, no way to hide it).
         if (!(await this.ensureReplicatorPBKDF2Salt(false))) {
-            this.showError("Failed to initialise the encryption key, preventing replication.");
+            //tagged as network error at beginning for error filtering with NetworkWarningStyles
+            this.showError("\u{200b}Failed to initialise the encryption key, preventing replication.");
             return false;
         }
         await this.processor.restoreFromSnapshotOnce();
@@ -218,7 +219,11 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
             return false;
         }
         if (!(await this.services.replication.onBeforeReplicate(showMessage))) {
-            this.showError($msg("Replicator.Message.SomeModuleFailed"), LOG_LEVEL_NOTICE);
+            // check for tagged network errors for filtering by NetworkWarningStyles
+            const hasNetworkError = [...this._previousErrors].some(e => e.startsWith("\u{200b}"));
+            if (!hasNetworkError) {
+                this.showError($msg("Replicator.Message.SomeModuleFailed"), LOG_LEVEL_NOTICE);
+            }
             return false;
         }
         this.clearErrors();

src/modules/coreObsidian/ModuleFileAccessObsidian.ts

@@ -108,7 +108,22 @@ export class ModuleFileAccessObsidian extends AbstractObsidianModule implements
                 // For safety, check existence
                 return await this.vaultAccess.adapterExists(path);
             } else {
-                return (await this.vaultAccess.vaultCreate(path, data, opt)) instanceof TFile;
+                // The same stale-index issue described above can also happen for .md files during
+                // concurrent initialisation (UPDATE STORAGE runs up to 10 ops in parallel).
+                // getAbstractFileByPath returns null because Obsidian's in-memory index hasn't
+                // caught up yet, but the file already exists on disk — causing vault.create() to
+                // throw "File already exists."
+                // Fall back to adapterWrite (same approach used for non-md files above) so the
+                // file is written correctly without an error.
+                try {
+                    return (await this.vaultAccess.vaultCreate(path, data, opt)) instanceof TFile;
+                } catch (ex) {
+                    if (ex instanceof Error && ex.message === "File already exists.") {
+                        await this.vaultAccess.adapterWrite(path, data, opt);
+                        return await this.vaultAccess.adapterExists(path);
+                    }
+                    throw ex;
+                }
             }
         } else {
             this._log(`Could not write file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE);

src/modules/essentialObsidian/ModuleObsidianAPI.ts

@@ -237,7 +237,7 @@ export class ModuleObsidianAPI extends AbstractObsidianModule {
                 } catch (ex: any) {
                     this._log(`HTTP:${method}${size} to:${localURL} -> failed`, LOG_LEVEL_VERBOSE);
                     const msg = ex instanceof Error ? `${ex?.name}:${ex?.message}` : ex?.toString();
-                    this.showError(`Failed to fetch: ${msg}`); // Do not show notice, due to throwing below
+                    this.showError(`\u{200b}Network Error: Failed to fetch: ${msg}`); // Do not show notice, due to throwing below
                     this._log(ex, LOG_LEVEL_VERBOSE);
                     // limit only in bulk_docs.
                     if (url.toString().indexOf("_bulk_docs") !== -1) {

src/modules/features/ModuleLog.ts

@@ -39,6 +39,7 @@ import {
     isValidFilenameInDarwin,
     isValidFilenameInWidows,
 } from "@/lib/src/string_and_binary/path.ts";
+import { NetworkWarningStyles } from "@lib/common/models/setting.const.ts"
 
 // This module cannot be a core module because it depends on the Obsidian UI.
 
@@ -155,14 +156,14 @@ export class ModuleLog extends AbstractObsidianModule {
                         lastSyncPushSeq == 0
                             ? ""
                             : lastSyncPushSeq >= maxPushSeq
-                              ? " (LIVE)"
-                              : ` (${maxPushSeq - lastSyncPushSeq})`;
+                                ? " (LIVE)"
+                                : ` (${maxPushSeq - lastSyncPushSeq})`;
                     pullLast =
                         lastSyncPullSeq == 0
                             ? ""
                             : lastSyncPullSeq >= maxPullSeq
-                              ? " (LIVE)"
-                              : ` (${maxPullSeq - lastSyncPullSeq})`;
+                                ? " (LIVE)"
+                                : ` (${maxPullSeq - lastSyncPullSeq})`;
                     break;
                 case "ERRORED":
                     w = "⚠";
@@ -281,7 +282,20 @@ export class ModuleLog extends AbstractObsidianModule {
             const fileStatus = this.activeFileStatus.value;
             if (fileStatus && !this.settings.hideFileWarningNotice) messageLines.push(fileStatus);
             const messages = (await this.services.appLifecycle.getUnresolvedMessages()).flat().filter((e) => e);
-            messageLines.push(...messages);
+            const stringMessages = messages.filter((m): m is string => typeof m === "string"); // for 'startsWith'
+            const networkMessages = stringMessages.filter(m => m.startsWith("\u{200b}"));
+            const otherMessages = stringMessages.filter(m => !m.startsWith("\u{200b}"));
+
+            messageLines.push(...otherMessages);
+
+            if (
+                this.settings.networkWarningStyle !== NetworkWarningStyles.ICON &&
+                this.settings.networkWarningStyle !== NetworkWarningStyles.HIDDEN
+            ) {
+                messageLines.push(...networkMessages);
+            } else if (this.settings.networkWarningStyle === NetworkWarningStyles.ICON) {
+                if (networkMessages.length > 0) messageLines.push("🔗❌");
+            }
             this.messageArea.innerText = messageLines.map((e) => `⚠️ ${e}`).join("\n");
         }
     }
@@ -439,8 +453,8 @@ export class ModuleLog extends AbstractObsidianModule {
             typeof message == "string"
                 ? message
                 : message instanceof Error
-                  ? `${errorInfo}`
-                  : JSON.stringify(message, null, 2);
+                    ? `${errorInfo}`
+                    : JSON.stringify(message, null, 2);
         const newMessage = timestamp + "->" + messageContent;
         if (message instanceof Error) {
             console.error(vaultName + ":" + newMessage);

src/modules/features/SettingDialogue/PaneGeneral.ts

@@ -4,6 +4,8 @@ import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
 import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts";
 import type { PageFunctions } from "./SettingPane.ts";
 import { visibleOnly } from "./SettingPane.ts";
+import { EVENT_ON_UNRESOLVED_ERROR, eventHub } from "@/common/events.ts";
+import { NetworkWarningStyles } from "@lib/common/models/setting.const.ts";
 export function paneGeneral(
     this: ObsidianLiveSyncSettingTab,
     paneEl: HTMLElement,
@@ -24,6 +26,16 @@ export function paneGeneral(
         });
         new Setting(paneEl).autoWireToggle("showStatusOnStatusbar");
         new Setting(paneEl).autoWireToggle("hideFileWarningNotice");
+        new Setting(paneEl).autoWireDropDown("networkWarningStyle", {
+            options: {
+                [NetworkWarningStyles.BANNER]: "Show full banner",
+                [NetworkWarningStyles.ICON]: "Show icon only",
+                [NetworkWarningStyles.HIDDEN]: "Hide completely",
+            },
+        });
+        this.addOnSaved("networkWarningStyle", () => {
+            eventHub.emitEvent(EVENT_ON_UNRESOLVED_ERROR);
+        });
     });
     void addPanel(paneEl, $msg("obsidianLiveSyncSettingTab.titleLogging")).then((paneEl) => {
         paneEl.addClass("wizardHidden");