- {peer.name} : ({peer.peerId.slice(0, 8)})
+ {peer.name} :
+ ({peer.peerId.slice(0, 8)})
{#if isCommunicating(peer.peerId)}
π‘
{/if}
@@ -460,11 +468,11 @@
+
+
SYNC
-
{:else}
+
+ {:else}
{getAcceptanceStatus(peer)}
@@ -571,7 +585,6 @@
display: flex;
flex-direction: column;
gap: 1rem;
- padding: 0.75rem;
}
.peers-section {
@@ -584,7 +597,7 @@
display: flex;
justify-content: space-between;
align-items: center;
- flex-wrap: nowrap;
+ flex-wrap: wrap;
gap: 0.5rem;
}
@@ -603,8 +616,9 @@
}
.remote-picker {
- max-width: 14rem;
- min-width: 8rem;
+ max-width: 10rem;
+ min-width: 1em;
+ flex-shrink: 1;
height: 1.9rem;
border: 1px solid var(--divider-color);
border-radius: 0.4rem;
@@ -648,6 +662,7 @@
.peers-header {
display: flex;
+ flex-wrap: wrap;
justify-content: space-between;
align-items: center;
gap: 0.5rem;
@@ -873,5 +888,4 @@
font-size: 0.9rem;
padding: 1rem;
}
-
-
\ No newline at end of file
+
diff --git a/src/modules/essentialObsidian/ModuleObsidianEvents.ts b/src/modules/essentialObsidian/ModuleObsidianEvents.ts
index 7445e11..33f0485 100644
--- a/src/modules/essentialObsidian/ModuleObsidianEvents.ts
+++ b/src/modules/essentialObsidian/ModuleObsidianEvents.ts
@@ -121,7 +121,7 @@ export class ModuleObsidianEvents extends AbstractObsidianModule {
return;
}
- const isHidden = document.hidden;
+ const isHidden = activeWindow.document.hidden;
if (this.isLastHidden === isHidden) {
return;
}
@@ -134,7 +134,7 @@ export class ModuleObsidianEvents extends AbstractObsidianModule {
} else {
// suspend all temporary.
if (this.services.appLifecycle.isSuspended()) return;
- if (!this.hasFocus) return;
+ // Do not block resume by focus state here; visibility recovery should be enough.
await this.services.appLifecycle.onResuming();
await this.services.appLifecycle.onResumed();
}
diff --git a/src/modules/features/ModuleLog.ts b/src/modules/features/ModuleLog.ts
index 16de2d9..1eb7945 100644
--- a/src/modules/features/ModuleLog.ts
+++ b/src/modules/features/ModuleLog.ts
@@ -25,7 +25,7 @@ import {
EVENT_ON_UNRESOLVED_ERROR,
} from "../../common/events.ts";
import { AbstractObsidianModule } from "../AbstractObsidianModule.ts";
-import { addIcon, normalizePath, Notice } from "../../deps.ts";
+import { addIcon, debounce, normalizePath, Notice, stringifyYaml, type WorkspaceLeaf } from "../../deps.ts";
import { LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger";
import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts";
import { serialized } from "octagonal-wheels/concurrency/lock";
@@ -41,6 +41,8 @@ import {
} from "@lib/string_and_binary/path.ts";
import { MARK_LOG_NETWORK_ERROR, MARK_LOG_SEPARATOR } from "@lib/services/lib/logUtils.ts";
import { NetworkWarningStyles } from "@lib/common/models/setting.const.ts";
+import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
+import { generateReport } from "@/common/reportTool.ts";
// This module cannot be a core module because it depends on the Obsidian UI.
@@ -50,18 +52,51 @@ const globalLogFunction = (message: any, level?: number, key?: string) => {
const messageX =
message instanceof Error
? new LiveSyncError("[Error Logged]: " + message.message, { cause: message })
- : message;
+ : typeof message === "string"
+ ? message
+ : JSON.stringify(message);
const entry = { message: messageX, level, key } as LogEntry;
recentLogEntries.value = [...recentLogEntries.value, entry];
};
setGlobalLogFunction(globalLogFunction);
-let recentLogs = [] as string[];
+// Keep the recent logs in memory for display, but also keep a longer history in logForDump for when the user wants to see more logs.
+// logForDump is not reactive and is only used for dumping logs when requested, while recentLogs is reactive and is used for displaying logs in the UI.
+const logForDump = [] as string[];
function addLog(log: string) {
- recentLogs = [...recentLogs, log].splice(-200);
- logMessages.value = recentLogs;
+ logForDump.push(log);
+ while (logForDump.length > 1000) {
+ logForDump.shift();
+ }
}
+
+// Display log is kept separate from the full log history to optimize performance and memory usage.
+// And debounce the updates to the display log to avoid excessive UI updates when there are many log entries in a short time.
+const logForDisplay = [] as string[];
+
+const updateLogMessage = debounce(() => {
+ logMessages.value = [...logForDisplay];
+}, 25);
+function addDisplayLog(log: string) {
+ logForDisplay.push(log);
+ while (logForDisplay.length > 200) {
+ logForDisplay.shift();
+ }
+ updateLogMessage();
+}
+
+const redactPatterns = [/PBKDF2 salt \(Security Seed\):.*$/];
+function redactLog(log: string) {
+ let redactedLog = log;
+ for (const pattern of redactPatterns) {
+ redactedLog = redactedLog.replace(pattern, (match) => {
+ return match.split(":")[0] + ": [REDACTED]";
+ });
+ }
+ return redactedLog;
+}
+
// logStore.intercept(e => e.slice(Math.min(e.length - 200, 0)));
const showDebugLog = false;
@@ -86,15 +121,15 @@ export class ModuleLog extends AbstractObsidianModule {
// const emptyMark = `\u{2003}`;
function padLeftSpComputed(numI: ReactiveValue, mark: string) {
const formatted = reactiveSource("");
- let timer: ReturnType | undefined = undefined;
+ let timer: number | undefined = undefined;
let maxLen = 1;
numI.onChanged((numX) => {
const num = numX.value;
const numLen = `${Math.abs(num)}`.length + 1;
maxLen = maxLen < numLen ? numLen : maxLen;
- if (timer) clearTimeout(timer);
+ if (timer) compatGlobal.clearTimeout(timer);
if (num == 0) {
- timer = setTimeout(() => {
+ timer = compatGlobal.setTimeout(() => {
formatted.value = "";
maxLen = 1;
}, 3000);
@@ -323,7 +358,7 @@ export class ModuleLog extends AbstractObsidianModule {
if (this.nextFrameQueue) {
return;
}
- this.nextFrameQueue = requestAnimationFrame(() => {
+ this.nextFrameQueue = compatGlobal.requestAnimationFrame(() => {
this.nextFrameQueue = undefined;
const { message, status } = this.statusBarLabels.value;
// const recent = logMessages.value;
@@ -346,7 +381,8 @@ export class ModuleLog extends AbstractObsidianModule {
(a, b) => (a < b.ttl ? a : b.ttl),
Number.MAX_SAFE_INTEGER
);
- if (this.logLines.length > 0) setTimeout(() => this.applyStatusBarText(), minimumNext - now);
+ if (this.logLines.length > 0)
+ compatGlobal.setTimeout(() => this.applyStatusBarText(), minimumNext - now);
const recent = this.logLines.map((e) => e.message);
const recentLogs = recent.reverse().join("\n");
if (isDirty("recentLogs", recentLogs)) this.logHistory!.innerText = recentLogs;
@@ -368,7 +404,7 @@ export class ModuleLog extends AbstractObsidianModule {
if (this.statusDiv) {
this.statusDiv.remove();
}
- document.querySelectorAll(`.livesync-status`)?.forEach((e) => e.remove());
+ compatGlobal.document.querySelectorAll(`.livesync-status`)?.forEach((e) => e.remove());
return Promise.resolve(true);
}
_everyOnloadStart(): Promise {
@@ -390,7 +426,28 @@ export class ModuleLog extends AbstractObsidianModule {
void this.services.API.showWindow(VIEW_TYPE_LOG);
},
});
- this.registerView(VIEW_TYPE_LOG, (leaf) => new LogPaneView(leaf, this.plugin));
+ this.addCommand({
+ id: "dump-debug-info",
+ name: "Generate full report for opening the issue with debug info",
+ callback: async () => {
+ const recentLog = [...logForDump];
+ const report = await generateReport(this.services.setting.currentSettings(), this.core);
+ const info = {
+ ...report,
+ recentLog: recentLog.map(redactLog),
+ };
+ const yaml = `\`\`\`\`
+# ---- Debug Info Dump ----
+${stringifyYaml(info)}
+\`\`\`\``;
+ if (await this.services.UI.promptCopyToClipboard("Debug info", yaml)) {
+ new Notice(
+ "Debug info copied to clipboard. You can paste it in the issue. Be careful as it may contain sensitive information, review it before sharing."
+ );
+ }
+ },
+ });
+ this.registerView(VIEW_TYPE_LOG, (leaf: WorkspaceLeaf) => new LogPaneView(leaf, this.plugin));
return Promise.resolve(true);
}
private _everyOnloadAfterLoadSettings(): Promise {
@@ -404,7 +461,7 @@ export class ModuleLog extends AbstractObsidianModule {
void this.setFileStatus();
});
- const w = document.querySelectorAll(`.livesync-status`);
+ const w = compatGlobal.document.querySelectorAll(`.livesync-status`);
w.forEach((e) => e.remove());
this.observeForLogs();
@@ -421,6 +478,8 @@ export class ModuleLog extends AbstractObsidianModule {
this.statusBar?.addClass("syncstatusbar");
}
this.adjustStatusDivPosition();
+ this._log("Log module loaded", LOG_LEVEL_INFO);
+ this._log("Verbose log", LOG_LEVEL_VERBOSE);
return Promise.resolve(true);
}
@@ -444,11 +503,12 @@ export class ModuleLog extends AbstractObsidianModule {
if (level == LOG_LEVEL_DEBUG && !showDebugLog) {
return;
}
+ let memoOnly = false;
if (level <= LOG_LEVEL_INFO && this.settings && this.settings.lessInformationInLog) {
- return;
+ memoOnly = true;
}
if (this.settings && !this.settings.showVerboseLog && level == LOG_LEVEL_VERBOSE) {
- return;
+ memoOnly = true;
}
const vaultName = this.services.vault.getVaultName();
const now = new Date();
@@ -469,6 +529,15 @@ export class ModuleLog extends AbstractObsidianModule {
? `${errorInfo}`
: JSON.stringify(message, null, 2);
const newMessage = timestamp + "->" + messageContent;
+
+ if (this.settings?.writeLogToTheFile) {
+ this.writeLogToTheFile(now, vaultName, newMessage);
+ }
+ addLog(newMessage);
+ if (memoOnly) {
+ return;
+ }
+ addDisplayLog(newMessage);
if (message instanceof Error) {
console.error(vaultName + ":" + newMessage);
} else if (level >= LOG_LEVEL_INFO) {
@@ -479,10 +548,6 @@ export class ModuleLog extends AbstractObsidianModule {
if (!this.settings?.showOnlyIconsOnEditor) {
this.statusLog.value = messageContent;
}
- if (this.settings?.writeLogToTheFile) {
- this.writeLogToTheFile(now, vaultName, newMessage);
- }
- addLog(newMessage);
this.logLines.push({ ttl: now.getTime() + 3000, message: newMessage });
if (level >= LOG_LEVEL_NOTICE) {
diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts
index ebfb377..d9ca27c 100644
--- a/src/modules/features/SettingDialogue/PaneHatch.ts
+++ b/src/modules/features/SettingDialogue/PaneHatch.ts
@@ -39,6 +39,7 @@ import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
import { generateCredentialObject } from "../../../lib/src/replication/httplib.ts";
import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts";
import type { PageFunctions } from "./SettingPane.ts";
+import { generateReport } from "@/common/reportTool.ts";
export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void {
// const hatchWarn = this.createEl(paneEl, "div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
// hatchWarn.addClass("op-warn-info");
@@ -69,140 +70,14 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
eventHub.emitEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE);
})
);
+
new Setting(paneEl).setName($msg("Prepare the 'report' to create an issue")).addButton((button) =>
button
.setButtonText($msg("Copy Report to clipboard"))
.setCta()
.setDisabled(false)
.onClick(async () => {
- let responseConfig: any = {};
- const REDACTED = "π
πΈπ·π΄πΆππΈπ·";
- if (this.editingSettings.remoteType == REMOTE_COUCHDB) {
- try {
- const credential = generateCredentialObject(this.editingSettings);
- const customHeaders = parseHeaderValues(this.editingSettings.couchDB_CustomHeaders);
- const r = await requestToCouchDBWithCredentials(
- this.editingSettings.couchDB_URI,
- credential,
- window.origin,
- undefined,
- undefined,
- undefined,
- customHeaders
- );
-
- Logger(JSON.stringify(r.json, null, 2));
-
- responseConfig = r.json;
- responseConfig["couch_httpd_auth"].secret = REDACTED;
- responseConfig["couch_httpd_auth"].authentication_db = REDACTED;
- responseConfig["couch_httpd_auth"].authentication_redirect = REDACTED;
- responseConfig["couchdb"].uuid = REDACTED;
- responseConfig["admins"] = REDACTED;
- delete responseConfig["jwt_keys"];
- if ("secret" in responseConfig["chttpd_auth"])
- responseConfig["chttpd_auth"].secret = REDACTED;
- } catch (ex) {
- Logger(ex, LOG_LEVEL_VERBOSE);
- responseConfig = {
- error: "Requesting information from the remote CouchDB has failed. If you are using IBM Cloudant, this is normal behaviour.",
- };
- }
- } else if (this.editingSettings.remoteType == REMOTE_MINIO) {
- responseConfig = { error: "Object Storage Synchronisation" };
- //
- }
- const defaultKeys = Object.keys(DEFAULT_SETTINGS) as (keyof ObsidianLiveSyncSettings)[];
- const pluginConfig = JSON.parse(JSON.stringify(this.editingSettings)) as ObsidianLiveSyncSettings;
- const pluginKeys = Object.keys(pluginConfig);
- for (const key of pluginKeys) {
- if (defaultKeys.includes(key as any)) continue;
- delete pluginConfig[key as keyof ObsidianLiveSyncSettings];
- }
-
- pluginConfig.couchDB_DBNAME = REDACTED;
- pluginConfig.couchDB_PASSWORD = REDACTED;
- const scheme = pluginConfig.couchDB_URI.startsWith("http:")
- ? "(HTTP)"
- : pluginConfig.couchDB_URI.startsWith("https:")
- ? "(HTTPS)"
- : "";
- pluginConfig.couchDB_URI = isCloudantURI(pluginConfig.couchDB_URI)
- ? "cloudant"
- : `self-hosted${scheme}`;
- pluginConfig.couchDB_USER = REDACTED;
- pluginConfig.passphrase = REDACTED;
- pluginConfig.encryptedPassphrase = REDACTED;
- pluginConfig.encryptedCouchDBConnection = REDACTED;
- pluginConfig.accessKey = REDACTED;
- pluginConfig.secretKey = REDACTED;
- const redact = (source: string) => `${REDACTED}(${source.length} letters)`;
- const toSchemeOnly = (uri: string) => {
- try {
- return `${new URL(uri).protocol}//`;
- } catch {
- const matched = uri.match(/^[A-Za-z][A-Za-z0-9+.-]*:\/\//);
- return matched?.[0] ?? REDACTED;
- }
- };
- pluginConfig.remoteConfigurations = Object.fromEntries(
- Object.entries(pluginConfig.remoteConfigurations || {}).map(([id, config]) => [
- id,
- {
- ...config,
- uri: toSchemeOnly(config.uri),
- },
- ])
- );
- pluginConfig.region = redact(pluginConfig.region);
- pluginConfig.bucket = redact(pluginConfig.bucket);
- pluginConfig.pluginSyncExtendedSetting = {};
- pluginConfig.P2P_AppID = redact(pluginConfig.P2P_AppID);
- pluginConfig.P2P_passphrase = redact(pluginConfig.P2P_passphrase);
- pluginConfig.P2P_roomID = redact(pluginConfig.P2P_roomID);
- pluginConfig.P2P_relays = redact(pluginConfig.P2P_relays);
- pluginConfig.jwtKey = redact(pluginConfig.jwtKey);
- pluginConfig.jwtSub = redact(pluginConfig.jwtSub);
- pluginConfig.jwtKid = redact(pluginConfig.jwtKid);
- pluginConfig.bucketCustomHeaders = redact(pluginConfig.bucketCustomHeaders);
- pluginConfig.couchDB_CustomHeaders = redact(pluginConfig.couchDB_CustomHeaders);
- pluginConfig.P2P_turnCredential = redact(pluginConfig.P2P_turnCredential);
- pluginConfig.P2P_turnUsername = redact(pluginConfig.P2P_turnUsername);
- pluginConfig.P2P_turnServers = `(${pluginConfig.P2P_turnServers.split(",").length} servers configured)`;
- const endpoint = pluginConfig.endpoint;
- if (endpoint == "") {
- pluginConfig.endpoint = "Not configured or AWS";
- } else {
- const endpointScheme = pluginConfig.endpoint.startsWith("http:")
- ? "(HTTP)"
- : pluginConfig.endpoint.startsWith("https:")
- ? "(HTTPS)"
- : "";
- pluginConfig.endpoint = `${endpoint.indexOf(".r2.cloudflarestorage.") !== -1 ? "R2" : "self-hosted?"}(${endpointScheme})`;
- }
- const obsidianInfo = {
- navigator: navigator.userAgent,
- fileSystem: this.core.services.vault.isStorageInsensitive() ? "insensitive" : "sensitive",
- };
- const msgConfig = `# ---- Obsidian info ----
-${stringifyYaml(obsidianInfo)}
----
-# ---- remote config ----
-${stringifyYaml(responseConfig)}
----
-# ---- Plug-in config ----
-${stringifyYaml({
- version: this.manifestVersion,
- ...pluginConfig,
-})}`;
- console.log(msgConfig);
- if ((await this.services.UI.promptCopyToClipboard("Generated report", msgConfig)) == true) {
- // await navigator.clipboard.writeText(msgConfig);
- // Logger(
- // `Generated report has been copied to clipboard. Please report the issue with this! Thank you for your cooperation!`,
- // LOG_LEVEL_NOTICE
- // );
- }
+ await this.app.commands.executeCommandById("obsidian-livesync:dump-debug-info");
})
);
new Setting(paneEl)
diff --git a/updates.md b/updates.md
index 4eedc41..7ee19e1 100644
--- a/updates.md
+++ b/updates.md
@@ -11,6 +11,24 @@ The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsid
### Improved
- Many messages related to tweak mismatch resolution have been updated for clarity.
+## 0.25.65
+
+19th May, 2026
+
+### Fixed
+- Fix an issue about resuming from background on iOS (#888).
+- Now Chunk Splitter: `V3: Fine Deduplication` is working fine again (#866).
+ - It has some drawbacks, such as fewer chunks are generated. However, it makes less transfer and storage when the files are modified but not completely changed.
+- Unsynchronised local changes (which means changes that have not been sent) are now correctly preserved as a conflict (Thank you so much for @SeleiXi!).
+- Avoid creating a new revision when the current and conflicted revisions have identical content (Thank you so much for @daichi-629).
+
+### Improved
+- Improved the error verbosity on concurrent processing during the start-up process.
+- Now the `report` includes recent logs (of verbosity `verbose` even settings is not set to `verbose`).
+- Updating logs is now debounced to avoid excessive updates during rapid log generation.
+- Added a `Generate full report for opening the issue with debug info` command to the command palette, which generates a report without opening the settings dialogue.
+
+
## 0.25.64
17th May, 2026
diff --git a/utils/bench/splitPiecesRabinKarp.ts b/utils/bench/splitPiecesRabinKarp.ts
new file mode 100644
index 0000000..1c4642c
--- /dev/null
+++ b/utils/bench/splitPiecesRabinKarp.ts
@@ -0,0 +1,197 @@
+import { glob } from "glob";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promises as fs } from "node:fs";
+import { isPlainText, shouldSplitAsPlainText } from "../../src/lib/src/string_and_binary/path";
+import { splitPiecesRabinKarp } from "../../src/lib/src/string_and_binary/chunks";
+import {
+ PREFERRED_BASE,
+ PREFERRED_JOURNAL_SYNC,
+ PREFERRED_SETTING_CLOUDANT,
+ PREFERRED_SETTING_SELF_HOSTED,
+} from "../../src/lib/src/common/models/setting.const.preferred";
+import { type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, MAX_DOC_SIZE_BIN } from "../../src/lib/src/common/types";
+
+async function blobFromString(content: string): Promise {
+ return new Blob([content], { type: "text/plain" });
+}
+
+const preferred = PREFERRED_BASE;
+const preferredJournal = PREFERRED_JOURNAL_SYNC;
+const preferredCouchDB = PREFERRED_SETTING_SELF_HOSTED;
+const preferredIBM = PREFERRED_SETTING_CLOUDANT;
+
+function computeChunkSize(overlay: Partial) {
+ const settings = { ...DEFAULT_SETTINGS, ...overlay };
+ const maxChunkSize = Math.floor(MAX_DOC_SIZE_BIN * ((settings.customChunkSize || 0) * 1 + 1));
+ const pieceSize = maxChunkSize;
+
+ const minimumChunkSize = settings.minimumChunkSize;
+ return { pieceSize, minimumChunkSize };
+}
+
+async function testSplit(
+ splitPiecesRabinKarpFn: typeof splitPiecesRabinKarp,
+ content: Blob,
+ settingsOverlay: Partial
+) {
+ const { pieceSize, minimumChunkSize } = computeChunkSize(settingsOverlay);
+ const isPlain = content.type === "text/plain";
+ const chunkGenerator = await splitPiecesRabinKarpFn(content, pieceSize, isPlain, minimumChunkSize);
+ const chunks = [] as string[];
+ for await (const chunk of chunkGenerator()) {
+ chunks.push(chunk);
+ }
+ // if there are few chunks, calculate average chunk size except the last chunk which can be smaller due to the way the algorithm works, especially for small files.
+ const averageChunkSize =
+ chunks.length > 1
+ ? chunks.slice(0, -1).reduce((acc, chunk) => acc + chunk.length, 0) / (chunks.length - 1)
+ : chunks.reduce((acc, chunk) => acc + chunk.length, 0) / chunks.length;
+ const lastChunk = chunks[chunks.length - 1];
+ // compute minimum chunk size if the last chunk is not the smallest.
+ const nonLastChunkSizes = chunks.slice(0, -1).map((c) => c.length);
+ const minChunkSize = nonLastChunkSizes.length > 0 ? Math.min(...nonLastChunkSizes) : lastChunk.length;
+ const result = {
+ isPlain,
+ originalSize: content.size,
+ chunkCount: chunks.length,
+ totalLength: chunks.reduce((acc, chunk) => acc + chunk.length, 0),
+ averageChunkSize: averageChunkSize,
+ maxChunkSize: Math.max(...chunks.map((c) => c.length)),
+ minChunkSize: minChunkSize,
+ uniqueChunks: new Set(chunks).size,
+ chunks: chunks,
+ };
+ return result;
+}
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = resolve(__filename, "..");
+async function loadFileAsBlob(filePath: string): Promise {
+ if (shouldSplitAsPlainText(filePath)) {
+ const content = await fs.readFile(filePath, "utf-8");
+ return blobFromString(content);
+ } else {
+ const buffer = await fs.readFile(filePath);
+ return new Blob([buffer]);
+ }
+}
+const testProfiles = [
+ { name: "CouchDB", settings: preferredCouchDB },
+ { name: "IBM Cloudant", settings: preferredIBM },
+ { name: "Journal Sync", settings: preferredJournal },
+ // { name: "Base", settings: preferred },
+];
+function modifyBlob(blob: Blob, position: number, insertText: string): Blob {
+ const before = blob.slice(0, position);
+ const after = blob.slice(position);
+ const insert = new Blob([insertText], { type: blob.type });
+ return new Blob([before, insert, after], { type: blob.type });
+}
+async function main() {
+ const results = [] as string[][];
+ console.log("directory:", __dirname);
+ const findPath = resolve(__dirname, "../../");
+ console.warn("CWD:", findPath);
+ let testFiles = await glob("**/*.*", {
+ cwd: findPath,
+ maxDepth: 20,
+ ignore: ["**/node_modules/**", "**/.obsidian/**", "**/dist/**", "**/build/**", "**/out/**"],
+ });
+ testFiles = testFiles.filter((file) => {
+ const ext = file.split(".").pop()?.toLowerCase() || "";
+ return ["md", "txt", "json", "csv", "png"].includes(ext);
+ });
+ const header = [
+ "Profile",
+ "Implementation",
+ "Edition",
+ "File",
+ "Mode",
+ "Original Size (bytes)",
+ "Chunk Count",
+ "Average Chunk Size",
+ "Max Chunk Size",
+ "Min Chunk Size",
+ "Unique Chunks",
+ "Shared Chunks",
+ "Savings",
+ "Newly added (count)",
+ "Newly consumed (bytes)",
+ ];
+ for (const profile of testProfiles) {
+ console.log(`Testing profile: ${profile.name}`);
+ for (const fn of [splitPiecesRabinKarp]) {
+ const funcProfile = fn !== splitPiecesRabinKarp ? "Old" : "New";
+ console.log(`Testing function: ${funcProfile}`);
+ for (const file of testFiles) {
+ const filePath = resolve(findPath, file);
+ const isPlain = shouldSplitAsPlainText(filePath);
+ const content = await loadFileAsBlob(filePath);
+ console.log(`Testing file: ${file} (size: ${content.size} bytes)`);
+ const result = await testSplit(fn, content, profile.settings);
+ const chunkSizes = result.chunks.map((c) => c.length);
+ const savings = result.originalSize - chunkSizes.reduce((acc, size) => acc + size, 0);
+ // console.log(`Result for ${file}:`, result);
+ results.push([
+ `${profile.name}`,
+ funcProfile,
+ "original",
+ file,
+ isPlain ? "plain" : "binary",
+ content.size.toString(),
+ result.chunkCount.toString(),
+ result.averageChunkSize.toFixed(2),
+ result.maxChunkSize.toString(),
+ result.minChunkSize.toString(),
+ result.uniqueChunks.toString(),
+ "",
+ savings.toString(),
+ "",
+ "",
+ ]);
+ // add editions (inserting "*") to content on head, 5%, middle, 95%, tail to see if it affects the chunking
+ const editions = [
+ { name: "head", content: modifyBlob(content, 0, "*") },
+ { name: "5%", content: modifyBlob(content, Math.floor(content.size * 0.05), "*") },
+ { name: "middle", content: modifyBlob(content, Math.floor(content.size * 0.5), "*") },
+ { name: "95%", content: modifyBlob(content, Math.floor(content.size * 0.95), "*") },
+ { name: "tail", content: modifyBlob(content, content.size, "*") },
+ ];
+ const baseChunks = result.chunks;
+ for (const edition of editions) {
+ console.log(`Testing edition: ${edition.name}`);
+ const editionResult = await testSplit(fn, edition.content, profile.settings);
+ const sharedChunks = editionResult.chunks.filter((chunk) => baseChunks.includes(chunk)).length;
+ const newChunks = editionResult.chunks.filter((chunk) => !baseChunks.includes(chunk));
+ const editionResultChunkLength = editionResult.chunks.map((c) => c.length);
+ // console.log(`Result for edition ${edition.name} of ${file}:`, editionResult);
+ const editionSavings =
+ editionResult.originalSize - editionResultChunkLength.reduce((acc, size) => acc + size, 0);
+ // newly added chunks size :
+ const newChunksSize = newChunks.reduce((acc, chunk) => acc + chunk.length, 0);
+ results.push([
+ `${profile.name}`,
+ funcProfile,
+ `${edition.name}`,
+ file,
+ isPlain ? "plain" : "binary",
+ edition.content.size.toString(),
+ editionResult.chunkCount.toString(),
+ editionResult.averageChunkSize.toFixed(2),
+ editionResult.maxChunkSize.toString(),
+ editionResult.minChunkSize.toString(),
+ editionResult.uniqueChunks.toString(),
+ sharedChunks.toString(),
+ editionSavings.toString(),
+ newChunks.length.toString(),
+ newChunksSize.toString(),
+ ]);
+ }
+ }
+ }
+ }
+
+ results.unshift(header);
+ await fs.writeFile(resolve(__dirname, "splitResults.csv"), results.map((r) => r.join(",")).join("\n"));
+}
+main();