fix: preserve U+FEFF at chunk boundaries (#1010)

This commit is contained in:
vorotamoroz
2026-07-14 02:15:57 +09:00
committed by GitHub
parent ff12778301
commit a41c829c10
7 changed files with 33 additions and 11 deletions
+7 -7
View File
@@ -29,7 +29,7 @@
"markdown-it": "^14.2.0",
"minimatch": "^10.2.5",
"obsidian": "^1.13.1",
"octagonal-wheels": "^0.1.47",
"octagonal-wheels": "^0.1.50",
"qrcode-generator": "^1.4.4",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
},
@@ -11700,9 +11700,9 @@
"license": "MIT"
},
"node_modules/octagonal-wheels": {
"version": "0.1.47",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.47.tgz",
"integrity": "sha512-pv+AoplTzDmcrYpxun/N7sab7KAVoPvUhwvnrXwrVihjj8sQ89HmovndpMvGJxdpp4+uOOY4SiO0DVWTp+YCkQ==",
"version": "0.1.50",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.50.tgz",
"integrity": "sha512-1JUU1+hFJKaF4aUchXYfFB9fCHagoYcYiYwSq0B64qwrmh+CGmMb6lSambHM21wBpyoKkO7FO3z7N+Xs5YXPug==",
"license": "MIT",
"dependencies": {
"idb": "^8.0.3"
@@ -16230,7 +16230,7 @@
"dependencies": {
"chokidar": "^4.0.0",
"minimatch": "^10.2.5",
"octagonal-wheels": "^0.1.47",
"octagonal-wheels": "^0.1.50",
"pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-leveldb": "^9.0.0",
"pouchdb-core": "^9.0.0",
@@ -16254,7 +16254,7 @@
"name": "livesync-webapp",
"version": "0.25.80-webapp",
"dependencies": {
"octagonal-wheels": "^0.1.47"
"octagonal-wheels": "^0.1.50"
},
"devDependencies": {
"@playwright/test": "^1.58.2",
@@ -16269,7 +16269,7 @@
"src/apps/webpeer": {
"version": "0.25.80-webpeer",
"dependencies": {
"octagonal-wheels": "^0.1.47"
"octagonal-wheels": "^0.1.50"
},
"devDependencies": {
"@sveltejs/vite-plugin-svelte": "^7.1.2",
+1 -1
View File
@@ -163,7 +163,7 @@
"markdown-it": "^14.2.0",
"minimatch": "^10.2.5",
"obsidian": "^1.13.1",
"octagonal-wheels": "^0.1.47",
"octagonal-wheels": "^0.1.50",
"qrcode-generator": "^1.4.4",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
},
+1 -1
View File
@@ -41,7 +41,7 @@
"dependencies": {
"chokidar": "^4.0.0",
"minimatch": "^10.2.5",
"octagonal-wheels": "^0.1.47",
"octagonal-wheels": "^0.1.50",
"pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-leveldb": "^9.0.0",
"pouchdb-core": "^9.0.0",
+1 -1
View File
@@ -12,7 +12,7 @@
"preview": "vite preview"
},
"dependencies": {
"octagonal-wheels": "^0.1.47"
"octagonal-wheels": "^0.1.50"
},
"devDependencies": {
"@playwright/test": "^1.58.2",
+1 -1
View File
@@ -12,7 +12,7 @@
"check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json"
},
"dependencies": {
"octagonal-wheels": "^0.1.47"
"octagonal-wheels": "^0.1.50"
},
"devDependencies": {
"eslint-plugin-svelte": "^3.19.0",
+18
View File
@@ -0,0 +1,18 @@
import { splitPiecesRabinKarp } from "@lib/string_and_binary/chunks.ts";
import { describe, expect, it } from "vitest";
describe("Rabin-Karp text splitting", () => {
it("preserves U+FEFF at the beginning of an internal chunk", async () => {
const content = `${"a".repeat(1024)}\uFEFF${"b".repeat(1024)}`;
const createChunks = await splitPiecesRabinKarp(new Blob([content], { type: "text/plain" }), 1024, true, 1024);
const chunks: string[] = [];
for await (const chunk of createChunks()) {
chunks.push(chunk);
}
expect(chunks).toHaveLength(3);
expect(chunks[1].startsWith("\uFEFF")).toBe(true);
expect(chunks.join("")).toBe(content);
});
});
+4
View File
@@ -5,6 +5,10 @@ The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsid
## Unreleased
### Fixed
- Fixed an issue where a U+FEFF character at a RabinKarp chunk boundary could be lost, changing the reconstructed file and causing repeated size-mismatch conflicts (#1000).
### Improved
- Improved vault scanning and CLI file filtering by reusing compiled ignore patterns, reducing processing overhead for vaults with many files or ignore rules (#1006, #1007, and #1008).