fix: preserve U+FEFF at chunk boundaries (#1010)

src/rabinKarpBom.unit.spec.ts

@@ -0,0 +1,18 @@
+import { splitPiecesRabinKarp } from "@lib/string_and_binary/chunks.ts";
+import { describe, expect, it } from "vitest";
+
+describe("Rabin-Karp text splitting", () => {
+    it("preserves U+FEFF at the beginning of an internal chunk", async () => {
+        const content = `${"a".repeat(1024)}\uFEFF${"b".repeat(1024)}`;
+        const createChunks = await splitPiecesRabinKarp(new Blob([content], { type: "text/plain" }), 1024, true, 1024);
+        const chunks: string[] = [];
+
+        for await (const chunk of createChunks()) {
+            chunks.push(chunk);
+        }
+
+        expect(chunks).toHaveLength(3);
+        expect(chunks[1].startsWith("\uFEFF")).toBe(true);
+        expect(chunks.join("")).toBe(content);
+    });
+});