Prepare consistent Chunk sizes and fresh Fast Fetch integration

utils/chunk-size-benchmark/cloudant-json-boundary.mjs

@@ -0,0 +1,108 @@
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const packageRoot = dirname(fileURLToPath(import.meta.resolve('@vrtmrz/livesync-commonlib/package.json')));
+const { splitPiecesRabinKarp } = await import(pathToFileURL(join(packageRoot, 'dist/string_and_binary/chunks.js')).href);
+const { getConfiguredFunctionsForEncryption } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/encryption.js')).href);
+const { compressDoc } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/compress.js')).href);
+const output = resolve(process.argv[2] || '.bench/rabin-window-fixed/cloudant-json-boundary');
+await fs.mkdir(output, { recursive: true });
+const uri = 'http://127.0.0.1:15994';
+const headers = { Authorization: `Basic ${Buffer.from('bench:benchmark-only-password').toString('base64')}`, 'Content-Type': 'application/json' };
+const configPath = '/_node/_local/_config/couchdb/max_document_size';
+const previousResponse = await fetch(uri + configPath, { headers });
+assert(previousResponse.ok || previousResponse.status === 404);
+const previous = previousResponse.ok ? await previousResponse.json() : undefined;
+const requestConfigPath = '/_node/_local/_config/chttpd/max_http_request_size';
+const previousRequestResponse = await fetch(uri + requestConfigPath, { headers });
+assert(previousRequestResponse.ok || previousRequestResponse.status === 404);
+const previousRequestLimit = previousRequestResponse.ok ? await previousRequestResponse.json() : undefined;
+const database = `chunkbench-json-budget-${Date.now()}`;
+const inputs = {
+    binary: new Blob([new Uint8Array(4 * 1024 * 1024)], { type: 'application/octet-stream' }),
+    nulText: new Blob(['\0'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }),
+    escapedText: new Blob(['\\'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }),
+};
+const cases = [
+    { kind: 'binary', customChunkSize: 4, encryption: 'v1', compression: false, expectedAccepted: true },
+    { kind: 'binary', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true },
+    { kind: 'binary', customChunkSize: 5, encryption: 'v2', compression: false, expectedAccepted: false },
+    { kind: 'nulText', customChunkSize: 0, encryption: 'none', compression: false, expectedAccepted: true },
+    { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: false },
+    { kind: 'escapedText', customChunkSize: 4, encryption: 'none', compression: false },
+    { kind: 'nulText', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true },
+    { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: true, expectedAccepted: true },
+];
+const records = [];
+const record = { scope: 'Synthetic leaf-document probes using the installed splitter, compression, encryption, and JSON serialisation, sent to the owned CouchDB fixture with a 1,000,000-byte document limit and 64,000,000-byte request limit. This is not an IBM Cloudant service measurement or a whole-file synchronisation test. CouchDB 3.5.0 estimates document size from parsed strings and does not count JSON escapes; Cloudant internal size accounting has not been verified.', source: 'https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits', sizeCalculationSource: 'https://github.com/apache/couchdb/blob/3.5.0/src/couch/src/couch_ejson_size.erl', documentLimitBytes: 1000000, requestLimitBytes: 64000000, previousDocumentLimit: previous ?? null, previousRequestLimit: previousRequestLimit ?? null, cases: records, batches: [] };
+try {
+    assert((await fetch(uri + configPath, { method: 'PUT', headers, body: JSON.stringify('1000000') })).ok);
+    assert((await fetch(uri + requestConfigPath, { method: 'PUT', headers, body: JSON.stringify('64000000') })).ok);
+    assert((await fetch(`${uri}/${database}`, { method: 'PUT', headers })).ok);
+    for (const condition of cases) {
+        const input = inputs[condition.kind];
+        const chunks = await splitPiecesRabinKarp(input, 102400 * (condition.customChunkSize + 1), condition.kind !== 'binary', 20);
+        let largest = '';
+        for await (const chunk of chunks()) if (chunk.length > largest.length) largest = chunk;
+        const id = `h:+${createHash('sha256').update(JSON.stringify(condition)).digest('hex')}`;
+        let doc = { _id: id, type: 'leaf', data: largest };
+        if (condition.compression) doc = await compressDoc(doc);
+        if (condition.encryption !== 'none') {
+            const { incoming, outgoing } = getConfiguredFunctionsForEncryption('public-json-budget-probe-passphrase', false, false, async () => new Uint8Array(16), condition.encryption);
+            const encrypted = await incoming(doc);
+            assert.equal(encrypted.e_, true);
+            const decoded = await outgoing(structuredClone(encrypted));
+            assert.equal(decoded.data, doc.data);
+            doc = encrypted;
+        }
+        doc = { ...doc, _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] } };
+        const jsonBytes = Buffer.byteLength(JSON.stringify(doc));
+        const body = JSON.stringify({ docs: [doc], new_edits: false });
+        const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body });
+        const result = await response.json();
+        const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error);
+        const measurement = { ...condition, largestPieceUtf8Bytes: Buffer.byteLength(largest), jsonBytes, jsonBytesUnderLimit: jsonBytes < record.documentLimitBytes, requestBodyBytes: Buffer.byteLength(body), status: response.status, accepted, response: result };
+        records.push(measurement);
+        await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2));
+        if (condition.expectedAccepted !== undefined) assert.equal(accepted, condition.expectedAccepted, JSON.stringify(measurement));
+        assert(Buffer.byteLength(body) < record.requestLimitBytes);
+        console.log(JSON.stringify({ ...measurement, response: undefined }));
+    }
+    const batchInput = new Uint8Array(512000 * 25);
+    for (let index = 0; index < 25; index++) batchInput[(index + 1) * 512000 - 1] = index + 1;
+    const splitBatch = await splitPiecesRabinKarp(new Blob([batchInput], { type: 'text/plain' }), 512000, true, 20);
+    const batchPieces = [];
+    for await (const piece of splitBatch()) batchPieces.push(piece);
+    assert.equal(batchPieces.length, 25);
+    assert(batchPieces.every(piece => Buffer.byteLength(piece) === 512000));
+    assert.equal(new Set(batchPieces).size, 25);
+    for (const count of [25, 20]) {
+        const docs = batchPieces.slice(0, count).map((piece, index) => ({
+            _id: `h:+${createHash('sha256').update(`escaped-batch-${count}-${index}`).digest('hex')}`,
+            type: 'leaf', data: piece,
+            _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] },
+        }));
+        const body = JSON.stringify({ docs, new_edits: false });
+        const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body });
+        const result = await response.json();
+        const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error);
+        const measurement = { count, rawBytesPerLeaf: 512000, requestBodyBytes: Buffer.byteLength(body), largestDocumentJsonBytes: Math.max(...docs.map(doc => Buffer.byteLength(JSON.stringify(doc)))), status: response.status, accepted, response: result };
+        record.batches.push(measurement);
+        await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2));
+        assert.equal(accepted, count === 20, JSON.stringify(measurement));
+        if (count === 25) assert.equal(response.status, 413);
+        console.log(JSON.stringify({ ...measurement, response: undefined }));
+    }
+} finally {
+    await fetch(`${uri}/${database}`, { method: 'DELETE', headers });
+    const restored = await fetch(uri + configPath, previous === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previous) });
+    record.configurationRestored = restored.ok;
+    const requestRestored = await fetch(uri + requestConfigPath, previousRequestLimit === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previousRequestLimit) });
+    record.requestConfigurationRestored = requestRestored.ok;
+    await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2));
+    assert(record.configurationRestored);
+    assert(record.requestConfigurationRestored);
+}