mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-10-25 02:23:22 +00:00
Prepare consistent Chunk sizes and fresh Fast Fetch integration
This commit is contained in:
vorotamoroz committed
1 parent
ce0524d49f
commit
d1e4c34054
24 files changed
+2042
-14
No files matched your search
@@ -0,0 +1,310 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFile, mkdir, readFile, readdir, stat, utimes, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DEFAULT_REPO_ROOT = path.resolve(MODULE_DIR, "../..");
|
||||
const SEED = 0x4c53594e;
|
||||
const KIB = 1024;
|
||||
const MIB = 1024 * KIB;
|
||||
const BINARY_BYTES = 8 * MIB;
|
||||
const SMALL_MARKDOWN_BYTES = 512 * KIB;
|
||||
const LARGE_MARKDOWN_BYTES = 5 * MIB;
|
||||
const BINARY_EDIT_BYTES = 4 * KIB;
|
||||
const INITIAL_MTIME = new Date("2020-01-01T00:00:00.000Z");
|
||||
const STAGE_MTIMES = {
|
||||
overwrite: new Date("2020-01-02T00:00:00.000Z"),
|
||||
insert: new Date("2020-01-03T00:00:00.000Z"),
|
||||
};
|
||||
|
||||
const MARKDOWN_FILES = [
|
||||
"README.md",
|
||||
"devs.md",
|
||||
"docs/settings.md",
|
||||
"docs/troubleshooting.md",
|
||||
"docs/quick_setup.md",
|
||||
"docs/datastructure.md",
|
||||
"docs/p2p.md",
|
||||
"docs/tips/fast-setup.md",
|
||||
"docs/design_docs/intention_of_chunks.md",
|
||||
"docs/specs_data_compression.md",
|
||||
];
|
||||
|
||||
const SNIPPET_FILES = [
|
||||
"docs/terms.md",
|
||||
"docs/glossary.md",
|
||||
"docs/quick_setup_ja.md",
|
||||
"docs/design_docs/unicode_path_identity.md",
|
||||
"docs/settings.md",
|
||||
];
|
||||
|
||||
const CORPUS_PATHS = {
|
||||
markdownSmall: "benchmark/generated-small.md",
|
||||
markdownLarge: "benchmark/generated-large.md",
|
||||
binary: "benchmark/random-8m.bin",
|
||||
cloudantImage: "assets/cloudant-setup.png",
|
||||
setupImage: "assets/quick-setup-first-setup-uri.png",
|
||||
};
|
||||
|
||||
function sha256(bytes) {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
function createPrng(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
return state >>> 0;
|
||||
};
|
||||
}
|
||||
|
||||
function makeRandomBytes(length, seed) {
|
||||
const next = createPrng(seed);
|
||||
const output = Buffer.allocUnsafe(length);
|
||||
for (let offset = 0; offset < length; offset += 4) {
|
||||
const value = next();
|
||||
output[offset] = value & 0xff;
|
||||
if (offset + 1 < length) output[offset + 1] = (value >>> 8) & 0xff;
|
||||
if (offset + 2 < length) output[offset + 2] = (value >>> 16) & 0xff;
|
||||
if (offset + 3 < length) output[offset + 3] = (value >>> 24) & 0xff;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function extractParagraphs(source) {
|
||||
const withoutFrontmatter = source.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/u, "");
|
||||
return withoutFrontmatter
|
||||
.replace(/\r\n/gu, "\n")
|
||||
.split(/\n{2,}/u)
|
||||
.map((paragraph) => paragraph.trim())
|
||||
.filter((paragraph) => {
|
||||
if (paragraph.length < 80) return false;
|
||||
if (/^(?:#{1,6}\s|!\[|\|)/u.test(paragraph)) return false;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
async function readSnippetParagraphs(repoRoot) {
|
||||
const excerpts = [];
|
||||
for (const file of SNIPPET_FILES) {
|
||||
const source = await readFile(path.join(repoRoot, file), "utf8");
|
||||
for (const paragraph of extractParagraphs(source)) {
|
||||
const excerpt = Array.from(paragraph).slice(0, 1000).join("");
|
||||
excerpts.push({ source: file, text: excerpt });
|
||||
}
|
||||
}
|
||||
if (excerpts.length === 0) throw new Error("No repository Markdown snippets were available.");
|
||||
return excerpts;
|
||||
}
|
||||
|
||||
function buildGeneratedMarkdown({ targetBytes, title, variant, excerpts }) {
|
||||
const lines = [
|
||||
`# ${title}`,
|
||||
"",
|
||||
"A deterministic UTF-8 fixture contains varied passages from the project documentation.",
|
||||
"",
|
||||
];
|
||||
let byteLength = Buffer.byteLength(lines.join("\n"), "utf8");
|
||||
const profiles = ["replication", "metadata", "recovery", "integrity", "configuration", "history", "incremental", "transport"];
|
||||
const unicodeLabels = ["résumé", "café", "日本語", "中文", "Δοκιμή", "ångström", "naïve", "雪", "λ", "mañana"];
|
||||
let record = 0;
|
||||
while (byteLength < targetBytes) {
|
||||
const excerpt = excerpts[(record * 7 + variant.length) % excerpts.length];
|
||||
const profile = profiles[(record * 3 + variant.length) % profiles.length];
|
||||
const unicode = unicodeLabels[(record * 5 + variant.length) % unicodeLabels.length];
|
||||
const tag = ((SEED ^ (record * 0x9e3779b1)) >>> 0).toString(16).padStart(8, "0");
|
||||
const paragraph = [
|
||||
`## Record ${String(record + 1).padStart(6, "0")} · ${profile} · ${variant}`,
|
||||
"",
|
||||
excerpt.text,
|
||||
"",
|
||||
`Fixture label ${tag} uses ${unicode}; source excerpt: ${excerpt.source}; revision ${record % 97}.`,
|
||||
"",
|
||||
].join("\n");
|
||||
lines.push(paragraph);
|
||||
byteLength += Buffer.byteLength(paragraph, "utf8");
|
||||
record += 1;
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function insertNearStart(markdown, paragraph) {
|
||||
const firstGap = markdown.indexOf("\n\n");
|
||||
const introGap = markdown.indexOf("\n\n", firstGap + 2);
|
||||
if (firstGap < 0 || introGap < 0) throw new Error("Generated Markdown fixture has no introductory paragraph.");
|
||||
const insertionPoint = introGap + 2;
|
||||
return `${markdown.slice(0, insertionPoint)}${paragraph}\n\n${markdown.slice(insertionPoint)}`;
|
||||
}
|
||||
|
||||
async function writeFileWithMtime(directory, relativePath, contents, mtime) {
|
||||
const outputPath = path.join(directory, relativePath);
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, contents);
|
||||
await utimes(outputPath, mtime, mtime);
|
||||
}
|
||||
|
||||
async function copyStage(sourceDirectory, targetDirectory, relativePaths) {
|
||||
for (const relativePath of relativePaths) {
|
||||
const sourcePath = path.join(sourceDirectory, relativePath);
|
||||
const targetPath = path.join(targetDirectory, relativePath);
|
||||
await mkdir(path.dirname(targetPath), { recursive: true });
|
||||
const sourceStat = await stat(sourcePath);
|
||||
await copyFile(sourcePath, targetPath);
|
||||
await utimes(targetPath, sourceStat.atime, sourceStat.mtime);
|
||||
}
|
||||
}
|
||||
|
||||
async function setMtime(directory, relativePath, mtime) {
|
||||
const outputPath = path.join(directory, relativePath);
|
||||
await utimes(outputPath, mtime, mtime);
|
||||
}
|
||||
|
||||
async function listFiles(directory) {
|
||||
const files = [];
|
||||
async function visit(currentDirectory) {
|
||||
const entries = await readdir(currentDirectory, { withFileTypes: true });
|
||||
entries.sort((left, right) => left.name.localeCompare(right.name));
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(currentDirectory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await visit(fullPath);
|
||||
} else if (entry.isFile()) {
|
||||
files.push(path.relative(directory, fullPath).split(path.sep).join("/"));
|
||||
}
|
||||
}
|
||||
}
|
||||
await visit(directory);
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
async function describeStage(stageName, stageDirectory, paths) {
|
||||
const files = [];
|
||||
for (const relativePath of paths) {
|
||||
const bytes = await readFile(path.join(stageDirectory, relativePath));
|
||||
files.push({ path: relativePath, bytes: bytes.byteLength, sha256: sha256(bytes) });
|
||||
}
|
||||
return { name: stageName, directory: stageName, files };
|
||||
}
|
||||
|
||||
export async function createCorpus(outDir, repoRoot = DEFAULT_REPO_ROOT) {
|
||||
const outputDirectory = path.resolve(outDir);
|
||||
const sourceRoot = path.resolve(repoRoot);
|
||||
const initialDirectory = path.join(outputDirectory, "initial");
|
||||
const overwriteDirectory = path.join(outputDirectory, "overwrite");
|
||||
const insertDirectory = path.join(outputDirectory, "insert");
|
||||
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
|
||||
const initialContents = new Map();
|
||||
for (const relativePath of MARKDOWN_FILES) {
|
||||
initialContents.set(relativePath, await readFile(path.join(sourceRoot, relativePath)));
|
||||
}
|
||||
|
||||
const excerpts = await readSnippetParagraphs(sourceRoot);
|
||||
initialContents.set(
|
||||
CORPUS_PATHS.markdownSmall,
|
||||
Buffer.from(
|
||||
buildGeneratedMarkdown({
|
||||
targetBytes: SMALL_MARKDOWN_BYTES,
|
||||
title: "Generated Markdown · 512 KiB",
|
||||
variant: "small",
|
||||
excerpts,
|
||||
}),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
initialContents.set(
|
||||
CORPUS_PATHS.markdownLarge,
|
||||
Buffer.from(
|
||||
buildGeneratedMarkdown({
|
||||
targetBytes: LARGE_MARKDOWN_BYTES,
|
||||
title: "Generated Markdown · 5 MiB",
|
||||
variant: "large",
|
||||
excerpts,
|
||||
}),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
initialContents.set(CORPUS_PATHS.binary, makeRandomBytes(BINARY_BYTES, SEED));
|
||||
initialContents.set(CORPUS_PATHS.cloudantImage, await readFile(path.join(sourceRoot, "instruction_images/cloudant_1.png")));
|
||||
initialContents.set(CORPUS_PATHS.setupImage, await readFile(path.join(sourceRoot, "images/quick-setup/guide-quick-setup-first-setup-uri.png")));
|
||||
|
||||
const relativePaths = [...initialContents.keys()].sort();
|
||||
for (const [relativePath, contents] of initialContents) {
|
||||
await writeFileWithMtime(initialDirectory, relativePath, contents, INITIAL_MTIME);
|
||||
}
|
||||
|
||||
await copyStage(initialDirectory, overwriteDirectory, relativePaths);
|
||||
const smallMarkdownPath = path.join(overwriteDirectory, CORPUS_PATHS.markdownSmall);
|
||||
const smallMarkdownBefore = await readFile(smallMarkdownPath, "utf8");
|
||||
const originalIntro = "A deterministic UTF-8 fixture contains varied passages from the project documentation.";
|
||||
const changedIntro = "A deterministic UTF-8 fixture records a short content update using varied project passages.";
|
||||
if (!smallMarkdownBefore.includes(originalIntro)) throw new Error("Could not locate the generated Markdown paragraph to overwrite.");
|
||||
await writeFile(smallMarkdownPath, smallMarkdownBefore.replace(originalIntro, changedIntro));
|
||||
const binaryPath = path.join(overwriteDirectory, CORPUS_PATHS.binary);
|
||||
const initialBinary = await readFile(binaryPath);
|
||||
const binaryOverwriteOffset = Math.floor(initialBinary.byteLength * 0.25);
|
||||
const overwrittenBinary = Buffer.from(initialBinary);
|
||||
makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x00a55a5a) >>> 0).copy(overwrittenBinary, binaryOverwriteOffset);
|
||||
await writeFile(binaryPath, overwrittenBinary);
|
||||
const alternatePng = await readFile(path.join(sourceRoot, "instruction_images/cloudant_2.png"));
|
||||
await writeFile(path.join(overwriteDirectory, CORPUS_PATHS.cloudantImage), alternatePng);
|
||||
for (const relativePath of [CORPUS_PATHS.markdownSmall, CORPUS_PATHS.binary, CORPUS_PATHS.cloudantImage]) {
|
||||
await setMtime(overwriteDirectory, relativePath, STAGE_MTIMES.overwrite);
|
||||
}
|
||||
|
||||
await copyStage(overwriteDirectory, insertDirectory, relativePaths);
|
||||
const insertedBinaryPath = path.join(insertDirectory, CORPUS_PATHS.binary);
|
||||
const overwrittenBinaryForInsert = await readFile(insertedBinaryPath);
|
||||
const binaryInsertOffset = Math.floor(overwrittenBinaryForInsert.byteLength * 0.25);
|
||||
const insertedBinary = Buffer.concat([
|
||||
overwrittenBinaryForInsert.subarray(0, binaryInsertOffset),
|
||||
makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x005aa5a5) >>> 0),
|
||||
overwrittenBinaryForInsert.subarray(binaryInsertOffset),
|
||||
]);
|
||||
await writeFile(insertedBinaryPath, insertedBinary);
|
||||
|
||||
const insertedParagraphs = [
|
||||
[CORPUS_PATHS.markdownSmall, "The insert stage adds record 0047, profile delta-small, while retaining Unicode labels such as 日本語 and café."],
|
||||
[CORPUS_PATHS.markdownLarge, "The insert stage adds record 0913, profile delta-large, while retaining Unicode labels such as 中文 and Δοκιμή."],
|
||||
];
|
||||
for (const [relativePath, paragraph] of insertedParagraphs) {
|
||||
const markdownPath = path.join(insertDirectory, relativePath);
|
||||
const markdown = await readFile(markdownPath, "utf8");
|
||||
await writeFile(markdownPath, insertNearStart(markdown, paragraph));
|
||||
}
|
||||
for (const relativePath of [CORPUS_PATHS.binary, ...insertedParagraphs.map(([relativePath]) => relativePath)]) {
|
||||
await setMtime(insertDirectory, relativePath, STAGE_MTIMES.insert);
|
||||
}
|
||||
|
||||
const actualPaths = await listFiles(initialDirectory);
|
||||
for (const directory of [overwriteDirectory, insertDirectory]) {
|
||||
const stagePaths = await listFiles(directory);
|
||||
if (JSON.stringify(stagePaths) !== JSON.stringify(actualPaths)) {
|
||||
throw new Error(`Corpus stage ${path.basename(directory)} has different relative paths.`);
|
||||
}
|
||||
}
|
||||
|
||||
const sourceForPath = new Map(MARKDOWN_FILES.map((file) => [file, `repo:${file}`]));
|
||||
sourceForPath.set(CORPUS_PATHS.markdownSmall, "generated:utf8-markdown-512KiB-from-repository-snippets");
|
||||
sourceForPath.set(CORPUS_PATHS.markdownLarge, "generated:utf8-markdown-5MiB-from-repository-snippets");
|
||||
sourceForPath.set(CORPUS_PATHS.binary, `generated:seeded-high-entropy-binary-seed-${SEED.toString(16)}`);
|
||||
sourceForPath.set(CORPUS_PATHS.cloudantImage, "repo:instruction_images/cloudant_1.png");
|
||||
sourceForPath.set(CORPUS_PATHS.setupImage, "repo:images/quick-setup/guide-quick-setup-first-setup-uri.png");
|
||||
|
||||
const files = actualPaths.map((relativePath) => {
|
||||
const kind = relativePath.endsWith(".md") ? "markdown" : "binary";
|
||||
return { path: relativePath, kind, source: sourceForPath.get(relativePath) };
|
||||
});
|
||||
const stages = [
|
||||
await describeStage("initial", initialDirectory, actualPaths),
|
||||
await describeStage("overwrite", overwriteDirectory, actualPaths),
|
||||
await describeStage("insert", insertDirectory, actualPaths),
|
||||
];
|
||||
const manifest = { seed: SEED, files, stages };
|
||||
await writeFile(path.join(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
return manifest;
|
||||
}
|
||||
Reference in new issue
Block a user