mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2025-12-19 04:31:29 +00:00
New features:
- Now remote database cleaning-up will be detected automatically. - A solution selection dialogue will be shown if synchronisation is rejected after cleaning or rebuilding the remote database. - During fetching or rebuilding, we can configure `Hidden file synchronisation` on the spot.
This commit is contained in:
vorotamoroz
295
src/utils.ts
295
src/utils.ts
@@ -1,10 +1,12 @@
|
||||
import { DataWriteOptions, normalizePath, TFile, Platform, TAbstractFile, App, Plugin_2 } from "./deps";
|
||||
import { DataWriteOptions, normalizePath, TFile, Platform, TAbstractFile, App, Plugin_2, RequestUrlParam, requestUrl } from "./deps";
|
||||
import { path2id_base, id2path_base, isValidFilenameInLinux, isValidFilenameInDarwin, isValidFilenameInWidows, isValidFilenameInAndroid, stripAllPrefixes } from "./lib/src/path";
|
||||
|
||||
import { Logger } from "./lib/src/logger";
|
||||
import { AnyEntry, DocumentID, EntryHasPath, FilePath, FilePathWithPrefix, LOG_LEVEL } from "./lib/src/types";
|
||||
import { AnyEntry, DocumentID, EntryDoc, EntryHasPath, FilePath, FilePathWithPrefix, LOG_LEVEL, NewEntry } from "./lib/src/types";
|
||||
import { CHeader, ICHeader, ICHeaderLength, PSCHeader } from "./types";
|
||||
import { InputStringDialog, PopoverSelectString } from "./dialogs";
|
||||
import ObsidianLiveSyncPlugin from "./main";
|
||||
import { runWithLock } from "./lib/src/lock";
|
||||
|
||||
// For backward compatibility, using the path for determining id.
|
||||
// Only CouchDB unacceptable ID (that starts with an underscore) has been prefixed with "/".
|
||||
@@ -428,3 +430,292 @@ export class PeriodicProcessor {
|
||||
if (this._timer) clearInterval(this._timer);
|
||||
}
|
||||
}
|
||||
|
||||
function sizeToHumanReadable(size: number | undefined) {
|
||||
if (!size) return "-";
|
||||
const i = Math.floor(Math.log(size) / Math.log(1024));
|
||||
return Number.parseInt((size / Math.pow(1024, i)).toFixed(2)) + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
|
||||
}
|
||||
|
||||
export const _requestToCouchDBFetch = async (baseUri: string, username: string, password: string, path?: string, body?: string | any, method?: string) => {
|
||||
const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
|
||||
const encoded = window.btoa(utf8str);
|
||||
const authHeader = "Basic " + encoded;
|
||||
const transformedHeaders: Record<string, string> = { authorization: authHeader, "content-type": "application/json" };
|
||||
const uri = `${baseUri}/${path}`;
|
||||
const requestParam = {
|
||||
url: uri,
|
||||
method: method || (body ? "PUT" : "GET"),
|
||||
headers: new Headers(transformedHeaders),
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify(body),
|
||||
};
|
||||
return await fetch(uri, requestParam);
|
||||
}
|
||||
|
||||
export const _requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, path?: string, body?: any, method?: string) => {
|
||||
const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
|
||||
const encoded = window.btoa(utf8str);
|
||||
const authHeader = "Basic " + encoded;
|
||||
const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin };
|
||||
const uri = `${baseUri}/${path}`;
|
||||
const requestParam: RequestUrlParam = {
|
||||
url: uri,
|
||||
method: method || (body ? "PUT" : "GET"),
|
||||
headers: transformedHeaders,
|
||||
contentType: "application/json",
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
};
|
||||
return await requestUrl(requestParam);
|
||||
}
|
||||
export const requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, key?: string, body?: string, method?: string) => {
|
||||
const uri = `_node/_local/_config${key ? "/" + key : ""}`;
|
||||
return await _requestToCouchDB(baseUri, username, password, origin, uri, body, method);
|
||||
};
|
||||
|
||||
export async function performRebuildDB(plugin: ObsidianLiveSyncPlugin, method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice") {
|
||||
if (method == "localOnly") {
|
||||
await plugin.addOnSetup.fetchLocal();
|
||||
}
|
||||
if (method == "remoteOnly") {
|
||||
await plugin.addOnSetup.rebuildRemote();
|
||||
}
|
||||
if (method == "rebuildBothByThisDevice") {
|
||||
await plugin.addOnSetup.rebuildEverything();
|
||||
}
|
||||
}
|
||||
|
||||
export const gatherChunkUsage = async (db: PouchDB.Database<EntryDoc>) => {
|
||||
const used = new Map();
|
||||
const unreferenced = new Map();
|
||||
const removed = new Map();
|
||||
const missing = new Map();
|
||||
const xx = await db.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
|
||||
for (const xxd of xx.rows) {
|
||||
const chunk = xxd.id
|
||||
unreferenced.set(chunk, xxd.value.rev);
|
||||
}
|
||||
|
||||
const x = await db.find({ limit: 999999999, selector: { children: { $exists: true, $type: "array" } }, fields: ["_id", "path", "mtime", "children"] });
|
||||
for (const temp of x.docs) {
|
||||
for (const chunk of (temp as NewEntry).children) {
|
||||
used.set(chunk, (used.has(chunk) ? used.get(chunk) : 0) + 1);
|
||||
if (unreferenced.has(chunk)) {
|
||||
removed.set(chunk, unreferenced.get(chunk));
|
||||
unreferenced.delete(chunk);
|
||||
} else {
|
||||
if (!removed.has(chunk)) {
|
||||
if (!missing.has(temp._id)) {
|
||||
missing.set(temp._id, []);
|
||||
}
|
||||
missing.get(temp._id).push(chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { used, unreferenced, missing };
|
||||
}
|
||||
|
||||
export const localDatabaseCleanUp = async (plugin: ObsidianLiveSyncPlugin, force: boolean, dryRun: boolean) => {
|
||||
|
||||
await runWithLock("clean-up:local", true, async () => {
|
||||
const db = plugin.localDatabase.localDatabase;
|
||||
if ((db as any)?.adapter != "indexeddb") {
|
||||
if (force && !dryRun) {
|
||||
Logger("Fetch from the remote database", LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
await performRebuildDB(plugin, "localOnly");
|
||||
return;
|
||||
} else {
|
||||
Logger("This feature requires enabling `Use new adapter`. Please enable it", LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
return;
|
||||
}
|
||||
}
|
||||
Logger(`The remote database has been locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
|
||||
const { unreferenced, missing } = await gatherChunkUsage(db);
|
||||
if (missing.size != 0) {
|
||||
Logger(`Some chunks are not found! We have to rescue`, LOG_LEVEL.NOTICE);
|
||||
Logger(missing, LOG_LEVEL.VERBOSE);
|
||||
} else {
|
||||
Logger(`All chunks are OK`, LOG_LEVEL.NOTICE);
|
||||
}
|
||||
const payload = {} as Record<string, string[]>;
|
||||
for (const [id, rev] of unreferenced) {
|
||||
payload[id] = [rev];
|
||||
}
|
||||
const removeItems = Object.keys(payload).length;
|
||||
if (removeItems == 0) {
|
||||
Logger(`No unreferenced chunks found (Local)`, LOG_LEVEL.NOTICE);
|
||||
await plugin.markRemoteResolved();
|
||||
}
|
||||
if (dryRun) {
|
||||
Logger(`There are ${removeItems} unreferenced chunks (Local)`, LOG_LEVEL.NOTICE);
|
||||
return;
|
||||
}
|
||||
Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
for (const [id, rev] of unreferenced) {
|
||||
//@ts-ignore
|
||||
const ret = await db.purge(id, rev);
|
||||
Logger(ret, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
plugin.localDatabase.refreshSettings();
|
||||
Logger(`Compacting local database...`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
await db.compact();
|
||||
await plugin.markRemoteResolved();
|
||||
Logger("Done!", LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
export const balanceChunks = async (plugin: ObsidianLiveSyncPlugin, dryRun: boolean) => {
|
||||
|
||||
await runWithLock("clean-up:balance", true, async () => {
|
||||
const localDB = plugin.localDatabase.localDatabase;
|
||||
Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
|
||||
const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
|
||||
if (typeof ret === "string") {
|
||||
Logger(`Connect error: ${ret}`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
return;
|
||||
}
|
||||
const localChunks = new Map<string, string>();
|
||||
const xx = await localDB.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
|
||||
for (const xxd of xx.rows) {
|
||||
const chunk = xxd.id
|
||||
localChunks.set(chunk, xxd.value.rev);
|
||||
}
|
||||
// const info = ret.info;
|
||||
const remoteDB = ret.db;
|
||||
const remoteChunks = new Map<string, string>();
|
||||
const xxr = await remoteDB.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
|
||||
for (const xxd of xxr.rows) {
|
||||
const chunk = xxd.id
|
||||
remoteChunks.set(chunk, xxd.value.rev);
|
||||
}
|
||||
const localToRemote = new Map<string, string>([...localChunks]);
|
||||
const remoteToLocal = new Map<string, string>([...remoteChunks]);
|
||||
for (const id of new Set([...localChunks.keys(), ...remoteChunks.keys()])) {
|
||||
if (remoteChunks.has(id)) {
|
||||
localToRemote.delete(id);
|
||||
}
|
||||
if (localChunks.has(id)) {
|
||||
remoteToLocal.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
function arrayToChunkedArray<T>(src: T[], size = 25) {
|
||||
const ret = [] as T[][];
|
||||
let i = 0;
|
||||
while (i < src.length) {
|
||||
ret.push(src.slice(i, i += size));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (localToRemote.size == 0) {
|
||||
Logger(`No chunks need to be sent`, LOG_LEVEL.NOTICE);
|
||||
} else {
|
||||
Logger(`${localToRemote.size} chunks need to be sent`, LOG_LEVEL.NOTICE);
|
||||
if (!dryRun) {
|
||||
const w = arrayToChunkedArray([...localToRemote]);
|
||||
for (const chunk of w) {
|
||||
for (const [id,] of chunk) {
|
||||
const queryRet = await localDB.allDocs({ keys: [id], include_docs: true });
|
||||
const docs = queryRet.rows.filter(e => !("error" in e)).map(x => x.doc);
|
||||
|
||||
const ret = await remoteDB.bulkDocs(docs, { new_edits: false });
|
||||
Logger(ret, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
}
|
||||
Logger(`Done! ${remoteToLocal.size} chunks have been sent`, LOG_LEVEL.NOTICE);
|
||||
}
|
||||
}
|
||||
if (remoteToLocal.size == 0) {
|
||||
Logger(`No chunks need to be retrieved`, LOG_LEVEL.NOTICE);
|
||||
} else {
|
||||
Logger(`${remoteToLocal.size} chunks need to be retrieved`, LOG_LEVEL.NOTICE);
|
||||
if (!dryRun) {
|
||||
const w = arrayToChunkedArray([...remoteToLocal]);
|
||||
for (const chunk of w) {
|
||||
for (const [id,] of chunk) {
|
||||
const queryRet = await remoteDB.allDocs({ keys: [id], include_docs: true });
|
||||
const docs = queryRet.rows.filter(e => !("error" in e)).map(x => x.doc);
|
||||
|
||||
const ret = await localDB.bulkDocs(docs, { new_edits: false });
|
||||
Logger(ret, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
}
|
||||
Logger(`Done! ${remoteToLocal.size} chunks have been retrieved`, LOG_LEVEL.NOTICE);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryRun: boolean) => {
|
||||
const getSize = function (info: PouchDB.Core.DatabaseInfo, key: "active" | "external" | "file") {
|
||||
return Number.parseInt((info as any)?.sizes?.[key] ?? 0);
|
||||
}
|
||||
await runWithLock("clean-up:remote", true, async () => {
|
||||
try {
|
||||
const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
|
||||
if (typeof ret === "string") {
|
||||
Logger(`Connect error: ${ret}`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
return;
|
||||
}
|
||||
const info = ret.info;
|
||||
Logger(JSON.stringify(info), LOG_LEVEL.VERBOSE, "clean-up-db");
|
||||
Logger(`Database active-size: ${sizeToHumanReadable(getSize(info, "active"))}, external-size:${sizeToHumanReadable(getSize(info, "external"))}, file-size: ${sizeToHumanReadable(getSize(info, "file"))}`, LOG_LEVEL.NOTICE);
|
||||
|
||||
if (!dryRun) {
|
||||
Logger(`The remote database has been locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
await plugin.markRemoteLocked(true);
|
||||
}
|
||||
Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
const db = ret.db;
|
||||
|
||||
const { unreferenced, missing } = await gatherChunkUsage(db);
|
||||
if (missing.size != 0) {
|
||||
Logger(`Some chunks are not found! We have to rescue`, LOG_LEVEL.NOTICE);
|
||||
Logger(missing, LOG_LEVEL.VERBOSE);
|
||||
} else {
|
||||
Logger(`All chunks are OK`, LOG_LEVEL.NOTICE);
|
||||
}
|
||||
const payload = {} as Record<string, string[]>;
|
||||
for (const [id, rev] of unreferenced) {
|
||||
payload[id] = [rev];
|
||||
}
|
||||
const removeItems = Object.keys(payload).length;
|
||||
if (removeItems == 0) {
|
||||
Logger(`No unreferenced chunk found (Remote)`, LOG_LEVEL.NOTICE);
|
||||
return;
|
||||
}
|
||||
if (dryRun) {
|
||||
Logger(`There are ${removeItems} unreferenced chunks (Remote)`, LOG_LEVEL.NOTICE);
|
||||
return;
|
||||
}
|
||||
Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
const rets = await _requestToCouchDBFetch(
|
||||
`${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
|
||||
plugin.settings.couchDB_USER,
|
||||
plugin.settings.couchDB_PASSWORD,
|
||||
"_purge",
|
||||
payload, "POST");
|
||||
// const result = await rets();
|
||||
Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
|
||||
Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
|
||||
await db.compact();
|
||||
const endInfo = await db.info();
|
||||
|
||||
Logger(`Processed database active-size: ${sizeToHumanReadable(getSize(endInfo, "active"))}, external-size:${sizeToHumanReadable(getSize(endInfo, "external"))}, file-size: ${sizeToHumanReadable(getSize(endInfo, "file"))}`, LOG_LEVEL.NOTICE);
|
||||
Logger(`Reduced sizes: active-size: ${sizeToHumanReadable(getSize(info, "active") - getSize(endInfo, "active"))}, external-size:${sizeToHumanReadable(getSize(info, "external") - getSize(endInfo, "external"))}, file-size: ${sizeToHumanReadable(getSize(info, "file") - getSize(endInfo, "file"))}`, LOG_LEVEL.NOTICE);
|
||||
Logger(JSON.stringify(endInfo), LOG_LEVEL.VERBOSE, "clean-up-db");
|
||||
Logger(`Local database cleaning up...`);
|
||||
await localDatabaseCleanUp(plugin, true, false);
|
||||
} catch (ex) {
|
||||
Logger("Failed to clean up db.")
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user