mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-02-09 22:01:35 +00:00
- Fixed: The Fetch that was broken at 0.17.33 has been fixed.
- Refactored again: Internal file sync, plug-in sync and Set up URI have been moved into each file.
This commit is contained in:
vorotamoroz
711
src/CmdHiddenFileSync.ts
Normal file
711
src/CmdHiddenFileSync.ts
Normal file
@@ -0,0 +1,711 @@
|
||||
import { Notice, normalizePath, PluginManifest } from "./deps";
|
||||
import { EntryDoc, LoadedEntry, LOG_LEVEL, InternalFileEntry } from "./lib/src/types";
|
||||
import { InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
|
||||
import { delay, isDocContentSame } from "./lib/src/utils";
|
||||
import { Logger } from "./lib/src/logger";
|
||||
import { PouchDB } from "./lib/src/pouchdb-browser.js";
|
||||
import { disposeMemoObject, id2path, memoIfNotExist, memoObject, path2id, retrieveMemoObject, scheduleTask, trimPrefix, isInternalMetadata, filename2idInternalMetadata, id2filenameInternalMetadata, PeriodicProcessor } from "./utils";
|
||||
import { WrappedNotice } from "./lib/src/wrapper";
|
||||
import { base64ToArrayBuffer, arrayBufferToBase64 } from "./lib/src/strbin";
|
||||
import { runWithLock } from "./lib/src/lock";
|
||||
import { Semaphore } from "./lib/src/semaphore";
|
||||
import { JsonResolveModal } from "./JsonResolveModal";
|
||||
import { LiveSyncCommands } from "./LiveSyncCommands";
|
||||
|
||||
export class HiddenFileSync extends LiveSyncCommands {
|
||||
periodicInternalFileScanProcessor: PeriodicProcessor = new PeriodicProcessor(this.plugin, async () => await this.syncInternalFilesAndDatabase("push", false));
|
||||
confirmPopup: WrappedNotice = null;
|
||||
get kvDB() {
|
||||
return this.plugin.kvDB;
|
||||
}
|
||||
ensureDirectoryEx(fullPath: string) {
|
||||
return this.plugin.ensureDirectoryEx(fullPath);
|
||||
}
|
||||
getConflictedDoc(path: string, rev: string) {
|
||||
return this.plugin.getConflictedDoc(path, rev);
|
||||
}
|
||||
onunload() {
|
||||
this.periodicInternalFileScanProcessor?.disable();
|
||||
}
|
||||
onload(): void | Promise<void> {
|
||||
this.plugin.addCommand({
|
||||
id: "livesync-scaninternal",
|
||||
name: "Sync hidden files",
|
||||
callback: () => {
|
||||
this.syncInternalFilesAndDatabase("safe", true);
|
||||
},
|
||||
});
|
||||
}
|
||||
async onInitializeDatabase(showNotice: boolean) {
|
||||
if (this.settings.syncInternalFiles) {
|
||||
try {
|
||||
Logger("Synchronizing hidden files...");
|
||||
await this.syncInternalFilesAndDatabase("push", showNotice);
|
||||
Logger("Synchronizing hidden files done");
|
||||
} catch (ex) {
|
||||
Logger("Synchronizing hidden files failed");
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
async beforeReplicate(showNotice: boolean) {
|
||||
if (this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication && !this.settings.watchInternalFileChanges) {
|
||||
await this.syncInternalFilesAndDatabase("push", showNotice);
|
||||
}
|
||||
}
|
||||
async onResume() {
|
||||
if (this.plugin.suspended)
|
||||
return;
|
||||
if (this.settings.syncInternalFiles) {
|
||||
await this.syncInternalFilesAndDatabase("safe", false);
|
||||
}
|
||||
this.periodicInternalFileScanProcessor.enable(this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
|
||||
}
|
||||
parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
|
||||
return false;
|
||||
}
|
||||
realizeSettingSyncMode(): Promise<void> {
|
||||
this.periodicInternalFileScanProcessor?.disable();
|
||||
this.periodicInternalFileScanProcessor.enable(this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
|
||||
return;
|
||||
}
|
||||
|
||||
procInternalFiles: string[] = [];
|
||||
async execInternalFile() {
|
||||
await runWithLock("execinternal", false, async () => {
|
||||
const w = [...this.procInternalFiles];
|
||||
this.procInternalFiles = [];
|
||||
Logger(`Applying hidden ${w.length} files change...`);
|
||||
await this.syncInternalFilesAndDatabase("pull", false, false, w);
|
||||
Logger(`Applying hidden ${w.length} files changed`);
|
||||
});
|
||||
}
|
||||
procInternalFile(filename: string) {
|
||||
this.procInternalFiles.push(filename);
|
||||
scheduleTask("procInternal", 500, async () => {
|
||||
await this.execInternalFile();
|
||||
});
|
||||
}
|
||||
|
||||
recentProcessedInternalFiles = [] as string[];
|
||||
async watchVaultRawEventsAsync(path: string) {
|
||||
const stat = await this.app.vault.adapter.stat(path);
|
||||
// sometimes folder is coming.
|
||||
if (stat && stat.type != "file")
|
||||
return;
|
||||
const storageMTime = ~~((stat && stat.mtime || 0) / 1000);
|
||||
const key = `${path}-${storageMTime}`;
|
||||
if (this.recentProcessedInternalFiles.contains(key)) {
|
||||
//If recently processed, it may caused by self.
|
||||
return;
|
||||
}
|
||||
this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100);
|
||||
const id = filename2idInternalMetadata(path);
|
||||
const filesOnDB = await this.localDatabase.getDBEntryMeta(id);
|
||||
const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000);
|
||||
|
||||
// Skip unchanged file.
|
||||
if (dbMTime == storageMTime) {
|
||||
// Logger(`STORAGE --> DB:${path}: (hidden) Nothing changed`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Do not compare timestamp. Always local data should be preferred except this plugin wrote one.
|
||||
if (storageMTime == 0) {
|
||||
await this.deleteInternalFileOnDatabase(path);
|
||||
} else {
|
||||
await this.storeInternalFileToDatabase({ path: path, ...stat });
|
||||
const pluginDir = this.app.vault.configDir + "/plugins/";
|
||||
const pluginFiles = ["manifest.json", "data.json", "style.css", "main.js"];
|
||||
if (path.startsWith(pluginDir) && pluginFiles.some(e => path.endsWith(e)) && this.settings.usePluginSync) {
|
||||
const pluginName = trimPrefix(path, pluginDir).split("/")[0];
|
||||
await this.plugin.addOnPluginAndTheirSettings.sweepPlugin(false, pluginName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async resolveConflictOnInternalFiles() {
|
||||
// Scan all conflicted internal files
|
||||
const conflicted = this.localDatabase.findEntries(ICHeader, ICHeaderEnd, { conflicts: true });
|
||||
for await (const doc of conflicted) {
|
||||
if (!("_conflicts" in doc))
|
||||
continue;
|
||||
if (isInternalMetadata(doc._id)) {
|
||||
await this.resolveConflictOnInternalFile(doc._id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async resolveConflictOnInternalFile(id: string): Promise<boolean> {
|
||||
try {
|
||||
// Retrieve data
|
||||
const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
|
||||
// If there is no conflict, return with false.
|
||||
if (!("_conflicts" in doc))
|
||||
return false;
|
||||
if (doc._conflicts.length == 0)
|
||||
return false;
|
||||
Logger(`Hidden file conflicted:${id2filenameInternalMetadata(id)}`);
|
||||
const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
|
||||
const revA = doc._rev;
|
||||
const revB = conflicts[0];
|
||||
|
||||
if (doc._id.endsWith(".json")) {
|
||||
const conflictedRev = conflicts[0];
|
||||
const conflictedRevNo = Number(conflictedRev.split("-")[0]);
|
||||
//Search
|
||||
const revFrom = (await this.localDatabase.localDatabase.get<EntryDoc>(id, { revs_info: true }));
|
||||
const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? "";
|
||||
const result = await this.plugin.mergeObject(id, commonBase, doc._rev, conflictedRev);
|
||||
if (result) {
|
||||
Logger(`Object merge:${id}`, LOG_LEVEL.INFO);
|
||||
const filename = id2filenameInternalMetadata(id);
|
||||
const isExists = await this.app.vault.adapter.exists(filename);
|
||||
if (!isExists) {
|
||||
await this.ensureDirectoryEx(filename);
|
||||
}
|
||||
await this.app.vault.adapter.write(filename, result);
|
||||
const stat = await this.app.vault.adapter.stat(filename);
|
||||
await this.storeInternalFileToDatabase({ path: filename, ...stat });
|
||||
await this.extractInternalFileFromDatabase(filename);
|
||||
await this.localDatabase.localDatabase.remove(id, revB);
|
||||
return this.resolveConflictOnInternalFile(id);
|
||||
} else {
|
||||
Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
|
||||
const docAMerge = await this.localDatabase.getDBEntry(id, { rev: revA });
|
||||
const docBMerge = await this.localDatabase.getDBEntry(id, { rev: revB });
|
||||
if (docAMerge != false && docBMerge != false) {
|
||||
if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
|
||||
await delay(200);
|
||||
// Again for other conflicted revisions.
|
||||
return this.resolveConflictOnInternalFile(id);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
|
||||
// determine which revision should been deleted.
|
||||
// simply check modified time
|
||||
const mtimeA = ("mtime" in doc && doc.mtime) || 0;
|
||||
const mtimeB = ("mtime" in revBDoc && revBDoc.mtime) || 0;
|
||||
// Logger(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`);
|
||||
// console.log(`mtime:${mtimeA} - ${mtimeB}`);
|
||||
const delRev = mtimeA < mtimeB ? revA : revB;
|
||||
// delete older one.
|
||||
await this.localDatabase.localDatabase.remove(id, delRev);
|
||||
Logger(`Older one has been deleted:${id2filenameInternalMetadata(id)}`);
|
||||
// check the file again
|
||||
return this.resolveConflictOnInternalFile(id);
|
||||
} catch (ex) {
|
||||
Logger("Failed to resolve conflict (Hidden)");
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: Tidy up. Even though it is experimental feature, So dirty...
|
||||
async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe", showMessage: boolean, files: InternalFileInfo[] | false = false, targetFiles: string[] | false = false) {
|
||||
await this.resolveConflictOnInternalFiles();
|
||||
const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
|
||||
Logger("Scanning hidden files.", logLevel, "sync_internal");
|
||||
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
|
||||
.replace(/\n| /g, "")
|
||||
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
|
||||
if (!files)
|
||||
files = await this.scanInternalFiles();
|
||||
const filesOnDB = ((await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
|
||||
|
||||
const allFileNamesSrc = [...new Set([...files.map(e => normalizePath(e.path)), ...filesOnDB.map(e => normalizePath(id2path(id2filenameInternalMetadata(e._id))))])];
|
||||
const allFileNames = allFileNamesSrc.filter(filename => !targetFiles || (targetFiles && targetFiles.indexOf(filename) !== -1));
|
||||
function compareMTime(a: number, b: number) {
|
||||
const wa = ~~(a / 1000);
|
||||
const wb = ~~(b / 1000);
|
||||
const diff = wa - wb;
|
||||
return diff;
|
||||
}
|
||||
|
||||
const fileCount = allFileNames.length;
|
||||
let processed = 0;
|
||||
let filesChanged = 0;
|
||||
// count updated files up as like this below:
|
||||
// .obsidian: 2
|
||||
// .obsidian/workspace: 1
|
||||
// .obsidian/plugins: 1
|
||||
// .obsidian/plugins/recent-files-obsidian: 1
|
||||
// .obsidian/plugins/recent-files-obsidian/data.json: 1
|
||||
const updatedFolders: { [key: string]: number; } = {};
|
||||
const countUpdatedFolder = (path: string) => {
|
||||
const pieces = path.split("/");
|
||||
let c = pieces.shift();
|
||||
let pathPieces = "";
|
||||
filesChanged++;
|
||||
while (c) {
|
||||
pathPieces += (pathPieces != "" ? "/" : "") + c;
|
||||
pathPieces = normalizePath(pathPieces);
|
||||
if (!(pathPieces in updatedFolders)) {
|
||||
updatedFolders[pathPieces] = 0;
|
||||
}
|
||||
updatedFolders[pathPieces]++;
|
||||
c = pieces.shift();
|
||||
}
|
||||
};
|
||||
const p = [] as Promise<void>[];
|
||||
const semaphore = Semaphore(10);
|
||||
// Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
|
||||
let caches: { [key: string]: { storageMtime: number; docMtime: number; }; } = {};
|
||||
caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number; }; }>("diff-caches-internal") || {};
|
||||
for (const filename of allFileNames) {
|
||||
processed++;
|
||||
if (processed % 100 == 0)
|
||||
Logger(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal");
|
||||
if (ignorePatterns.some(e => filename.match(e)))
|
||||
continue;
|
||||
|
||||
const fileOnStorage = files.find(e => e.path == filename);
|
||||
const fileOnDatabase = filesOnDB.find(e => e._id == filename2idInternalMetadata(id2path(filename)));
|
||||
const addProc = async (p: () => Promise<void>): Promise<void> => {
|
||||
const releaser = await semaphore.acquire(1);
|
||||
try {
|
||||
return p();
|
||||
} catch (ex) {
|
||||
Logger("Some process failed", logLevel);
|
||||
Logger(ex);
|
||||
} finally {
|
||||
releaser();
|
||||
}
|
||||
};
|
||||
const cache = filename in caches ? caches[filename] : { storageMtime: 0, docMtime: 0 };
|
||||
|
||||
p.push(addProc(async () => {
|
||||
if (fileOnStorage && fileOnDatabase) {
|
||||
// Both => Synchronize
|
||||
if (fileOnDatabase.mtime == cache.docMtime && fileOnStorage.mtime == cache.storageMtime) {
|
||||
return;
|
||||
}
|
||||
const nw = compareMTime(fileOnStorage.mtime, fileOnDatabase.mtime);
|
||||
if (nw > 0) {
|
||||
await this.storeInternalFileToDatabase(fileOnStorage);
|
||||
}
|
||||
if (nw < 0) {
|
||||
// skip if not extraction performed.
|
||||
if (!await this.extractInternalFileFromDatabase(filename))
|
||||
return;
|
||||
}
|
||||
// If process successfully updated or file contents are same, update cache.
|
||||
cache.docMtime = fileOnDatabase.mtime;
|
||||
cache.storageMtime = fileOnStorage.mtime;
|
||||
caches[filename] = cache;
|
||||
countUpdatedFolder(filename);
|
||||
} else if (!fileOnStorage && fileOnDatabase) {
|
||||
if (direction == "push") {
|
||||
if (fileOnDatabase.deleted)
|
||||
return;
|
||||
await this.deleteInternalFileOnDatabase(filename, false);
|
||||
} else if (direction == "pull") {
|
||||
if (await this.extractInternalFileFromDatabase(filename)) {
|
||||
countUpdatedFolder(filename);
|
||||
}
|
||||
} else if (direction == "safe") {
|
||||
if (fileOnDatabase.deleted)
|
||||
return;
|
||||
if (await this.extractInternalFileFromDatabase(filename)) {
|
||||
countUpdatedFolder(filename);
|
||||
}
|
||||
}
|
||||
} else if (fileOnStorage && !fileOnDatabase) {
|
||||
await this.storeInternalFileToDatabase(fileOnStorage);
|
||||
} else {
|
||||
throw new Error("Invalid state on hidden file sync");
|
||||
// Something corrupted?
|
||||
}
|
||||
}));
|
||||
}
|
||||
await Promise.all(p);
|
||||
await this.kvDB.set("diff-caches-internal", caches);
|
||||
|
||||
// When files has been retrieved from the database. they must be reloaded.
|
||||
if (direction == "pull" && filesChanged != 0) {
|
||||
const configDir = normalizePath(this.app.vault.configDir);
|
||||
// Show notification to restart obsidian when something has been changed in configDir.
|
||||
if (configDir in updatedFolders) {
|
||||
// Numbers of updated files that is below of configDir.
|
||||
let updatedCount = updatedFolders[configDir];
|
||||
try {
|
||||
//@ts-ignore
|
||||
const manifests = Object.values(this.app.plugins.manifests) as PluginManifest[];
|
||||
//@ts-ignore
|
||||
const enabledPlugins = this.app.plugins.enabledPlugins as Set<string>;
|
||||
const enabledPluginManifests = manifests.filter(e => enabledPlugins.has(e.id));
|
||||
for (const manifest of enabledPluginManifests) {
|
||||
if (manifest.dir in updatedFolders) {
|
||||
// If notified about plug-ins, reloading Obsidian may not be necessary.
|
||||
updatedCount -= updatedFolders[manifest.dir];
|
||||
const updatePluginId = manifest.id;
|
||||
const updatePluginName = manifest.name;
|
||||
const fragment = createFragment((doc) => {
|
||||
doc.createEl("span", null, (a) => {
|
||||
a.appendText(`Files in ${updatePluginName} has been updated, Press `);
|
||||
a.appendChild(a.createEl("a", null, (anchor) => {
|
||||
anchor.text = "HERE";
|
||||
anchor.addEventListener("click", async () => {
|
||||
Logger(`Unloading plugin: ${updatePluginName}`, LOG_LEVEL.NOTICE, "plugin-reload-" + updatePluginId);
|
||||
// @ts-ignore
|
||||
await this.app.plugins.unloadPlugin(updatePluginId);
|
||||
// @ts-ignore
|
||||
await this.app.plugins.loadPlugin(updatePluginId);
|
||||
Logger(`Plugin reloaded: ${updatePluginName}`, LOG_LEVEL.NOTICE, "plugin-reload-" + updatePluginId);
|
||||
});
|
||||
}));
|
||||
|
||||
a.appendText(` to reload ${updatePluginName}, or press elsewhere to dismiss this message.`);
|
||||
});
|
||||
});
|
||||
|
||||
const updatedPluginKey = "popupUpdated-" + updatePluginId;
|
||||
scheduleTask(updatedPluginKey, 1000, async () => {
|
||||
const popup = await memoIfNotExist(updatedPluginKey, () => new Notice(fragment, 0));
|
||||
//@ts-ignore
|
||||
const isShown = popup?.noticeEl?.isShown();
|
||||
if (!isShown) {
|
||||
memoObject(updatedPluginKey, new Notice(fragment, 0));
|
||||
}
|
||||
scheduleTask(updatedPluginKey + "-close", 20000, () => {
|
||||
const popup = retrieveMemoObject<Notice>(updatedPluginKey);
|
||||
if (!popup)
|
||||
return;
|
||||
//@ts-ignore
|
||||
if (popup?.noticeEl?.isShown()) {
|
||||
popup.hide();
|
||||
}
|
||||
disposeMemoObject(updatedPluginKey);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (ex) {
|
||||
Logger("Error on checking plugin status.");
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
|
||||
}
|
||||
|
||||
// If something changes left, notify for reloading Obsidian.
|
||||
if (updatedCount != 0) {
|
||||
const fragment = createFragment((doc) => {
|
||||
doc.createEl("span", null, (a) => {
|
||||
a.appendText(`Hidden files have been synchronized, Press `);
|
||||
a.appendChild(a.createEl("a", null, (anchor) => {
|
||||
anchor.text = "HERE";
|
||||
anchor.addEventListener("click", () => {
|
||||
// @ts-ignore
|
||||
this.app.commands.executeCommandById("app:reload");
|
||||
});
|
||||
}));
|
||||
|
||||
a.appendText(` to reload obsidian, or press elsewhere to dismiss this message.`);
|
||||
});
|
||||
});
|
||||
|
||||
scheduleTask("popupUpdated-" + configDir, 1000, () => {
|
||||
//@ts-ignore
|
||||
const isShown = this.confirmPopup?.noticeEl?.isShown();
|
||||
if (!isShown) {
|
||||
this.confirmPopup = new Notice(fragment, 0);
|
||||
}
|
||||
scheduleTask("popupClose" + configDir, 20000, () => {
|
||||
this.confirmPopup?.hide();
|
||||
this.confirmPopup = null;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Logger(`Hidden files scanned: ${filesChanged} files had been modified`, logLevel, "sync_internal");
|
||||
}
|
||||
|
||||
async storeInternalFileToDatabase(file: InternalFileInfo, forceWrite = false) {
|
||||
const id = filename2idInternalMetadata(path2id(file.path));
|
||||
const contentBin = await this.app.vault.adapter.readBinary(file.path);
|
||||
let content: string[];
|
||||
try {
|
||||
content = await arrayBufferToBase64(contentBin);
|
||||
} catch (ex) {
|
||||
Logger(`The file ${file.path} could not be encoded`);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
return false;
|
||||
}
|
||||
const mtime = file.mtime;
|
||||
return await runWithLock("file-" + id, false, async () => {
|
||||
try {
|
||||
const old = await this.localDatabase.getDBEntry(id, null, false, false);
|
||||
let saveData: LoadedEntry;
|
||||
if (old === false) {
|
||||
saveData = {
|
||||
_id: id,
|
||||
data: content,
|
||||
mtime,
|
||||
ctime: mtime,
|
||||
datatype: "newnote",
|
||||
size: file.size,
|
||||
children: [],
|
||||
deleted: false,
|
||||
type: "newnote",
|
||||
};
|
||||
} else {
|
||||
if (isDocContentSame(old.data, content) && !forceWrite) {
|
||||
// Logger(`internal files STORAGE --> DB:${file.path}: Not changed`);
|
||||
return;
|
||||
}
|
||||
saveData =
|
||||
{
|
||||
...old,
|
||||
data: content,
|
||||
mtime,
|
||||
size: file.size,
|
||||
datatype: "newnote",
|
||||
children: [],
|
||||
deleted: false,
|
||||
type: "newnote",
|
||||
};
|
||||
}
|
||||
|
||||
const ret = await this.localDatabase.putDBEntry(saveData, true);
|
||||
Logger(`STORAGE --> DB:${file.path}: (hidden) Done`);
|
||||
return ret;
|
||||
} catch (ex) {
|
||||
Logger(`STORAGE --> DB:${file.path}: (hidden) Failed`);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async deleteInternalFileOnDatabase(filename: string, forceWrite = false) {
|
||||
const id = filename2idInternalMetadata(path2id(filename));
|
||||
const mtime = new Date().getTime();
|
||||
await runWithLock("file-" + id, false, async () => {
|
||||
try {
|
||||
const old = await this.localDatabase.getDBEntry(id, null, false, false) as InternalFileEntry | false;
|
||||
let saveData: InternalFileEntry;
|
||||
if (old === false) {
|
||||
saveData = {
|
||||
_id: id,
|
||||
mtime,
|
||||
ctime: mtime,
|
||||
size: 0,
|
||||
children: [],
|
||||
deleted: true,
|
||||
type: "newnote",
|
||||
};
|
||||
} else {
|
||||
if (old.deleted) {
|
||||
Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`);
|
||||
return;
|
||||
}
|
||||
saveData =
|
||||
{
|
||||
...old,
|
||||
mtime,
|
||||
size: 0,
|
||||
children: [],
|
||||
deleted: true,
|
||||
type: "newnote",
|
||||
};
|
||||
}
|
||||
await this.localDatabase.localDatabase.put(saveData);
|
||||
Logger(`STORAGE -x> DB:${filename}: (hidden) Done`);
|
||||
} catch (ex) {
|
||||
Logger(`STORAGE -x> DB:${filename}: (hidden) Failed`);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async extractInternalFileFromDatabase(filename: string, force = false) {
|
||||
const isExists = await this.app.vault.adapter.exists(filename);
|
||||
const id = filename2idInternalMetadata(path2id(filename));
|
||||
|
||||
return await runWithLock("file-" + id, false, async () => {
|
||||
try {
|
||||
// Check conflicted status
|
||||
//TODO option
|
||||
const fileOnDB = await this.localDatabase.getDBEntry(id, { conflicts: true }, false, false) as false | LoadedEntry;
|
||||
if (fileOnDB === false)
|
||||
throw new Error(`File not found on database.:${id}`);
|
||||
// Prevent overrite for Prevent overwriting while some conflicted revision exists.
|
||||
if (fileOnDB?._conflicts?.length) {
|
||||
Logger(`Hidden file ${id} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL.INFO);
|
||||
return;
|
||||
}
|
||||
const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false;
|
||||
if (deleted) {
|
||||
if (!isExists) {
|
||||
Logger(`STORAGE <x- DB:${filename}: deleted (hidden) Deleted on DB, but the file is already not found on storage.`);
|
||||
} else {
|
||||
Logger(`STORAGE <x- DB:${filename}: deleted (hidden).`);
|
||||
await this.app.vault.adapter.remove(filename);
|
||||
try {
|
||||
//@ts-ignore internalAPI
|
||||
await app.vault.adapter.reconcileInternalFile(filename);
|
||||
} catch (ex) {
|
||||
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (!isExists) {
|
||||
await this.ensureDirectoryEx(filename);
|
||||
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
|
||||
try {
|
||||
//@ts-ignore internalAPI
|
||||
await app.vault.adapter.reconcileInternalFile(filename);
|
||||
} catch (ex) {
|
||||
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
Logger(`STORAGE <-- DB:${filename}: written (hidden,new${force ? ", force" : ""})`);
|
||||
return true;
|
||||
} else {
|
||||
const contentBin = await this.app.vault.adapter.readBinary(filename);
|
||||
const content = await arrayBufferToBase64(contentBin);
|
||||
if (content == fileOnDB.data && !force) {
|
||||
// Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL.VERBOSE);
|
||||
return true;
|
||||
}
|
||||
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
|
||||
try {
|
||||
//@ts-ignore internalAPI
|
||||
await app.vault.adapter.reconcileInternalFile(filename);
|
||||
} catch (ex) {
|
||||
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""})`);
|
||||
return true;
|
||||
|
||||
}
|
||||
} catch (ex) {
|
||||
Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""}) Failed`);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
|
||||
return new Promise((res) => {
|
||||
Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE);
|
||||
const docs = [docA, docB];
|
||||
const modal = new JsonResolveModal(this.app, id2path(docA._id), [docA, docB], async (keep, result) => {
|
||||
// modal.close();
|
||||
try {
|
||||
const filename = id2filenameInternalMetadata(docA._id);
|
||||
let needFlush = false;
|
||||
if (!result && !keep) {
|
||||
Logger(`Skipped merging: ${filename}`);
|
||||
}
|
||||
//Delete old revisions
|
||||
if (result || keep) {
|
||||
for (const doc of docs) {
|
||||
if (doc._rev != keep) {
|
||||
if (await this.localDatabase.deleteDBEntry(doc._id, { rev: doc._rev })) {
|
||||
Logger(`Conflicted revision has been deleted: ${filename}`);
|
||||
needFlush = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!keep && result) {
|
||||
const isExists = await this.app.vault.adapter.exists(filename);
|
||||
if (!isExists) {
|
||||
await this.ensureDirectoryEx(filename);
|
||||
}
|
||||
await this.app.vault.adapter.write(filename, result);
|
||||
const stat = await this.app.vault.adapter.stat(filename);
|
||||
await this.storeInternalFileToDatabase({ path: filename, ...stat }, true);
|
||||
try {
|
||||
//@ts-ignore internalAPI
|
||||
await app.vault.adapter.reconcileInternalFile(filename);
|
||||
} catch (ex) {
|
||||
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
}
|
||||
Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`);
|
||||
}
|
||||
if (needFlush) {
|
||||
await this.extractInternalFileFromDatabase(filename, false);
|
||||
Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`);
|
||||
}
|
||||
res(true);
|
||||
} catch (ex) {
|
||||
Logger("Could not merge conflicted json");
|
||||
Logger(ex, LOG_LEVEL.VERBOSE);
|
||||
res(false);
|
||||
}
|
||||
});
|
||||
modal.open();
|
||||
});
|
||||
}
|
||||
|
||||
async scanInternalFiles(): Promise<InternalFileInfo[]> {
|
||||
const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
|
||||
.replace(/\n| /g, "")
|
||||
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
|
||||
const root = this.app.vault.getRoot();
|
||||
const findRoot = root.path;
|
||||
const filenames = (await this.getFiles(findRoot, [], null, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
|
||||
const files = filenames.map(async (e) => {
|
||||
return {
|
||||
path: e,
|
||||
stat: await this.app.vault.adapter.stat(e)
|
||||
};
|
||||
});
|
||||
const result: InternalFileInfo[] = [];
|
||||
for (const f of files) {
|
||||
const w = await f;
|
||||
result.push({
|
||||
...w,
|
||||
...w.stat
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
async getFiles(
|
||||
path: string,
|
||||
ignoreList: string[],
|
||||
filter: RegExp[],
|
||||
ignoreFilter: RegExp[]
|
||||
) {
|
||||
|
||||
const w = await this.app.vault.adapter.list(path);
|
||||
let files = [
|
||||
...w.files
|
||||
.filter((e) => !ignoreList.some((ee) => e.endsWith(ee)))
|
||||
.filter((e) => !filter || filter.some((ee) => e.match(ee)))
|
||||
.filter((e) => !ignoreFilter || ignoreFilter.every((ee) => !e.match(ee))),
|
||||
];
|
||||
|
||||
L1: for (const v of w.folders) {
|
||||
for (const ignore of ignoreList) {
|
||||
if (v.endsWith(ignore)) {
|
||||
continue L1;
|
||||
}
|
||||
}
|
||||
if (ignoreFilter && ignoreFilter.some(e => v.match(e))) {
|
||||
continue L1;
|
||||
}
|
||||
files = files.concat(await this.getFiles(v, ignoreList, filter, ignoreFilter));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user