diff --git a/utils/.gitignore b/utils/.gitignore new file mode 100644 index 0000000..642838a --- /dev/null +++ b/utils/.gitignore @@ -0,0 +1 @@ +fly.toml \ No newline at end of file diff --git a/utils/couchdb/couchdb-init.sh b/utils/couchdb/couchdb-init.sh new file mode 100755 index 0000000..a18fa03 --- /dev/null +++ b/utils/couchdb/couchdb-init.sh @@ -0,0 +1,29 @@ +#!/bin/bash +if [[ -z "$hostname" ]]; then + echo "ERROR: Hostname missing" + exit 1 +fi +if [[ -z "$username" ]]; then + echo "ERROR: Username missing" + exit 1 +fi + +if [[ -z "$password" ]]; then + echo "ERROR: Password missing" + exit 1 +fi + +echo "-- Configuring CouchDB by REST APIs... -->" + +until (curl -X POST "${hostname}/_cluster_setup" -H "Content-Type: application/json" -d "{\"action\":\"enable_single_node\",\"username\":\"${username}\",\"password\":\"${password}\",\"bind_address\":\"0.0.0.0\",\"port\":5984,\"singlenode\":true}" --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/require_valid_user" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd_auth/require_valid_user" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/httpd/WWW-Authenticate" -H "Content-Type: application/json" -d '"Basic realm=\"couchdb\""' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/httpd/enable_cors" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/enable_cors" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/max_http_request_size" -H "Content-Type: application/json" -d '"4294967296"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/couchdb/max_document_size" -H "Content-Type: application/json" -d '"50000000"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/cors/credentials" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done +until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/cors/origins" -H "Content-Type: application/json" -d '"app://obsidian.md,capacitor://localhost,http://localhost"' --user "${username}:${password}"); do sleep 5; done + +echo "<-- Configuring CouchDB by REST APIs Done!" \ No newline at end of file diff --git a/utils/flyio/delete-server.sh b/utils/flyio/delete-server.sh new file mode 100755 index 0000000..1f51c51 --- /dev/null +++ b/utils/flyio/delete-server.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +fly scale count 0 -y +fly apps destroy $(fly status -j | jq -r .Name) -y diff --git a/utils/flyio/deploy-server.sh b/utils/flyio/deploy-server.sh new file mode 100755 index 0000000..56476ec --- /dev/null +++ b/utils/flyio/deploy-server.sh @@ -0,0 +1,43 @@ +#!/bin/bash +## Script for deploy and automatic setup CouchDB onto fly.io. +## We need Deno for generating the Setup-URI. + +source setenv.sh $@ + +export hostname="https://$appname.fly.dev" + +echo "-- YOUR CONFIGURATION --" +echo "URL : $hostname" +echo "username: $username" +echo "password: $password" +echo "region : $region" +echo "" +echo "-- START DEPLOYING --> " + +set -e +fly launch --name=$appname --env="COUCHDB_USER=$username" --copy-config=true --detach --no-deploy --region ${region} --yes +fly secrets set COUCHDB_PASSWORD=$password +fly deploy + +set +e +../couchdb/couchdb-init.sh +# flyctl deploy +echo "OK!" + +if command -v deno >/dev/null 2>&1; then + echo "Setup finished! Also, we can set up Self-hosted LiveSync instantly, by the following setup uri." + echo "Passphrase of setup-uri is \`welcome\`". + echo "--- configured ---" + echo "database : ${database}" + echo "E2EE passphrase: ${passphrase}" + echo "--- setup uri ---" + deno run -A generate_setupuri.ts +else + echo "Setup finished! Here is the configured values (reprise)!" + echo "-- YOUR CONFIGURATION --" + echo "URL : $hostname" + echo "username: $username" + echo "password: $password" + echo "-- YOUR CONFIGURATION --" + echo "If we had Deno, we would got the setup uri directly!" +fi diff --git a/utils/flyio/fly.template.toml b/utils/flyio/fly.template.toml new file mode 100644 index 0000000..cdfd1b3 --- /dev/null +++ b/utils/flyio/fly.template.toml @@ -0,0 +1,40 @@ +## CouchDB for fly.io image + +app = '' +primary_region = 'nrt' +swap_size_mb = 512 + +[build] + image = "couchdb:latest" + +[mounts] + source = "couchdata" + destination = "/opt/couchdb/data" + initial_size = "1GB" + auto_extend_size_threshold = 90 + auto_extend_size_increment = "1GB" + auto_extend_size_limit = "2GB" + +[env] + COUCHDB_USER = "" + ERL_FLAGS = "-couch_ini /opt/couchdb/etc/default.ini /opt/couchdb/etc/default.d/ /opt/couchdb/etc/local.d /opt/couchdb/etc/local.ini /opt/couchdb/data/persistence.ini" + +[http_service] + internal_port = 5984 + force_https = true + auto_stop_machines = true + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[vm]] + cpu_kind = 'shared' + cpus = 1 + memory_mb = 256 + +[[files]] + guest_path = "/docker-entrypoint2.sh" + raw_value = "#!/bin/bash\ntouch /opt/couchdb/data/persistence.ini\nchmod +w /opt/couchdb/data/persistence.ini\n/docker-entrypoint.sh $@" + +[experimental] + entrypoint = ["tini", "--", "/docker-entrypoint2.sh"] diff --git a/utils/flyio/generate_setupuri.ts b/utils/flyio/generate_setupuri.ts new file mode 100644 index 0000000..03bb607 --- /dev/null +++ b/utils/flyio/generate_setupuri.ts @@ -0,0 +1,180 @@ +import { webcrypto } from "node:crypto"; + +const KEY_RECYCLE_COUNT = 100; +type KeyBuffer = { + key: CryptoKey; + salt: Uint8Array; + count: number; +}; + +let semiStaticFieldBuffer: Uint8Array; +const nonceBuffer: Uint32Array = new Uint32Array(1); +const writeString = (string: string) => { + // Prepare enough buffer. + const buffer = new Uint8Array(string.length * 4); + const length = string.length; + let index = 0; + let chr = 0; + let idx = 0; + while (idx < length) { + chr = string.charCodeAt(idx++); + if (chr < 128) { + buffer[index++] = chr; + } else if (chr < 0x800) { + // 2 bytes + buffer[index++] = 0xC0 | (chr >>> 6); + buffer[index++] = 0x80 | (chr & 0x3F); + } else if (chr < 0xD800 || chr > 0xDFFF) { + // 3 bytes + buffer[index++] = 0xE0 | (chr >>> 12); + buffer[index++] = 0x80 | ((chr >>> 6) & 0x3F); + buffer[index++] = 0x80 | (chr & 0x3F); + } else { + // 4 bytes - surrogate pair + chr = (((chr - 0xD800) << 10) | (string.charCodeAt(idx++) - 0xDC00)) + 0x10000; + buffer[index++] = 0xF0 | (chr >>> 18); + buffer[index++] = 0x80 | ((chr >>> 12) & 0x3F); + buffer[index++] = 0x80 | ((chr >>> 6) & 0x3F); + buffer[index++] = 0x80 | (chr & 0x3F); + } + } + return buffer.slice(0, index); +}; +const KeyBuffs = new Map(); +async function getKeyForEncrypt(passphrase: string, autoCalculateIterations: boolean): Promise<[CryptoKey, Uint8Array]> { + // For performance, the plugin reuses the key KEY_RECYCLE_COUNT times. + const buffKey = `${passphrase}-${autoCalculateIterations}`; + const f = KeyBuffs.get(buffKey); + if (f) { + f.count--; + if (f.count > 0) { + return [f.key, f.salt]; + } + f.count--; + } + const passphraseLen = 15 - passphrase.length; + const iteration = autoCalculateIterations ? ((passphraseLen > 0 ? passphraseLen : 0) * 1000) + 121 - passphraseLen : 100000; + const passphraseBin = new TextEncoder().encode(passphrase); + const digest = await webcrypto.subtle.digest({ name: "SHA-256" }, passphraseBin); + const keyMaterial = await webcrypto.subtle.importKey("raw", digest, { name: "PBKDF2" }, false, ["deriveKey"]); + const salt = webcrypto.getRandomValues(new Uint8Array(16)); + const key = await webcrypto.subtle.deriveKey( + { + name: "PBKDF2", + salt, + iterations: iteration, + hash: "SHA-256", + }, + keyMaterial, + { name: "AES-GCM", length: 256 }, + false, + ["encrypt"] + ); + KeyBuffs.set(buffKey, { + key, + salt, + count: KEY_RECYCLE_COUNT, + }); + return [key, salt]; +} + +function getSemiStaticField(reset?: boolean) { + // return fixed field of iv. + if (semiStaticFieldBuffer != null && !reset) { + return semiStaticFieldBuffer; + } + semiStaticFieldBuffer = webcrypto.getRandomValues(new Uint8Array(12)); + return semiStaticFieldBuffer; +} + +function getNonce() { + // This is nonce, so do not send same thing. + nonceBuffer[0]++; + if (nonceBuffer[0] > 10000) { + // reset semi-static field. + getSemiStaticField(true); + } + return nonceBuffer; +} +function arrayBufferToBase64internalBrowser(buffer: DataView | Uint8Array): Promise { + return new Promise((res, rej) => { + const blob = new Blob([buffer], { type: "application/octet-binary" }); + const reader = new FileReader(); + reader.onload = function (evt) { + const dataURI = evt.target?.result?.toString() || ""; + if (buffer.byteLength != 0 && (dataURI == "" || dataURI == "data:")) return rej(new TypeError("Could not parse the encoded string")); + const result = dataURI.substring(dataURI.indexOf(",") + 1); + res(result); + }; + reader.readAsDataURL(blob); + }); +} + +// Map for converting hexString +const revMap: { [key: string]: number } = {}; +const numMap: { [key: number]: string } = {}; +for (let i = 0; i < 256; i++) { + revMap[(`00${i.toString(16)}`.slice(-2))] = i; + numMap[i] = (`00${i.toString(16)}`.slice(-2)); +} + + +function uint8ArrayToHexString(src: Uint8Array): string { + return [...src].map(e => numMap[e]).join(""); +} + +const QUANTUM = 32768; +async function arrayBufferToBase64Single(buffer: ArrayBuffer): Promise { + const buf = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer); + if (buf.byteLength < QUANTUM) return btoa(String.fromCharCode.apply(null, [...buf])); + return await arrayBufferToBase64internalBrowser(buf); +} + + +export async function encrypt(input: string, passphrase: string, autoCalculateIterations: boolean) { + const [key, salt] = await getKeyForEncrypt(passphrase, autoCalculateIterations); + // Create initial vector with semi-fixed part and incremental part + // I think it's not good against related-key attacks. + const fixedPart = getSemiStaticField(); + const invocationPart = getNonce(); + const iv = new Uint8Array([...fixedPart, ...new Uint8Array(invocationPart.buffer)]); + const plainStringified = JSON.stringify(input); + + // const plainStringBuffer: Uint8Array = tex.encode(plainStringified) + const plainStringBuffer: Uint8Array = writeString(plainStringified); + const encryptedDataArrayBuffer = await webcrypto.subtle.encrypt({ name: "AES-GCM", iv }, key, plainStringBuffer); + const encryptedData2 = (await arrayBufferToBase64Single(encryptedDataArrayBuffer)); + //return data with iv and salt. + const ret = `["${encryptedData2}","${uint8ArrayToHexString(iv)}","${uint8ArrayToHexString(salt)}"]`; + return ret; +} + +const URIBASE = "obsidian://setuplivesync?settings="; +async function main() { + const conf = { + "couchDB_URI": `${Deno.env.get("hostname")}`, + "couchDB_USER": `${Deno.env.get("username")}`, + "couchDB_PASSWORD": `${Deno.env.get("password")}`, + "couchDB_DBNAME": `${Deno.env.get("database")}`, + "syncOnStart": true, + "gcDelay": 0, + "periodicReplication": true, + "syncOnFileOpen": true, + "encrypt": true, + "passphrase": `${Deno.env.get("passphrase")}`, + "usePathObfuscation": true, + "batchSave": true, + "batch_size": 50, + "batches_limit": 50, + "useHistory": true, + "disableRequestURI": true, + "customChunkSize": 50, + "syncAfterMerge": false, + "concurrencyOfReadChunksOnline": 100, + "minimumIntervalOfReadChunksOnline": 100, + } + const encryptedConf = encodeURIComponent(await encrypt(JSON.stringify(conf), "welcome", false)); + const theURI = `${URIBASE}${encryptedConf}`; + console.log(theURI); +} +await main(); \ No newline at end of file diff --git a/utils/flyio/setenv.sh b/utils/flyio/setenv.sh new file mode 100755 index 0000000..1c1f4c1 --- /dev/null +++ b/utils/flyio/setenv.sh @@ -0,0 +1,30 @@ +random_num() { + echo $RANDOM +} +random_noun() { + nouns=("waterfall" "river" "breeze" "moon" "rain" "wind" "sea" "morning" "snow" "lake" "sunset" "pine" "shadow" "leaf" "dawn" "glitter" "forest" "hill" "cloud" "meadow" "sun" "glade" "bird" "brook" "butterfly" "bush" "dew" "dust" "field" "fire" "flower" "firefly" "feather" "grass" "haze" "mountain" "night" "pond" "darkness" "snowflake" "silence" "sound" "sky" "shape" "surf" "thunder" "violet" "water" "wildflower" "wave" "water" "resonance" "sun" "log" "dream" "cherry" "tree" "fog" "frost" "voice" "paper" "frog" "smoke" "star") + echo ${nouns[$(($RANDOM % ${#nouns[*]}))]} +} + +random_adjective() { + adjectives=("autumn" "hidden" "bitter" "misty" "silent" "empty" "dry" "dark" "summer" "icy" "delicate" "quiet" "white" "cool" "spring" "winter" "patient" "twilight" "dawn" "crimson" "wispy" "weathered" "blue" "billowing" "broken" "cold" "damp" "falling" "frosty" "green" "long" "late" "lingering" "bold" "little" "morning" "muddy" "old" "red" "rough" "still" "small" "sparkling" "thrumming" "shy" "wandering" "withered" "wild" "black" "young" "holy" "solitary" "fragrant" "aged" "snowy" "proud" "floral" "restless" "divine" "polished" "ancient" "purple" "lively" "nameless") + echo ${adjectives[$(($RANDOM % ${#adjectives[*]}))]} +} + +cp ./fly.template.toml ./fly.toml + +if [ "$1" = "renew" ]; then + unset appname + unset username + unset password + unset database + unset passphrase + unset region +fi + +[ -z $appname ] && export appname=$(random_adjective)-$(random_noun)-$(random_num) +[ -z $username ] && export username=$(random_adjective)-$(random_noun)-$(random_num) +[ -z $password ] && export password=$(random_adjective)-$(random_noun)-$(random_num) +[ -z $database ] && export database="obsidiannotes" +[ -z $passphrase ] && export passphrase=$(random_adjective)-$(random_noun)-$(random_num) +[ -z $region ] && export region="nrt" diff --git a/utils/readme.md b/utils/readme.md new file mode 100644 index 0000000..6cc5540 --- /dev/null +++ b/utils/readme.md @@ -0,0 +1,164 @@ + +# Utilities +Here are some useful things. + +## couchdb + +### couchdb-init.sh +This script can configure CouchDB with the necessary settings by REST APIs. + +#### Materials +- Mandatory: curl + +#### Usage + +```sh +export hostname=http://localhost:5984/ +export username=couchdb-admin-username +export password=couchdb-admin-password +./couchdb-init.sh +``` + +curl result will be shown, however, all of them can be ignored if the script has been run completely. + +## fly.io + +### deploy-server.sh + +A fully automated CouchDB deployment script. We can deploy CouchDB onto fly.io. The only we need is an account of it. + +All omitted configurations will be determined at random. (And, it is preferred). The region is configured to `nrt`. +If Japan is not close to you, please choose a region closer to you. However, the deployed database will work if you leave it at all. + +#### Materials +- Mandatory: curl, flyctl +- Recommended: deno + +#### Usage +```sh +#export appname= +#export username= +#export password= +#export database= +#export passphrase= +export region=nrt #pick your nearest location +./deploy-server.sh +``` + +The result of this command is as follows. + +``` +-- YOUR CONFIGURATION -- +URL : https://young-darkness-25342.fly.dev +username: billowing-cherry-22580 +password: misty-dew-13571 +region : nrt + +-- START DEPLOYING --> +An existing fly.toml file was found +Using build strategies '[the "couchdb:latest" docker image]'. Remove [build] from fly.toml to force a rescan +Creating app in /home/vorotamoroz/dev/obsidian-livesync/utils/flyio +We're about to launch your app on Fly.io. Here's what you're getting: + +Organization: vorotamoroz (fly launch defaults to the personal org) +Name: young-darkness-25342 (specified on the command line) +Region: Tokyo, Japan (specified on the command line) +App Machines: shared-cpu-1x, 256MB RAM (specified on the command line) +Postgres: (not requested) +Redis: (not requested) + +Created app 'young-darkness-25342' in organization 'personal' +Admin URL: https://fly.io/apps/young-darkness-25342 +Hostname: young-darkness-25342.fly.dev +Wrote config file fly.toml +Validating /home/vorotamoroz/dev/obsidian-livesync/utils/flyio/fly.toml +Platform: machines +✓ Configuration is valid +Your app is ready! Deploy with `flyctl deploy` +Secrets are staged for the first deployment +==> Verifying app config +Validating /home/vorotamoroz/dev/obsidian-livesync/utils/flyio/fly.toml +Platform: machines +✓ Configuration is valid +--> Verified app config +==> Building image +Searching for image 'couchdb:latest' remotely... +image found: img_ox20prk63084j1zq + +Watch your deployment at https://fly.io/apps/young-darkness-25342/monitoring + +Provisioning ips for young-darkness-25342 + Dedicated ipv6: 2a09:8280:1::37:fde9 + Shared ipv4: 66.241.124.163 + Add a dedicated ipv4 with: fly ips allocate-v4 + +Creating a 1 GB volume named 'couchdata' for process group 'app'. Use 'fly vol extend' to increase its size +This deployment will: + * create 1 "app" machine + +No machines in group app, launching a new machine + +WARNING The app is not listening on the expected address and will not be reachable by fly-proxy. +You can fix this by configuring your app to listen on the following addresses: + - 0.0.0.0:5984 +Found these processes inside the machine with open listening sockets: + PROCESS | ADDRESSES +-----------------*--------------------------------------- + /.fly/hallpass | [fdaa:0:73b9:a7b:22e:3851:7f28:2]:22 + +Finished launching new machines + +NOTE: The machines for [app] have services with 'auto_stop_machines = true' that will be stopped when idling + +------- +Checking DNS configuration for young-darkness-25342.fly.dev + +Visit your newly deployed app at https://young-darkness-25342.fly.dev/ +-- Configuring CouchDB by REST APIs... --> +curl: (35) OpenSSL SSL_connect: Connection reset by peer in connection to young-darkness-25342.fly.dev:443 +{"ok":true} +"" +"" +"" +"" +"" +"" +"" +"" +"" +<-- Configuring CouchDB by REST APIs Done! +OK! +Setup finished! Also, we can set up Self-hosted LiveSync instantly, by the following setup uri. +Passphrase of setup-uri is `welcome`. +--- configured --- +database : obsidiannotes +E2EE passphrase: dark-wildflower-26467 +--- setup uri --- +obsidian://setuplivesync?settings=%5B%22gZkBwjFbLqxbdSIbJymU%2FmTPBPAKUiHVGDRKYiNnKhW0auQeBgJOfvnxexZtMCn8sNiIUTAlxNaMGF2t%2BCEhpJoeCP%2FO%2BrwfN5LaNDQyky1Uf7E%2B64A5UWyjOYvZDOgq4iCKSdBAXp9oO%2BwKh4MQjUZ78vIVvJp8Mo6NWHfm5fkiWoAoddki1xBMvi%2BmmN%2FhZatQGcslVb9oyYWpZocduTl0a5Dv%2FQviGwlYQ%2F4NY0dVDIoOdvaYS%2FX4GhNAnLzyJKMXhPEJHo9FvR%2FEOBuwyfMdftV1SQUZ8YDCuiR3T7fh7Kn1c6OFgaFMpFm%2BWgIJ%2FZpmAyhZFpEcjpd7ty%2BN9kfd9gQsZM4%2BYyU9OwDd2DahVMBWkqoV12QIJ8OlJScHHdcUfMW5ex%2F4UZTWKNEHJsigITXBrtq11qGk3rBfHys8O0vY6sz%2FaYNM3iAOsR1aoZGyvwZm4O6VwtzK8edg0T15TL4O%2B7UajQgtCGxgKNYxb8EMOGeskv7NifYhjCWcveeTYOJzBhnIDyRbYaWbkAXQgHPBxzJRkkG%2FpBPfBBoJarj7wgjMvhLJ9xtL4FbP6sBNlr8jtAUCoq4L7LJcRNF4hlgvjJpL2BpFZMzkRNtUBcsRYR5J%2BM1X2buWi2BHncbSiRRDKEwNOQkc%2FmhMJjbAn%2F8eNKRuIICOLD5OvxD7FZNCJ0R%2BWzgrzcNV%22%2C%22ec7edc900516b4fcedb4c7cc01000000%22%2C%22fceb5fe54f6619ee266ed9a887634e07%22%5D +``` + +All we have to do is copy the setup-URI (`obsidian`://...`) and open it from Self-hosted LiveSync on Obsidian. + +If you did not install Deno, configurations will be printed again, instead of the setup-URI. In this case, we should configure it manually. + +### delete-server.sh + +The pair script of `deploy-server.sh`. We can delete the deployed server by this with fly.toml. + +#### Materials + +- Mandatory: flyctl, jq +- Recommended: none + +#### Usage +```sh +./delete-server.sh +``` + +``` +App 'young-darkness-25342 is going to be scaled according to this plan: + -1 machines for group 'app' on region 'nrt' of size 'shared-cpu-1x' +Executing scale plan + Destroyed e28667eec57158 group:app region:nrt size:shared-cpu-1x +Destroyed app young-darkness-25342 +``` \ No newline at end of file