diff --git a/docs/setup_object_storage.md b/docs/setup_object_storage.md index cad31264..c2a5237b 100644 --- a/docs/setup_object_storage.md +++ b/docs/setup_object_storage.md @@ -24,7 +24,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=my-vault -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/docs/setup_own_server.md b/docs/setup_own_server.md index 1b7d0e9c..6924ec55 100644 --- a/docs/setup_own_server.md +++ b/docs/setup_own_server.md @@ -128,6 +128,9 @@ export database=obsidiannotes curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | bash ``` +> [!TIP] +> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname=http://localhost:5984 username=<...> password=<...> database=obsidiannotes bash`. + If it results like the following: ``` CouchDB provisioning completed. @@ -173,20 +176,20 @@ Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our serv ### 1. Generate the setup URI on a desktop device or server -The `username` and `password` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. +The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. ```bash -export hostname=https://tiles-photograph-routine-groundwater.trycloudflare.com -export database=obsidiannotes -export username= -export password= -export passphrase= +export LIVESYNC_HOST=https://tiles-photograph-routine-groundwater.trycloudflare.com +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_USER= +export LIVESYNC_PASS= +export LIVESYNC_PASSPHRASE= export uri_passphrase= # Optional deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` > [!TIP] -> `passphrase` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. +> `LIVESYNC_PASSPHRASE` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. > > If `uri_passphrase` is omitted, the generator creates a cryptographically random value and prints it once. diff --git a/docs/setup_p2p.md b/docs/setup_p2p.md index 4797f420..c755ce0a 100644 --- a/docs/setup_p2p.md +++ b/docs/setup_p2p.md @@ -130,7 +130,7 @@ export remote_type=p2p export p2p_relays=wss://relay.example.com export p2p_room_id= # Optional; generated when omitted export p2p_passphrase= # Optional; generated when omitted -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/updates.md b/updates.md index 6a92c03f..ed26ab2f 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Setup tools + +#### Fixed + +- We can generate a Setup URI with prefixed connection and encryption variables, avoiding collisions with shell or system variables such as `hostname` and `USERNAME`. (#817, #1274) + - The generator reads `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE`. The previous names remain supported when the corresponding prefixed value is unset or blank. + ### Synchronisation - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) diff --git a/utils/readme.md b/utils/readme.md index 6ee10742..8f0dca6f 100644 --- a/utils/readme.md +++ b/utils/readme.md @@ -31,21 +31,23 @@ Authentication and other non-retryable HTTP failures stop immediately. Network a The existing `flyio/generate_setupuri.ts` path remains a CouchDB-only compatibility wrapper for the Fly.io deployment script. +The connection and Vault encryption values use a `LIVESYNC_` prefix so that they do not collide with shell or system variables such as `hostname` and `USERNAME`. The unprefixed names `hostname`, `username`, `password`, `database`, and `passphrase` are still read when the corresponding prefixed value is unset or blank. + The generator creates a fresh random ID key by default and includes it in the encrypted Setup URI. It prints a tagged `sls-id-v1:` recovery code. Set `id_recovery_code` to that code when generating another URI for the same Vault; a new run without it creates a different key. This restores only the ID key: reuse the original connection details too. For P2P, provide the original `p2p_room_id` and `p2p_passphrase` because omitted values are generated afresh. Set `id_mode=legacy` to generate a URI with the previous ID behaviour. `id_mode=legacy` and `id_recovery_code` cannot be combined. Keep the recovery code private and retain it if every device might be lost. ### CouchDB ```sh -export hostname=https://couch.example.com -export username=couchdb-admin-username -export password=couchdb-admin-password -export database=obsidiannotes -export passphrase=a-strong-vault-encryption-passphrase +export LIVESYNC_HOST=https://couch.example.com +export LIVESYNC_USER=couchdb-admin-username +export LIVESYNC_PASS=couchdb-admin-password +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_PASSPHRASE=a-strong-vault-encryption-passphrase export uri_passphrase=a-separate-setup-uri-passphrase # Optional deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` -If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `passphrase` value protects synchronised Vault data and must also be stored safely. +If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `LIVESYNC_PASSPHRASE` value protects synchronised Vault data and must also be stored safely. The generator defaults to `uri_mode=ephemeral`. The URI opens only during the current fixed seven-day UTC window, and the tool prints its exact end time. It may have less than seven days remaining when generated. For an indefinitely reusable URI, set `uri_mode=persistent` before running the command. Persistent uses the existing encrypted format, which older clients that already support that format can read. The time condition controls opening the URI; it does not revoke settings or credentials after import. @@ -59,7 +61,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=team-a # Optional -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` @@ -69,7 +71,7 @@ Optional Object Storage variables are `use_custom_request_handler`, `force_path_ ```sh export remote_type=p2p -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` diff --git a/utils/setup/generate_setup_uri.test.ts b/utils/setup/generate_setup_uri.test.ts index 99e11e30..0574b49d 100644 --- a/utils/setup/generate_setup_uri.test.ts +++ b/utils/setup/generate_setup_uri.test.ts @@ -1,3 +1,4 @@ +import { fileURLToPath } from "node:url"; import { decodeSettingsFromSetupURI, DEFAULT_SETTINGS, @@ -9,6 +10,76 @@ function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); } +Deno.test("reads LIVESYNC_ variables before the unprefixed names", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: "https://couch.example.test", + LIVESYNC_USER: "alice", + LIVESYNC_PASS: "couch-secret", + LIVESYNC_DB: "notes", + LIVESYNC_PASSPHRASE: "vault-secret", + hostname: "https://shell.example.test", + username: "shell-user", + password: "shell-secret", + database: "shell-db", + passphrase: "shell-passphrase", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes", + "the CouchDB connection did not come from the LIVESYNC_ variables", + ); + assert( + effective.passphrase === "vault-secret", + "the Vault passphrase did not come from LIVESYNC_PASSPHRASE", + ); +}); + +Deno.test("falls back to the unprefixed names when LIVESYNC_ variables are blank", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: " ", + hostname: "https://couch.example.test", + username: "alice", + password: "couch-secret", + database: "notes", + passphrase: "vault-secret", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes" && + effective.passphrase === "vault-secret", + "the generator did not fall back to the unprefixed names", + ); +}); + +Deno.test("names both variables when a CouchDB value is missing", async () => { + let rejected = false; + try { + await generateSetupURI({ uri_passphrase: "setup-secret" }); + } catch (error) { + rejected = error instanceof Error && + error.message === "LIVESYNC_HOST (or hostname) is required"; + } + assert(rejected, "the generator did not name both host variables"); +}); + Deno.test("generates an Object Storage Setup URI with a selected S3 profile", async () => { const generated = await generateSetupURI({ remote_type: "s3", @@ -259,3 +330,171 @@ for (const mode of ["ephemeral", "persistent"] as const) { assert(rejected, "legacy mode silently ignored a recovery code"); }); } + +const couchDBConnection = { + hostname: "https://couch.example.test", + username: "alice", + password: "couch-secret", + database: "notes", + passphrase: "vault-secret", +}; +const prefixedCouchDBConnection = { + LIVESYNC_HOST: couchDBConnection.hostname, + LIVESYNC_USER: couchDBConnection.username, + LIVESYNC_PASS: couchDBConnection.password, + LIVESYNC_DB: couchDBConnection.database, + LIVESYNC_PASSPHRASE: couchDBConnection.passphrase, +}; +const commandEnvironments: { name: string; values: Record }[] = + [ + { name: "prefixed values only", values: prefixedCouchDBConnection }, + { + name: "prefixed values before conflicting legacy values", + values: { + ...prefixedCouchDBConnection, + hostname: "https://shell.example.test", + username: "shell-user", + password: "shell-password", + database: "shell-database", + passphrase: "shell-passphrase", + }, + }, + { + name: "blank prefixed values fall back to legacy values", + values: { + ...couchDBConnection, + LIVESYNC_HOST: " ", + LIVESYNC_USER: "\t", + LIVESYNC_PASS: " ", + LIVESYNC_DB: "\t", + LIVESYNC_PASSPHRASE: " ", + }, + }, + { name: "compatibility: legacy values only", values: couchDBConnection }, + ]; + +async function runGeneratorCommand( + script: URL, + values: Record, +) { + const environment = Deno.env.toObject(); + const inputNames = new Set([ + ...Object.keys(couchDBConnection), + ...Object.keys(prefixedCouchDBConnection), + "remote_type", + "uri_passphrase", + "uri_mode", + "id_mode", + "id_recovery_code", + "endpoint", + "access_key", + "secret_key", + "bucket", + "region", + "bucket_prefix", + "bucket_custom_headers", + "use_custom_request_handler", + "force_path_style", + "p2p_room_id", + "p2p_passphrase", + "p2p_relays", + "p2p_app_id", + "p2p_auto_start", + "p2p_auto_broadcast", + ].map((name) => name.toLowerCase())); + for (const name of Object.keys(environment)) { + if (inputNames.has(name.toLowerCase())) delete environment[name]; + } + const result = await new Deno.Command(Deno.execPath(), { + args: [ + "run", + "--config", + fileURLToPath(new URL("../flyio/deno.jsonc", import.meta.url)), + "--frozen", + "--lock", + fileURLToPath(new URL("../flyio/deno.lock", import.meta.url)), + "--allow-env", + script.href, + ], + clearEnv: true, + env: { + ...environment, + uri_passphrase: "setup-secret", + uri_mode: "persistent", + ...values, + }, + stdout: "piped", + stderr: "piped", + }).output(); + assert( + result.success, + `generator failed: ${new TextDecoder().decode(result.stderr)}`, + ); + const output = new TextDecoder().decode(result.stdout); + const uri = output.match(/obsidian:\/\/setuplivesync\?settings=\S+/u)?.[0]; + assert(uri, "the command did not print a Setup URI"); + const decoded = await decodeSettingsFromSetupURI(uri, "setup-secret"); + assert(decoded, "the command output could not be decoded"); + return { ...DEFAULT_SETTINGS, ...decoded }; +} + +for ( + const [name, path] of [ + ["Setup generator", "./generate_setup_uri.ts"], + ["Fly.io wrapper", "../flyio/generate_setupuri.ts"], + ] +) { + for (const scenario of commandEnvironments) { + Deno.test(`E2E ${name}: ${scenario.name}`, async () => { + const settings = await runGeneratorCommand( + new URL(path, import.meta.url), + scenario.values, + ); + assert( + settings.couchDB_URI === couchDBConnection.hostname && + settings.couchDB_USER === couchDBConnection.username && + settings.couchDB_PASSWORD === couchDBConnection.password && + settings.couchDB_DBNAME === couchDBConnection.database && + settings.passphrase === couchDBConnection.passphrase, + "the command did not preserve the selected connection and Vault passphrase", + ); + const profileId = settings.activeConfigurationId; + const profile = profileId + ? settings.remoteConfigurations?.[profileId] + : undefined; + assert( + profile?.uri.startsWith("sls+https://"), + "the generated URI did not select its CouchDB profile", + ); + }); + } +} + +for (const remoteType of ["s3", "p2p"]) { + Deno.test(`E2E Setup generator: prefixed Vault passphrase for ${remoteType}`, async () => { + const settings = await runGeneratorCommand( + new URL("./generate_setup_uri.ts", import.meta.url), + { + remote_type: remoteType, + endpoint: "https://objects.example.test", + access_key: "access-key", + secret_key: "secret-key", + bucket: "vault-data", + LIVESYNC_PASSPHRASE: "vault-secret", + passphrase: "shell-passphrase", + }, + ); + assert( + settings.passphrase === "vault-secret", + "the command did not prefer LIVESYNC_PASSPHRASE", + ); + const profileId = settings.activeConfigurationId; + const profile = profileId + ? settings.remoteConfigurations?.[profileId] + : undefined; + assert( + profile?.uri.startsWith(`sls+${remoteType}://`), + "the command did not select the requested remote profile", + ); + }); +} diff --git a/utils/setup/generate_setup_uri.ts b/utils/setup/generate_setup_uri.ts index 6bf1ebdd..14c67724 100644 --- a/utils/setup/generate_setup_uri.ts +++ b/utils/setup/generate_setup_uri.ts @@ -64,9 +64,17 @@ function configureIdDerivation( function requireValue( environment: SetupGeneratorEnvironment, name: string, + legacyName?: string, ): string { - const value = environment[name]?.trim(); - if (!value) throw new Error(`${name} is required`); + const value = environment[name]?.trim() || + (legacyName && environment[legacyName]?.trim()); + if (!value) { + throw new Error( + legacyName + ? `${name} (or ${legacyName}) is required` + : `${name} is required`, + ); + } return value; } @@ -97,7 +105,7 @@ function applyEncryptedVaultSettings( Object.assign(settings, { isConfigured: true, encrypt: true, - passphrase: requireValue(environment, "passphrase"), + passphrase: requireValue(environment, "LIVESYNC_PASSPHRASE", "passphrase"), usePathObfuscation: true, }); } @@ -107,10 +115,10 @@ function createCouchDBSettings( ): ObsidianLiveSyncSettings { const settings = createNewVaultSettings(); Object.assign(settings, PREFERRED_SETTING_SELF_HOSTED, { - couchDB_URI: requireValue(environment, "hostname"), - couchDB_USER: requireValue(environment, "username"), - couchDB_PASSWORD: requireValue(environment, "password"), - couchDB_DBNAME: requireValue(environment, "database"), + couchDB_URI: requireValue(environment, "LIVESYNC_HOST", "hostname"), + couchDB_USER: requireValue(environment, "LIVESYNC_USER", "username"), + couchDB_PASSWORD: requireValue(environment, "LIVESYNC_PASS", "password"), + couchDB_DBNAME: requireValue(environment, "LIVESYNC_DB", "database"), batchSave: true, periodicReplication: true, syncOnStart: true,