diff --git a/.dockerignore b/.dockerignore index 32a91a7a..cd0a8a31 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,4 @@ /.cache /public/build /build +/dev-secrets diff --git a/.env.example b/.env.example index 89de0013..25aca83a 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,12 @@ # Connect to MySQL Docker container locally -DATABASE_URL="mysql://starchart:starchart_password@localhost:3306/starchart" +# NOTE: this value is also mirrored in dev-secrets/DATABASE_URL. +# We set it here for Prisma scripts to work, which expect it +# in the env, but at runtime we read it as a secret in docker swarm. +DATABASE_URL="mysql://starchart:starchart_password@127.0.0.1:3306/starchart" # Connect to Redis container locally REDIS_URL=redis://localhost:6379 -# Used to compute hash against the session IDs. Should be long, random. -SESSION_SECRET="super-duper-s3cret" - # https://letsencrypt.org/docs/expiration-emails/ LETS_ENCRYPT_ACCOUNT_EMAIL="nx@senecacollege.ca" diff --git a/.eslintignore b/.eslintignore index b6ded6ed..3d13a6f3 100644 --- a/.eslintignore +++ b/.eslintignore @@ -4,3 +4,4 @@ /mysql-data /redis-data /img +/dev-secrets diff --git a/.prettierignore b/.prettierignore index c6859554..bcbdaf14 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,4 +7,5 @@ node_modules /mysql-data /img /playwright-report/ +/dev-secrets diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54971774..70cb22dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,8 @@ - [Development](#development) - [System prerequisites](#system-prerequisites) - [Dev environment set up](#dev-environment-set-up) + - [`.env` and `./dev-secrets/*`](#env-and-dev-secrets) + - [Usage](#usage) - [Pull requests](#pull-requests) - [Merging to main](#merging-to-main) - [Stages: Draft and Ready for review](#stages-draft-and-ready-for-review) @@ -56,6 +58,27 @@ $ npm run db:studio > **Note** `npm run build` needs to be executed the first time running the project. As it generates a `build/server.js` script that `npm run dev` depends on. Subsequent times, only `npm run dev` is needed to run the app in development mode. +## `.env` and `./dev-secrets/*` + +Some application configuration is managed via environment variables, others as secrets (i.e., files). + +Your `.env` file should define any environment variables you want to use via `process.env.*`. For example, the line `MY_ENV_VAR=data` in `.env` will mean that `process.env.MY_ENV_VAR` is available at runtime with `data` as its value. + +For secrets, we use the [docker-secret](https://github.com/hwkd/docker-secret) module to load and expose secrets via Docker Swarm's [secrets](https://docs.docker.com/engine/swarm/secrets/), which are files that get mounted into the container at `/run/secrets/*`. + +In development, if you want to override the Docker secrets used by the app, +set the following in your env: `SECRETS_OVERRIDE=1` (we do this automatically +in many development/testing scripts in `package.json`). This will +load secrets from `./dev-secrets/*` instead of using Docker Swarm secrets. The `./dev-secrets/*` folder contains files we want to expose as secrets to the running app. + +### Usage + +If you need to add a secret, for example, a secret named `MY_SECRET` with a value of `this-is-secret`: + +1. Create a new file `dev-secrets/MY_SECRET` with contents `this-is-secret` +2. In your code, `import secrets from '~/lib/secrets.server'` +3. Use your secret, `secrets.MY_SECRET` + ## Pull requests - To avoid duplicate work, create a draft pull request. diff --git a/app/lib/dns.server.ts b/app/lib/dns.server.ts index 33c1cfda..6dbba87f 100644 --- a/app/lib/dns.server.ts +++ b/app/lib/dns.server.ts @@ -1,4 +1,3 @@ -import logger from './logger.server'; import { Route53Client, CreateHostedZoneCommand, @@ -8,6 +7,9 @@ import { import isFQDN from 'validator/lib/isFQDN'; import isIP from 'validator/lib/isIP'; +import logger from '~/lib/logger.server'; +import secrets from '~/lib/secrets.server'; + import type { CreateHostedZoneResponse, ChangeResourceRecordSetsResponse, @@ -21,14 +23,34 @@ if (process.env.NODE_ENV === 'production') { } } +const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = secrets; + +const credentials = () => { + if (process.env.NODE_ENV === 'production') { + if (!AWS_ACCESS_KEY_ID) { + throw new Error('Missing AWS_ACCESS_KEY_ID secret'); + } + if (!AWS_SECRET_ACCESS_KEY) { + throw new Error('Missing AWS_SECRET_ACCESS_KEY secret'); + } + + return { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY, + }; + } + + return { + accessKeyId: process.env.AWS_ACCESS_KEY_ID || AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || AWS_SECRET_ACCESS_KEY, + sessionToken: process.env.AWS_SESSION_TOKEN, + }; +}; + export const route53Client = new Route53Client({ endpoint: process.env.AWS_ENDPOINT_URL || 'http://localhost:5053', region: process.env.AWS_REGION || 'us-east-1', - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', - sessionToken: process.env.AWS_SESSION_TOKEN, - }, + credentials: credentials(), }); export const createHostedZone = async (domain: string) => { diff --git a/app/lib/lets-encrypt.server.ts b/app/lib/lets-encrypt.server.ts index 35f331ff..1b7ede2b 100644 --- a/app/lib/lets-encrypt.server.ts +++ b/app/lib/lets-encrypt.server.ts @@ -1,7 +1,10 @@ import https from 'https'; import dnsPromises from 'dns/promises'; -import type { Resolver as PromisesResolver } from 'dns/promises'; import acme from 'acme-client'; + +import secrets from '~/lib/secrets.server'; + +import type { Resolver as PromisesResolver } from 'dns/promises'; import type { Client as AcmeClient, Order as AcmeOrder, @@ -11,7 +14,8 @@ import type { Account as AcmeAccount, DnsChallenge as AcmeDnsChallenge, } from 'acme-client/types/rfc8555'; -import { secrets } from 'docker-secret'; + +const { LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM } = secrets; /** * Get the zone domain (i.e., drop subdomains that are not in the root level of the zone) @@ -74,8 +78,6 @@ interface ChallengeBundle extends AcmeDnsChallenge { value: String; } -const { LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM } = secrets ?? {}; - /** * This code is based on the official example * https://github.com/publishlab/node-acme-client/blob/master/examples/api.js @@ -173,21 +175,19 @@ class LetsEncrypt { }; initialize = async () => { - if (process.env.NODE_ENV === 'production') { - if (!LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM) { - throw new Error('The docker secret LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM is missing'); - } + if (!LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM) { + throw new Error('The docker secret LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM is missing'); + } + this.#accountKey = LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM; + + if (process.env.NODE_ENV === 'production') { if (!process.env.LETS_ENCRYPT_ACCOUNT_EMAIL) { throw new Error('The env LETS_ENCRYPT_ACCOUNT_EMAIL is missing'); } - this.#accountKey = LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM; this.#directoryUrl = acme.directory.letsencrypt.production; } else { - // For testing and local development, let's use an ad-hoc generated key - - this.#accountKey = (await acme.crypto.createPrivateKey()).toString(); this.#directoryUrl = 'https://127.0.0.1:14000/dir'; /** diff --git a/app/lib/notifications.server.ts b/app/lib/notifications.server.ts index c5e420ed..cf0dcd26 100644 --- a/app/lib/notifications.server.ts +++ b/app/lib/notifications.server.ts @@ -1,9 +1,10 @@ import { createTransport } from 'nodemailer'; -import { secrets } from 'docker-secret'; + +import secrets from '~/lib/secrets.server'; import logger from './logger.server'; const { NOTIFICATIONS_EMAIL_USER, NODE_ENV, MAILHOG_SMTP_PORT } = process.env; -const { NOTIFICATIONS_USERNAME, NOTIFICATIONS_PASSWORD } = secrets ?? {}; +const { NOTIFICATIONS_USERNAME, NOTIFICATIONS_PASSWORD } = secrets; const initializeTransport = () => { if (!NOTIFICATIONS_EMAIL_USER) { diff --git a/app/lib/secrets.server.ts b/app/lib/secrets.server.ts new file mode 100644 index 00000000..a6dacc5f --- /dev/null +++ b/app/lib/secrets.server.ts @@ -0,0 +1,9 @@ +import { resolve } from 'path'; +import { getSecrets } from 'docker-secret'; + +// To override the secrets we use, set SECRETS_OVERRIDE=1 in the env +const { SECRETS_OVERRIDE } = process.env; +const secretsDir = SECRETS_OVERRIDE && resolve(process.cwd(), './dev-secrets'); +const secrets = getSecrets(secretsDir); + +export default secrets; diff --git a/app/session.server.ts b/app/session.server.ts index a4111c90..e10d8151 100644 --- a/app/session.server.ts +++ b/app/session.server.ts @@ -2,9 +2,10 @@ import { createCookieSessionStorage, redirect } from '@remix-run/node'; import type { User } from '~/models/user.server'; import { getUserByUsername } from '~/models/user.server'; +import secrets from '~/lib/secrets.server'; -if (typeof process.env.SESSION_SECRET !== 'string') { - throw new Error('SESSION_SECRET env var must be set'); +if (!secrets.SESSION_SECRET?.length) { + throw new Error('SESSION_SECRET must be set'); } export const sessionStorage = createCookieSessionStorage({ @@ -13,7 +14,7 @@ export const sessionStorage = createCookieSessionStorage({ httpOnly: true, path: '/', sameSite: 'lax', - secrets: [process.env.SESSION_SECRET], + secrets: [secrets.SESSION_SECRET], secure: process.env.NODE_ENV === 'production', }, }); diff --git a/dev-secrets/AWS_ACCESS_KEY_ID b/dev-secrets/AWS_ACCESS_KEY_ID new file mode 100644 index 00000000..038d718d --- /dev/null +++ b/dev-secrets/AWS_ACCESS_KEY_ID @@ -0,0 +1 @@ +testing diff --git a/dev-secrets/AWS_SECRET_ACCESS_KEY b/dev-secrets/AWS_SECRET_ACCESS_KEY new file mode 100644 index 00000000..038d718d --- /dev/null +++ b/dev-secrets/AWS_SECRET_ACCESS_KEY @@ -0,0 +1 @@ +testing diff --git a/dev-secrets/DATABASE_URL b/dev-secrets/DATABASE_URL new file mode 100644 index 00000000..024ab772 --- /dev/null +++ b/dev-secrets/DATABASE_URL @@ -0,0 +1 @@ +mysql://starchart:starchart_password@127.0.0.1:3306/starchart diff --git a/dev-secrets/LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM b/dev-secrets/LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM new file mode 100644 index 00000000..63e039e7 --- /dev/null +++ b/dev-secrets/LETS_ENCRYPT_ACCOUNT_PRIVATE_KEY_PEM @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAz1Gi3i5f8ywDNTktilBiSj899kl4X/ZjnI+pTtBz7s95YVwF ++ojDiwkAJHtIQX7XYEHKvJG2pFt4/d72mhUvLXr0hnQLy6cfsPsVgTqV2Um9CSV3 +bW+vpO68akC3rXb4lTrBawBI9s/XdBdsVdTDcyeHEvu7q+ahk0wYqNpW7i3h5oBH +8nrcQ0LMPHMeQ68dv+Kzqct9UaPlVZHSfxhUewjpKJfXHJ8pna3G74AJC2+IMj1b +qgJDvp4s2FTTnuOF1jtAe0ZQG/O3UCLJlFoWvrEWK6irXANjg/2p2CdGzlNtsGus +HqXsWH93BgTJrddzS0zAxQr4c4jbIfLXZjvKVQIDAQABAoIBAEGxrblX2qG0vaN8 +5dhhVnQOjDTh3RoTekcfIthNp57x8ZWPUnmsIsKI7Jmi0yel6NugnXyZc9WrArD3 +mQ8ETXnM73U3ipFS+PDc71iO1vMOsa8XRzvPW0oZOG026r016Nloz+d3JKfI/o3T +6klcbT6tNNkoGbUEFQkW4O2ImmlIYKDfir6gpcR3dVjH/tfJvKtXXgMqK4uLESlK +FZCv/dM/48ZBGUfmFLq0wQLCIqGjrNQHUqHl27EJGO5RNd0hsyKHj8YHCALO2H6M +lEhNH/Jw3iC9lmnmQQHGuxu8L/SEA0F61txYbA+Ci9rHikCdO3/D3ri/VZjjHTx+ +Fve2waECgYEA79EeRgIhebnvr6Zhimovx2U6E/znPIwkI97fAQaJiDmAiwlc/M7L +JaQ91YpaEMnMnFZa/DpOtkneSjz9/wo+yeIxp1O7wVbpW0Ggs4gVi1ZiHNQkQJrn +t42AxcADeSoYuSpGITYEOFlT6NV6I2CGwFAqMC/9YGwF47CwlCzOWy0CgYEA3U8b ++AaSAuzECz0iurhXoCgK5GfR3UT62/ml2g2yxlknu+sTKiwb04sQwWLwn/vV80vA +kDZpHQVAjrPRwWhVGMyeYmoJbeUX1F2ix7KIXbQPzVnJ+k+9DIWV906GJ2ZfH2hT +a6HYWEVBYesfOsT+mSfKxsPlXaUOscYN0RnfhMkCgYAXJ+H8cIg68LEsDKyuaMJk +RmntNCY/umhi7koqFy+Ab8zxn93Sq0UCRXGTBODdbh7Lmar/X8Hp6AgGswzza1HU +vHp+5Z7jdDjkDtote55Y7eZbUkCN3GczWf5tGbU8JcxtVJ+g5U5TAo0Plk1MzS01 +tIfeT0Pv435OFel25TynoQKBgQDDPjm0lUdXoT5LlAIBrQRRXUJOw3EYvvR6AUNa +nl8sMhel5/weZo+eD8AWfI1A91KHtDsMf5Q4cBvGScox0TPSDyNkO5xaAZUGXB1y +BIXd9S9DwYU5egOU+n2Vkwcz11LwSH/gIwbUyTSniGEi3gynXb29obHG7gmKuOoT +obnaUQKBgAnyof4vHHrFSTzzK0d9M7Ouze5qv7+QUcy4Igt2wItFPgT/HYeq77d5 +ztk1dDKLhWCPTHbL/Uvo6DKgw7smnqUq1yly8JeY5a2M7V9xjGkU4a8Y16SukBYk +9y12DbwL03QPe3OkDd6xr8gOYEVVQE2LJ6aK/kIi64ElpSY9cGv3 +-----END RSA PRIVATE KEY----- diff --git a/dev-secrets/NOTIFICATIONS_PASSWORD b/dev-secrets/NOTIFICATIONS_PASSWORD new file mode 100644 index 00000000..53858e8b --- /dev/null +++ b/dev-secrets/NOTIFICATIONS_PASSWORD @@ -0,0 +1 @@ +dev-smtp-password diff --git a/dev-secrets/NOTIFICATIONS_USERNAME b/dev-secrets/NOTIFICATIONS_USERNAME new file mode 100644 index 00000000..27a924b1 --- /dev/null +++ b/dev-secrets/NOTIFICATIONS_USERNAME @@ -0,0 +1 @@ +dev-smtp-username diff --git a/dev-secrets/SESSION_SECRET b/dev-secrets/SESSION_SECRET new file mode 100644 index 00000000..320318f1 --- /dev/null +++ b/dev-secrets/SESSION_SECRET @@ -0,0 +1 @@ +this-is-our-development-session-secret!!!! diff --git a/package.json b/package.json index d1b94b8b..dba64849 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "build": "run-s build:*", "build:remix": "remix build", "build:server": "esbuild --platform=node --format=cjs ./server.ts --outdir=build --bundle", - "dev": "run-p dev:*", + "dev": "SECRETS_OVERRIDE=1 run-p dev:*", "dev:build": "cross-env NODE_ENV=development npm run build:server -- --watch", "dev:remix": "cross-env NODE_ENV=development remix watch", "dev:server": "cross-env NODE_ENV=development node --inspect --require ./node_modules/dotenv/config ./build/server.js", @@ -21,10 +21,10 @@ "setup": "run-s db:generate db:push db:seed", "start": "cross-env NODE_ENV=production node ./build/server.js", "start:e2e": "cross-env NODE_ENV=production node --require dotenv/config ./build/server.js", - "test": "cross-env NODE_OPTIONS=--require=dotenv/config vitest", + "test": "cross-env SECRETS_OVERRIDE=1 NODE_OPTIONS=--require=dotenv/config vitest", "test:e2e:dev": "cross-env PORT=8080 start-server-and-test dev http://localhost:8080 \"playwright test\"", - "pretest:e2e:run": "npm run build", - "test:e2e:run": "cross-env PORT=8811 start-server-and-test start:e2e http://localhost:8811 \"playwright test\"", + "pretest:e2e:run": "cross-env SECRETS_OVERRIDE=1 run-s build", + "test:e2e:run": "cross-env SECRETS_OVERRIDE=1 PORT=8811 start-server-and-test start:e2e http://localhost:8811 \"playwright test\"", "typecheck": "tsc", "validate": "run-p \"test -- --run\" lint typecheck test:e2e:run", "prepare": "husky install"