Refactor docker setup (#1060)

* Use envsubst to allow frontend to find backend with env vars

* Refactors docker compose file setup

This structure more naturally fits what docker expects.
A docker-compose file at the root, with docker files
for the respective services in those modules.

I have tested this locally for the prod build
and pushed things and I believe it works for
the other cases too.

TODO:
 - dockerx build instructions
 - automating the build & versioning of images with new releases

* Refactors docker compose file setup

This structure more naturally fits what docker expects.
A docker-compose file at the root, with docker files
for the respective services in those modules.

I have tested this locally for the prod build
and pushed things and I believe it works for
the other cases too.

TODO:
 - dockerx build instructions
 - automating the build & versioning of images with new releases

---------
Co-authored-by: Fran Boon <[email protected]>
Co-authored-by: Stefan Krawczyk <[email protected]>

ui/README.md

@@ -72,7 +72,7 @@ git clone https:/dagworks-inc/hamilton
 # change into the UI directory
 cd hamilton/ui
 # run docker
-./deployment/run.sh
+./run.sh
 ```
 Once docker is running navigate to http://localhost:8242 and create an email and a project; then follow
 instructions on integrating with Hamilton.

ui/deployment/Dockerfile.backend → ui/backend/Dockerfile.backend

@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \
  libffi-dev \
  && rm -rf /var/lib/apt/lists/*
 
-COPY backend/server/requirements.txt /code/
+COPY server/requirements.txt /code/
 RUN pip install -r /code/requirements.txt
 
-COPY ./backend /code/
+COPY . /code/

ui/deployment/Dockerfile.backend-prod → ui/backend/Dockerfile.backend-prod

@@ -12,9 +12,9 @@ RUN apt-get update && apt-get install -y \
  libffi-dev \
  && rm -rf /var/lib/apt/lists/*
 
-COPY backend/ /code/
-COPY backend/server/requirements.txt /code/
+COPY . /code/
+COPY server/requirements.txt /code/
 
 RUN pip install -r /code/requirements.txt
 
-COPY ./backend /code/
+COPY . /code/

ui/common.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ -z $(which docker-compose) ]];
+then
+ function docker-compose() {
+ docker compose --compatibility $@
+ }
+fi

ui/dev.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. common.sh
+
+# Check if --build parameter is passed
+if [[ $1 == "--build" ]]; then
+ # Run docker-compose up with project directory, verbose mode and build
+ docker-compose --verbose -f docker-compose.yml up --build
+else
+ # Run docker-compose up with project directory and verbose mode
+ docker-compose --verbose -f docker-compose.yml up
+fi

ui/deployment/docker-compose-prod.yml → ui/docker-compose-prod.yml

@@ -17,11 +17,12 @@ services:
  retries: 5
 
  backend:
+ container_name: ui-backend
  image: dagworks/ui-backend:latest
  build:
- context: ./
- dockerfile: deployment/Dockerfile.backend-prod
- entrypoint: ["/bin/bash", "-c", "cd /code/server && ls && ./entrypoint.sh"]
+ context: backend
+ dockerfile: Dockerfile.backend-prod
+ entrypoint: ["/bin/bash", "-c", "cd /code/server && ./entrypoint.sh"]
  ports:
  - "8241:8241"
  environment:
@@ -42,22 +43,23 @@ services:
  - backend_data:/data/
 
  frontend:
+ container_name: ui-frontend
  image: dagworks/ui-frontend:latest
  build:
- context: ./
- dockerfile: deployment/Dockerfile.frontend-prod
+ context: frontend
+ dockerfile: Dockerfile.frontend-prod
  args:
+ - NGINX_PORT=8242
  - REACT_APP_AUTH_MODE=local
  - REACT_APP_USE_POSTHOG=false
  ports:
  - "8242:8242"
  environment:
+ - NGINX_PORT=8242 # NB Custom port won't be visible in console & will mean the backend's shortcut will fail
  - NODE_ENV=development
  - REACT_APP_AUTH_MODE=local
  - REACT_APP_USE_POSTHOG=false
-# TODO -- use envsubst to replace the backend url in the nginx config
-# this is currently hardcoded in the nginx config
-# - REACT_APP_API_URL=http://backend:8000
+ - REACT_APP_API_URL=http://backend:8241
  depends_on:
  - backend
 

ui/deployment/docker-compose.yml → ui/docker-compose.yml

@@ -16,10 +16,11 @@ services:
  retries: 5
 
  backend:
+ container_name: ui-backend
  image: dagworks/ui-backend:latest
  build:
- context: ./
- dockerfile: deployment/Dockerfile.backend
+ context: backend
+ dockerfile: Dockerfile.backend
  entrypoint: ["/bin/bash", "-c", "cd /code/server && ls && ./entrypoint.sh"]
  volumes:
  - ./backend:/code
@@ -42,10 +43,11 @@ services:
  - db
 
  frontend:
+ container_name: ui-frontend
  image: dagworks/ui-frontend:latest
  build:
- context: ./
- dockerfile: deployment/Dockerfile.frontend
+ context: frontend
+ dockerfile: Dockerfile.frontend
  args:
  - REACT_APP_AUTH_MODE=local
  - REACT_APP_USE_POSTHOG=false

ui/deployment/Dockerfile.frontend → ui/frontend/Dockerfile.frontend

@@ -8,13 +8,13 @@ ARG REACT_APP_AUTH_MODE
 ARG REACT_APP_USE_POSTHOG
 
 # Install app dependencies by copying package.json and package-lock.json
-COPY frontend/package.json frontend/package-lock.json ./
+COPY package.json package-lock.json ./
 
 # Install dependencies
 RUN npm install
 
 # Copy the rest of the frontend directory
-COPY frontend/ ./
+COPY ./ ./
 
 # Environment variables
 ENV REACT_APP_AUTH_MODE=${REACT_APP_AUTH_MODE}

ui/deployment/Dockerfile.frontend-prod → ui/frontend/Dockerfile.frontend-prod

@@ -1,4 +1,4 @@
-FROM node:20 as build-stage
+FROM node:20 AS build-stage
 
 # Set working directory
 WORKDIR /usr/src/app
@@ -7,15 +7,16 @@ WORKDIR /usr/src/app
 ARG REACT_APP_AUTH_MODE
 ARG REACT_APP_USE_POSTHOG
 ARG REACT_APP_API_URL
+ARG NGINX_PORT=8242
 
 # Install app dependencies by copying package.json and package-lock.json
-COPY frontend/package.json frontend/package-lock.json ./
+COPY package.json package-lock.json ./
 
 # Install dependencies
 RUN npm install
 
 # Copy the rest of the frontend directory
-COPY frontend .
+COPY . .
 
 # Environment variables
 ENV REACT_APP_AUTH_MODE=${REACT_APP_AUTH_MODE}
@@ -25,18 +26,24 @@ ENV NODE_OPTIONS="--max-old-space-size=8192"
 
 RUN npm run build
 
-FROM nginx:stable-alpine as production-stage
+FROM nginx:stable-alpine AS production-stage
+
+# Build Args to pass through to production-stage
+ARG NGINX_PORT
+
+ENV NGINX_PORT=${NGINX_PORT}
 
 # Copy the build output to replace the default nginx contents.
 COPY --from=build-stage /usr/src/app/build /usr/share/nginx/html
 
-# Expose port 80 to the outside once the container has launched
-EXPOSE 8242
-
-# Use the default nginx.conf provided by tiangolo/nginx-rtmp
-COPY ./deployment/nginx/nginx.conf /etc/nginx/nginx.conf
+# Expose port to the outside once the container has launched
+EXPOSE ${NGINX_PORT}
 
-CMD ["echo", "Frontend running on port 8242, go to http://localhost:8242 to view the app."]
+# Use envsubst to allow the frontend to find the backend from ${REACT_APP_API_URL}
+RUN mkdir /etc/nginx/templates
+COPY nginx/templates/default.conf.template /etc/nginx/templates/default.conf.template
 
 # Start Nginx and keep the process from backgrounding and the container from quitting
+# This prevents envsubst from running! (nginx needs to be the 1st value of the command)
+#CMD ["sh", "-c", "echo \"Frontend running on port ${NGINX_PORT}, go to http://localhost:${NGINX_PORT} to view the app.\"; nginx -g 'daemon off;'"]
 CMD ["nginx", "-g", "daemon off;"]

ui/frontend/README.md

@@ -1,4 +1,4 @@
-# Func2ETL frontend
+# Hamilton UI frontend
 
 ## Dev setup
 
@@ -10,3 +10,55 @@
 
 To generate the code for the typescript backend client, run : `npx @rtk-query/codegen-openapi openapi-config.json` from
 the `frontend` directory, while the backend is running on localhost:8000.
+
+
+## Building docker
+## Dev mode
+For development you'll want to run
+
+```bash
+cd hamilton/ui
+./dev.sh --build # to build it all
+./dev.sh # to pull docker images but use local code
+```
+## You need 9GB assigned to Docker or more to build the frontend
+The frontend build requires around 8GB of memory to be assigned to docker to build.
+If you run into this, bump your docker memory allocation up to 9GB or more.
+
+
+## Prod mode
+For production build you'll want to run
+
+```bash
+cd hamilton/ui
+./run.sh # to pull from docker and run
+./run.sh --build # to rebuild images for prod
+
+```
+### Caveats:
+You'll want to clean the `backend/dist/` directory to not add unnecessary files to the docker image.
+
+
+## Pushing
+How to push to docker hub:
+```bash
+# retag if needed
+docker tag local-image:tagname dagworks/ui-backend:VERSION
+# push built image
+docker push dagworks/ui-backend:VERSION
+# retag as latest
+docker tag dagworks/ui-backend:VERSION dagworks/ui-backend:latest
+# push latest
+docker push dagworks/ui-backend:latest
+```
+
+```bash
+# retag if needed
+docker tag local-image:tagname dagworks/ui-frontend:VERSION
+# push built image
+docker push dagworks/ui-frontend:VERSION
+# retag as latest
+docker tag dagworks/ui-backend:VERSION dagworks/ui-backend:latest
+# push latest
+docker push dagworks/ui-backend:latest
+```

ui/frontend/nginx/templates/default.conf.template

@@ -0,0 +1,29 @@
+server {
+ listen ${NGINX_PORT};
+ server_name localhost;
+
+ root /usr/share/nginx/html; # Common root for all files.
+
+ location / {
+ index index.html index.htm;
+ try_files $uri $uri/ /index.html;
+ }
+
+ location ~* \.(css|js|png|jpg|jpeg|gif|ico)$ {
+ try_files $uri $uri/ =404;
+ }
+
+ # Proxy /api requests to backend
+ location /api {
+ proxy_pass ${REACT_APP_API_URL};
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 5000;
+ }
+}

ui/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. common.sh
+
+# Check if --build parameter is passed
+if [[ $1 == "--build" ]]; then
+ # Run docker-compose up with project directory, verbose mode and build
+ docker-compose --verbose -f docker-compose-prod.yml up --build
+else
+ # Run docker-compose up with project directory and verbose mode
+ docker-compose --verbose -f docker-compose-prod.yml up
+fi

ui/stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+. common.sh
+
+# Run docker-compose up with project directory and verbose mode
+docker-compose --verbose -f docker-compose.yml down