Ci: Add workflows

.github/workflows/backend_check.yml

@@ -0,0 +1,43 @@
+name: check backend
+
+on: [push, pull_request]
+
+jobs:
+ build_backend_and_check_codestyle:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: set up python 3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --no-interaction
+ working-directory: backend/df_designer
+
+ - name: build wheel
+ run: |
+ python -m poetry build
+ working-directory: backend/df_designer
+
+ - name: run black codestyle
+ run: |
+ python -m poetry run black --line-length=120 --check .
+ working-directory: backend/df_designer
+
+ - name: run flake8 codestyle
+ run: |
+ ls
+ bash ./bin/run_codestyle.sh DIFF_BRANCH=${BASE_BRANCH}
+ env:
+ BASE_BRANCH: ${{github.base_ref}}
+
+ - name: run isort codestyle
+ run: |
+ python -m poetry run isort --line-length=120 --diff .
+ working-directory: backend/df_designer

.github/workflows/codestyle.yml

@@ -0,0 +1,32 @@
+name: codestyle
+
+on: [push, pull_request]
+
+jobs:
+ check_backend_codestyle:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: set up python 3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --quiet --no-interaction
+ working-directory: backend/df_designer
+
+ - name: run black codestyle
+ run: |
+ python -m poetry run black --line-length=120 --check .
+ working-directory: backend/df_designer
+
+ - name: run flake8 codestyle
+ run: |
+ bash ./bin/run_codestyle.sh DIFF_BRANCH=${BASE_BRANCH}
+ env:
+ BASE_BRANCH: ${{github.base_ref}}

.github/workflows/docker_check.yml

@@ -0,0 +1,29 @@
+name: Build Docker Images
+
+on:
+ push:
+ branches:
+ - dev
+ pull_request:
+ branches:
+ - dev
+
+jobs:
+ build_images:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --no-ansi --no-interaction
+ working-directory: backend/df_designer
+
+ - name: Create new project
+ run: python -m poetry run dflowd init --destination ../../ --no-input --overwrite-if-exists
+ working-directory: backend/df_designer
+
+ - name: Build Frontend and Backend Images
+ run: docker build -f Dockerfile --build-arg PROJECT_DIR=df_designer_project --target=runtime .

.github/workflows/frontend_check.yml

@@ -0,0 +1,20 @@
+name: front check
+
+on: [push, pull_request]
+
+jobs:
+ build_and_check_frontend:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install bun
+ run: npm install -g bun
+
+ - name: Install dependencies
+ run: bun install
+ working-directory: frontend
+
+ - name: Build frontend
+ run: bun run build
+ working-directory: frontend

Dockerfile

@@ -1,25 +1,24 @@
 FROM oven/bun:1 as frontend-base
 FROM frontend-base AS frontend-builder
 
-WORKDIR /src
-COPY ./frontend/package.json /src/frontend/package.json
-COPY ./frontend/bun.lockb /src/frontend/bun.lockb
+WORKDIR /temp
+COPY ./frontend/package.json /temp/frontend/package.json
+COPY ./frontend/bun.lockb /temp/frontend/bun.lockb
 
-RUN cd /src/frontend && bun install --frozen-lockfile
+RUN cd /temp/frontend && bun install --frozen-lockfile
 
 # Copy the rest of the application code
-COPY ./frontend/ /src/frontend/
-WORKDIR /src/frontend/
+COPY ./frontend/ /temp/frontend/
+WORKDIR /temp/frontend/
 
 RUN bun run build
-RUN ls /src/frontend/dist
 
 #---------------------------------------------------------
 
 # Use a slim variant to reduce image size where possible
 FROM python:3.10-slim as backend-builder
 
-WORKDIR /src
+WORKDIR /temp
 
 ARG PROJECT_DIR
 # ENV PROJECT_DIR ${PROJECT_DIR}
@@ -35,13 +34,13 @@ RUN python3 -m venv $POETRY_VENV \
 
 ENV PATH="${PATH}:${POETRY_VENV}/bin"
 
-COPY ./backend/df_designer /src/backend/df_designer
-COPY --from=frontend-builder /src/frontend/dist /src/backend/df_designer/app/static
+COPY ./backend/df_designer /temp/backend/df_designer
+COPY --from=frontend-builder /temp/frontend/dist /temp/backend/df_designer/app/static
 
-COPY ./${PROJECT_DIR} /src/${PROJECT_DIR}
+COPY ./${PROJECT_DIR} /temp/${PROJECT_DIR}
 
 # Build the wheel
-WORKDIR /src/backend/df_designer
+WORKDIR /temp/backend/df_designer
 RUN poetry build
 
 #---------------------------------------------------------
@@ -58,11 +57,11 @@ COPY --from=backend-builder /poetry-venv /poetry-venv
 ENV PATH="/poetry-venv/bin:$PATH"
 
 # Copy only the necessary files
-COPY --from=backend-builder /src/backend/df_designer /df_designer
-COPY ./${PROJECT_DIR} /${PROJECT_DIR}
+COPY --from=backend-builder /temp/backend/df_designer /src2/backend/df_designer
+COPY ./${PROJECT_DIR} /src2/${PROJECT_DIR}
 
 # Install the wheel
-WORKDIR /${PROJECT_DIR}
+WORKDIR /src2/${PROJECT_DIR}
 RUN poetry lock --no-update \
  && poetry install
 

backend/df_designer/app/api/api_v1/endpoints/bot.py

@@ -1,31 +1,32 @@
 import asyncio
-from fastapi import APIRouter, HTTPException, Depends, WebSocket, WebSocketException, status, BackgroundTasks
 from typing import Optional, Union
 
-from app.schemas.preset import Preset
-from app.schemas.pagination import Pagination
-from app.core.logger_config import get_logger
-from app.services.process_manager import ProcessManager, BuildManager, RunManager
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, WebSocket, WebSocketException, status
+
 from app.api import deps
+from app.core.logger_config import get_logger
+from app.schemas.pagination import Pagination
+from app.schemas.preset import Preset
+from app.services.process_manager import BuildManager, ProcessManager, RunManager
 from app.services.websocket_manager import WebSocketManager
 
-
 router = APIRouter()
 
 logger = get_logger(__name__)
 
 
-async def _stop_process(
- id_: int, process_manager: ProcessManager, process= "run"
-):
+async def _stop_process(id_: int, process_manager: ProcessManager, process="run"):
  if id_ not in process_manager.processes:
  raise HTTPException(
  status_code=status.HTTP_404_NOT_FOUND, detail="Process not found. It may have already exited."
  )
  try:
  await process_manager.stop(id_)
  except (RuntimeError, ProcessLookupError) as e:
- raise HTTPException(status_code=404, detail="Process not found. It may have already exited or not started yet. Please check logs.") from e
+ raise HTTPException(
+ status_code=404,
+ detail="Process not found. It may have already exited or not started yet. Please check logs.",
+ ) from e
 
  logger.info("%s process '%s' has stopped", process.capitalize(), id_)
  return {"status": "ok"}
@@ -34,14 +35,17 @@ async def _stop_process(
 def _check_process_status(id_: int, process_manager: ProcessManager) -> dict[str, str]:
  if id_ not in process_manager.processes:
  raise HTTPException(
- status_code=status.HTTP_404_NOT_FOUND, detail="Process not found. It may have already exited.",
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail="Process not found. It may have already exited.",
  )
  process_status = process_manager.get_status(id_)
  return {"status": process_status}
 
 
 @router.post("/build/start", status_code=201)
-async def start_build(preset: Preset, background_tasks: BackgroundTasks, build_manager: BuildManager = Depends(deps.get_build_manager)):
+async def start_build(
+ preset: Preset, background_tasks: BackgroundTasks, build_manager: BuildManager = Depends(deps.get_build_manager)
+):
  await asyncio.sleep(preset.wait_time)
  await build_manager.start(preset)
  build_id = build_manager.get_last_id()
@@ -64,25 +68,30 @@ async def check_build_status(*, build_id: int, build_manager: BuildManager = Dep
 async def check_build_processes(
  build_id: Optional[int] = None,
  build_manager: BuildManager = Depends(deps.get_build_manager),
- pagination: Pagination = Depends()
+ pagination: Pagination = Depends(),
 ):
  if build_id is not None:
  return await build_manager.get_build_info(build_id)
  else:
  return await build_manager.get_full_info(offset=pagination.offset(), limit=pagination.limit)
 
+
 @router.get("/builds/logs/{build_id}", response_model=Optional[list], status_code=200)
 async def get_build_logs(
- build_id: int,
- build_manager: BuildManager = Depends(deps.get_build_manager),
- pagination: Pagination = Depends()
+ build_id: int, build_manager: BuildManager = Depends(deps.get_build_manager), pagination: Pagination = Depends()
 ):
  if build_id is not None:
  return await build_manager.fetch_build_logs(build_id, pagination.offset(), pagination.limit)
 
 
 @router.post("/run/start/{build_id}", status_code=201)
-async def start_run(*, build_id: int, preset: Preset, background_tasks: BackgroundTasks, run_manager: RunManager = Depends(deps.get_run_manager)):
+async def start_run(
+ *,
+ build_id: int,
+ preset: Preset,
+ background_tasks: BackgroundTasks,
+ run_manager: RunManager = Depends(deps.get_run_manager)
+):
  await asyncio.sleep(preset.wait_time)
  await run_manager.start(build_id, preset)
  run_id = run_manager.get_last_id()
@@ -105,7 +114,7 @@ async def check_run_status(*, run_id: int, run_manager: RunManager = Depends(dep
 async def check_run_processes(
  run_id: Optional[int] = None,
  run_manager: RunManager = Depends(deps.get_run_manager),
- pagination: Pagination = Depends()
+ pagination: Pagination = Depends(),
 ):
  if run_id is not None:
  return await run_manager.get_run_info(run_id)
@@ -115,9 +124,7 @@ async def check_run_processes(
 
 @router.get("/runs/logs/{run_id}", response_model=Optional[list], status_code=200)
 async def get_run_logs(
- run_id: int,
- run_manager: RunManager = Depends(deps.get_run_manager),
- pagination: Pagination = Depends()
+ run_id: int, run_manager: RunManager = Depends(deps.get_run_manager), pagination: Pagination = Depends()
 ):
  if run_id is not None:
  return await run_manager.fetch_run_logs(run_id, pagination.offset(), pagination.limit)
@@ -147,12 +154,15 @@ async def connect(
  logger.error("process with run_id '%s' exited or never existed", run_id)
  raise WebSocketException(code=status.WS_1014_BAD_GATEWAY)
 
-
  await websocket_manager.connect(websocket)
  logger.info("Websocket for run process '%s' has been opened", run_id)
 
- output_task = asyncio.create_task(websocket_manager.send_process_output_to_websocket(run_id, run_manager, websocket))
- input_task = asyncio.create_task(websocket_manager.forward_websocket_messages_to_process(run_id, run_manager, websocket))
+ output_task = asyncio.create_task(
+ websocket_manager.send_process_output_to_websocket(run_id, run_manager, websocket)
+ )
+ input_task = asyncio.create_task(
+ websocket_manager.forward_websocket_messages_to_process(run_id, run_manager, websocket)
+ )
 
  # Wait for either task to finish
  _, websocket_manager.pending_tasks[websocket] = await asyncio.wait(

backend/df_designer/app/api/api_v1/endpoints/flows.py

@@ -1,9 +1,9 @@
 from fastapi import APIRouter
 from omegaconf import OmegaConf
 
-from app.core.logger_config import get_logger
 from app.core.config import settings
-from app.db.base import write_conf, read_conf
+from app.core.logger_config import get_logger
+from app.db.base import read_conf, write_conf
 
 router = APIRouter()
 

backend/df_designer/app/api/deps.py

@@ -2,13 +2,21 @@
 from app.services.websocket_manager import WebSocketManager
 
 build_manager = BuildManager()
+
+
 def get_build_manager() -> BuildManager:
  return build_manager
 
+
 run_manager = RunManager()
+
+
 def get_run_manager() -> RunManager:
  return run_manager
 
+
 websocket_manager = WebSocketManager()
+
+
 def get_websocket_manager() -> WebSocketManager:
  return websocket_manager