Ci: Add workflows

.github/workflows/backend_check.yml

@@ -0,0 +1,43 @@
+name: check backend
+
+on: [push, pull_request]
+
+jobs:
+ build_backend_and_check_codestyle:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: set up python 3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --no-interaction
+ working-directory: backend/df_designer
+
+ - name: build wheel
+ run: |
+ python -m poetry build
+ working-directory: backend/df_designer
+
+ - name: run black codestyle
+ run: |
+ python -m poetry run black --line-length=120 --check .
+ working-directory: backend/df_designer
+
+ - name: run flake8 codestyle
+ run: |
+ ls
+ bash ./bin/run_codestyle.sh DIFF_BRANCH=${BASE_BRANCH}
+ env:
+ BASE_BRANCH: ${{github.base_ref}}
+
+ - name: run isort codestyle
+ run: |
+ python -m poetry run isort --line-length=120 --diff .
+ working-directory: backend/df_designer

.github/workflows/codestyle.yml

@@ -0,0 +1,32 @@
+name: codestyle
+
+on: [push, pull_request]
+
+jobs:
+ check_backend_codestyle:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: set up python 3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --quiet --no-interaction
+ working-directory: backend/df_designer
+
+ - name: run black codestyle
+ run: |
+ python -m poetry run black --line-length=120 --check .
+ working-directory: backend/df_designer
+
+ - name: run flake8 codestyle
+ run: |
+ bash ./bin/run_codestyle.sh DIFF_BRANCH=${BASE_BRANCH}
+ env:
+ BASE_BRANCH: ${{github.base_ref}}

.github/workflows/docker_check.yml

@@ -0,0 +1,29 @@
+name: Build Docker Images
+
+on:
+ push:
+ branches:
+ - dev
+ pull_request:
+ branches:
+ - dev
+
+jobs:
+ build_images:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: setup poetry and install dependencies
+ run: |
+ python -m pip install --upgrade pip poetry
+ python -m poetry lock --no-update
+ python -m poetry install --with lint --no-ansi --no-interaction
+ working-directory: backend/df_designer
+
+ - name: Create new project
+ run: python -m poetry run dflowd init --destination ../../ --no-input --overwrite-if-exists
+ working-directory: backend/df_designer
+
+ - name: Build Frontend and Backend Images
+ run: docker build -f Dockerfile --build-arg PROJECT_DIR=df_designer_project --target=runtime .

.github/workflows/frontend_check.yml

@@ -0,0 +1,20 @@
+name: front check
+
+on: [push, pull_request]
+
+jobs:
+ build_and_check_frontend:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install bun
+ run: npm install -g bun
+
+ - name: Install dependencies
+ run: bun install
+ working-directory: frontend
+
+ - name: Build frontend
+ run: bun run build
+ working-directory: frontend

Dockerfile

@@ -1,25 +1,24 @@
 FROM oven/bun:1 as frontend-base
 FROM frontend-base AS frontend-builder
 
-WORKDIR /src
-COPY ./frontend/package.json /src/frontend/package.json
-COPY ./frontend/bun.lockb /src/frontend/bun.lockb
+WORKDIR /temp
+COPY ./frontend/package.json /temp/frontend/package.json
+COPY ./frontend/bun.lockb /temp/frontend/bun.lockb
 
-RUN cd /src/frontend && bun install --frozen-lockfile
+RUN cd /temp/frontend && bun install --frozen-lockfile
 
 # Copy the rest of the application code
-COPY ./frontend/ /src/frontend/
-WORKDIR /src/frontend/
+COPY ./frontend/ /temp/frontend/
+WORKDIR /temp/frontend/
 
 RUN bun run build
-RUN ls /src/frontend/dist
 
 #---------------------------------------------------------
 
 # Use a slim variant to reduce image size where possible
 FROM python:3.10-slim as backend-builder
 
-WORKDIR /src
+WORKDIR /temp
 
 ARG PROJECT_DIR
 # ENV PROJECT_DIR ${PROJECT_DIR}
@@ -35,13 +34,13 @@ RUN python3 -m venv $POETRY_VENV \
 
 ENV PATH="${PATH}:${POETRY_VENV}/bin"
 
-COPY ./backend/df_designer /src/backend/df_designer
-COPY --from=frontend-builder /src/frontend/dist /src/backend/df_designer/app/static
+COPY ./backend/df_designer /temp/backend/df_designer
+COPY --from=frontend-builder /temp/frontend/dist /temp/backend/df_designer/app/static
 
-COPY ./${PROJECT_DIR} /src/${PROJECT_DIR}
+COPY ./${PROJECT_DIR} /temp/${PROJECT_DIR}
 
 # Build the wheel
-WORKDIR /src/backend/df_designer
+WORKDIR /temp/backend/df_designer
 RUN poetry build
 
 #---------------------------------------------------------
@@ -58,11 +57,11 @@ COPY --from=backend-builder /poetry-venv /poetry-venv
 ENV PATH="/poetry-venv/bin:$PATH"
 
 # Copy only the necessary files
-COPY --from=backend-builder /src/backend/df_designer /df_designer
-COPY ./${PROJECT_DIR} /${PROJECT_DIR}
+COPY --from=backend-builder /temp/backend/df_designer /src2/backend/df_designer
+COPY ./${PROJECT_DIR} /src2/${PROJECT_DIR}
 
 # Install the wheel
-WORKDIR /${PROJECT_DIR}
+WORKDIR /src2/${PROJECT_DIR}
 RUN poetry lock --no-update \
  && poetry install
 

backend/df_designer/app/cli.py

@@ -101,10 +101,14 @@ def run_backend(
 
 
 @cli.command("init")
-def init(destination: str = settings.work_directory):
+def init(destination: str = settings.work_directory, no_input: bool = False, overwrite_if_exists: bool = True):
  original_dir = os.getcwd()
  try:
  os.chdir(destination)
- cookiecutter("https:/Ramimashkouk/df_d_template.git")
+ cookiecutter(
+ "https:/Ramimashkouk/df_d_template.git",
+ no_input=no_input,
+ overwrite_if_exists=overwrite_if_exists,
+ )
  finally:
  os.chdir(original_dir)

backend/df_designer/app/core/config.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
 import uvicorn
-from omegaconf import OmegaConf
 from pydantic_settings import BaseSettings
 
 

backend/df_designer/app/services/process.py

@@ -5,7 +5,6 @@
 from typing import List
 
 import aiofiles
-from omegaconf import OmegaConf
 
 from app.core.config import settings
 from app.core.logger_config import get_logger, setup_logging

backend/df_designer/app/services/process_manager.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, Optional, Type
+from typing import List
 
 from omegaconf import OmegaConf
 
@@ -15,12 +15,15 @@
 class ProcessManager:
  def __init__(self):
  self.processes = {}
+ self.last_id: int
 
  def get_last_id(self):
- """Get the process_id of the last started process"""
- return list(self.processes.keys())[-1]
+ return self.last_id
 
  async def stop(self, id_):
+ if id_ not in self.processes:
+ logger.error("Process with id '%s' not found in recent running processes", id_)
+ raise ProcessLookupError
  try:
  await self.processes[id_].stop()
  except (RuntimeError, ProcessLookupError) as exc:
@@ -67,9 +70,6 @@ class RunManager(ProcessManager):
  def __init__(self):
  super().__init__()
 
- def get_last_id(self):
- return self.last_id
-
  async def start(self, build_id: int, preset: Preset):
  cmd_to_run = f"dflowd run_bot {build_id} --preset {preset.end_status}"
  self.last_id = max([run["id"] for run in await self.get_full_info(0, 10000)])
@@ -79,16 +79,6 @@ async def start(self, build_id: int, preset: Preset):
  await process.start(cmd_to_run)
  self.processes[id_] = process
 
- async def get_min_info(self) -> List[dict]:
- runs_conf = await read_conf(settings.runs_path)
- minimum_params = ["id", "build_id", "preset_end_status", "status", "timestamp"]
-
- minimum_info = []
- for run in runs_conf:
- minimum_info.append({param: getattr(run, param) for param in minimum_params})
-
- return minimum_info
-
  async def get_run_info(self, id_: int):
  return await super().get_process_info(id_, settings.runs_path)
 
@@ -103,9 +93,6 @@ class BuildManager(ProcessManager):
  def __init__(self):
  super().__init__()
 
- def get_last_id(self):
- return self.last_id
-
  async def start(self, preset: Preset):
  cmd_to_run = f"dflowd build_bot --preset {preset.end_status}"
  self.last_id = max([build["id"] for build in await self.get_full_info(0, 10000)])
@@ -116,21 +103,6 @@ async def start(self, preset: Preset):
  await process.start(cmd_to_run)
  self.processes[id_] = process
 
- async def get_min_info(self) -> List[dict]:
- builds_conf = await read_conf(settings.builds_path)
- minimum_params = ["id", "preset_end_status", "status", "timestamp", "runs"]
-
- minimum_info = []
- for build in builds_conf:
- info = {}
- for param in minimum_params:
- if param != "runs":
- info.update({param: getattr(build, param)})
- else:
- info.update({"run_ids": [run.id for run in build.runs]})
- minimum_info.append(info)
- return minimum_info
-
  async def get_build_info(self, id_: int):
  return await super().get_process_info(id_, settings.builds_path)
 

backend/df_designer/app/services/websocket_manager.py

@@ -1,6 +1,6 @@
 import asyncio
 from asyncio.tasks import Task
-from typing import Dict, Optional, Set
+from typing import Dict, Set
 
 from fastapi import WebSocket, WebSocketDisconnect
 
@@ -29,7 +29,7 @@ def disconnect(self, websocket: WebSocket):
 
  def check_status(self, websocket: WebSocket):
  if websocket in self.active_connections:
- return websocket ## return Status!
+ return websocket # return Status!
 
  async def send_process_output_to_websocket(
  self, run_id: int, process_manager: ProcessManager, websocket: WebSocket