Resolves #3 adding gene expr correlation with str length in crc suppo…

…rt + docker-compose + alembic migrations (optional) (#4)

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https:/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+ "name": "Existing Dockerfile",
+ "build": {
+ // Sets the run context to one level up instead of the .devcontainer folder.
+ "context": "..",
+ // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+ "dockerfile": "../Dockerfile"
+ },
+ "features": {
+ "ghcr.io/devcontainers/features/python:1": {}
+ }
+
+ // Features to add to the dev container. More info: https://containers.dev/features.
+ // "features": {},
+
+ // Use 'forwardPorts' to make a list of ports inside the container available locally.
+ // "forwardPorts": [],
+
+ // Uncomment the next line to run commands after the container is created.
+ // "postCreateCommand": "cat /etc/os-release",
+
+ // Configure tool-specific properties.
+ // "customizations": {},
+
+ // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+ // "remoteUser": "devcontainer"
+}

.dockerignore

@@ -0,0 +1,28 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+**/db/
+LICENSE
+README.md

.env.example

@@ -0,0 +1,7 @@
+# Rename this file to .env and docker compose wil automatically use envornment variables fro it
+DATABASE_URL=postgresql://webstr:webstr@db:5432/webstr
+# DATABASE_URL=sqlite:///db/debug.sqlite
+PYTHONPATH="./" 
+WEBSTR_DATABASE_DATA_UPGRADE=True
+WEBSTR_DEVELOPMENT=False
+WEBSTR_SOURCE_MOUNT_PATH=/temp/src #/usr/src/strs

.gitignore

@@ -4,4 +4,9 @@
 
 strAPI/local_queries.ipynb
 data/repeats
+db/docker_data/
 .venv/
+db/debug.db
+
+*/migrations/versions0/*
+*.env

.vscode/launch.json

@@ -0,0 +1,46 @@
+{
+ "configurations": [
+ {
+ "name": "Python: main",
+ "type": "python",
+ "request": "launch",
+ "program": "strAPI/main.py",
+ "cwd": "${workspaceFolder}",
+ "console": "integratedTerminal",
+ "justMyCode": true,
+ "env": {
+ // "PYTHONPATH": "${workspaceFolder}/strAPI/"
+ }
+ },
+ {
+ "name": "Docker: Python - Fastapi",
+ "type": "docker",
+ "request": "launch",
+ "preLaunchTask": "docker-run: debug",
+ "python": {
+ "pathMappings": [
+ {
+ "localRoot": "${workspaceFolder}",
+ "remoteRoot": "/usr/src/strs"
+ }
+ ],
+ "projectType": "fastapi"
+ },
+ },
+ {
+ "name": "Python: Remote Attach 5000",
+ "type": "python",
+ "request": "attach",
+ "connect": {
+ "host": "0.0.0.0",
+ "port": 5000
+ },
+ "pathMappings": [
+ {
+ "localRoot": "${workspaceFolder}",
+ "remoteRoot": "/usr/src/app"
+ }
+ ]
+ }
+ ]
+}

.vscode/tasks.json

@@ -0,0 +1,43 @@
+{
+ "version": "2.0.0",
+ "tasks": [
+ {
+ "type": "docker-build",
+ "label": "docker-build",
+ "platform": "python",
+ "dockerBuild": {
+ "tag": "webstrapi:latest",
+ "dockerfile": "${workspaceFolder}/Dockerfile",
+ "context": "${workspaceFolder}",
+ "pull": true,
+ }
+ },
+ {
+ "type": "docker-run",
+ "label": "docker-run: debug",
+ "dependsOn": [
+ "docker-build"
+ ],
+ "dockerRun": {
+ "env": {
+ "DATABASE_URL": "sqlite:///db/debug.db",
+ "WEBSTR_DATABAS_UPGRADE": "True"
+ },
+ "mounts": [
+ "source=${workspaceFolder},target=/app,type=bind,consistency=cached"
+ ],
+ },
+ "python": {
+ "args": [
+ "strAPI.main:app",
+ "--host",
+ "0.0.0.0",
+ "--port",
+ "5000",
+ "--reload"
+ ],
+ "module": "uvicorn",
+ }
+ }
+ ]
+}

Dockerfile

@@ -1,18 +1,33 @@
-FROM python:3.8-slim-buster
+FROM python:3.8-slim-buster as base
 LABEL maintainer="merenlin -- follow me on medium https://medium.com/@merenlin"
 
-RUN apt-get update && apt-get install -y python3-dev build-essential
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
 
 RUN mkdir -p /usr/src/strs
 WORKDIR /usr/src/strs
 
 COPY requirements.txt .
-RUN pip3 install -r requirements.txt
+
+RUN apt-get update && \
+ apt-get install -y gcc python3-dev build-essential && \
+ python -m pip install -r requirements.txt --no-cache-dir && \
+ apt-get remove -y gcc python3-dev build-essential && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV DATABASE_URL=sqlite:///db/debug.db
+
+FROM base as prod
+LABEL maintainer="merenlin -- follow me on medium https://medium.com/@merenlin"
+
+RUN adduser -u 1000 --disabled-password --gecos "" appuser && chown -R appuser /usr/src/strs
+USER appuser
 
 COPY . .
 
-EXPOSE 5000
+ARG PORT=5000
+EXPOSE $PORT
 
-CMD uvicorn strAPI.main:app --host=0.0.0.0 --port=${PORT:-5000}
+CMD bash entrypoint.sh
 
 

README.md

@@ -23,10 +23,17 @@ Yes! It is possible and we encourage it if you would like to add your own data t
 
 ### Instructions on how to set-up webSTR-API locally (for development): 
 
-#### Step 0: Set up the database
+#### Step 0: (docker and local ways) Set up the database 
  Install and configure PostgreSQL on your machine and create an empty database called strdb. We provide an sql_dump backup of the current version of the database on request. Restore the database from this backup. 
-
-#### Step 1: Install all the requirements
+
+ Alternatively you can use `docker compose` with provided docker-compose.yml file to set up the PostgreSQL database.
+ 1. rename `.env.example` to `.env`
+ 2. copy backup to folder ./db/docker_data/pgdata/webstr_backup.dump
+ 2. `docker compose up -d db`
+ 3. `docker exec -it webstr-api-db-1 pg_restore -d strdb /var/lib/postgresql/data/pgdata/webstr_backup.dump`
+
+#### Step 1: (only for non-docker way) Install all the requirements 
+
 
 a) Set up python3 and virtualenv on your machine:
 [For Mac, follow instructions here.](https://gist.github.com/pandafulmanda/730a9355e088a9970b18275cb9eadef3)
@@ -38,22 +45,40 @@ b) Create new virtual env and install all the requirements with the following co
 `pip install -r requirements.txt`
 
 
-#### Step 2: Set environmental variable DATABASE_URL on your machine (or your IDE) to 
-
+#### Step 2: (only for non-docker way) Set environmental variable DATABASE_URL on your machine (or your IDE) to 
 `export DATABASE_URL="postgres://postgres:YOURPASSWORD@localhost:5432/strdb"`
 
 Note that this is using the default user postgres, if you created your db on a different user, adjust this variable accordingly. 
 
 Optional: add this line to `~/.bashrc` and restart your terminal. 
 
-#### Step 3: Start API server
+#### Step 3: (only for non-docker way) Start API server 
 
 Run the following command from the root folder of this repo: 
 
 `uvicorn strAPI.main:app --host=0.0.0.0 --port=${PORT:-5000} --reload`
 
-#### Step 4: You can now access the api at `http://0.0.0.0:5000` 
+#### Step 4: You can now access the api at `http://localhost:5000` 
+
+***
 
+### How to build and run application using docker way
+
+#### Step 0: Set up the database 
+##### if you want to use PostgreSQL and backup
+See docker part of Step 1 from the previous instructions
+##### if sqlite is enough
+Change database url in .env file to DATABASE_URL=sqlite:///db/debug.sqlite
+Set WEBSTR_DATABASE_DATA_UPGRADE=True and WEBSTR_DEVELOPMENT=True in .env file
+#### Step 1: Start containers
+Run `docker compose up`
+
+Now you can access api on localhost:5000 and frontend on localhost:5001
+
+##### debugging in docker container
+If you want to debug the code and you use VSCode - you can run code in container using vscode and tasks defined in launch.json in .vscode folder. Or you can use devcontaienr plugin to do the same. This is a bit more advanced, so you need to study how debuggin in containers works in vscode a bit before that. And you will have to stop webster-api-api-1 container first if you have it running through docker compose.
+
+***
 
 ## How to build the database from scratch or import my own data to WebSTR? 
 
@@ -76,3 +101,14 @@ Explore "database_setup" directory for different utilities to import data into t
 * If you would like **to import a new reference panel** we recommend making a csv corresponding to the repeats table structure and importing it directly to SQL to save time. Alternatively see ` insert_repeats.py ` and 
 ` import_data_ensembltrs.py ` utilities that we made for repeats data coming in different formats. Feel free to contact us for more details if you would like to make your own reference STR panel. 
 
+***
+
+### Database migrations using Alembic - Proof of Concept, not used in production.
+@slmjy added alembic migrations as a proof of concept to the codebase.
+Following changes were introduced:
+1. Database migrations in /database_setup/migrations
+2. alembic.ini and env.py files 
+3. entrypoint.sh can run alembic migratiosn given environment variable WEBSTR_DATABASE_MIGRATE is set to True
+4. in database.py there is a disabled check if database is on the latest version
+
+If someone wants to start using alembic migrations, they can enable the version check and start generating new migrations using alembic and using them in production.