Clean up turbiniactl and other minor updates (#1463)

* Clean up turbiniactl * Updates to turbiniactl * Update tests * Update tests * Updates to e2e tests and docker-compose * Updates to tests and fix linter error * test * Updates * Fix e2e action * Updates * Test * Test * Test * Updates to actions * Updates to e2e-local script * Updates * Minor updates * Update actions/setup-python to v5 * Python 3.12 * Python version 3.12 * Update workflows * Updates to e2e-local sh * Update actions.yml
google · Apr 4, 2024 · 435de73 · 435de73
1 parent a1e143b
commit 435de73
Show file tree

Hide file tree

Showing 9 changed files with 147 additions and 1,268 deletions.
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -8,15 +8,12 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- include:
- - docker_base_image: "ubuntu:22.04"
- gift_ppa_track: "staging"
- - docker_base_image: "ubuntu:22.04"
- gift_ppa_track: "stable"
-
+ os:
+ - ubuntu-22.04
+ gift_ppa_track: ["staging", "stable"]
  steps:
  - name: checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  - name: Set up Docker Buildx
  uses: docker/setup-buildx-action@v3
  - name: Buid Turbinia Unit Tests Docker image

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -5,18 +5,15 @@ on: [push, pull_request]
 jobs:
  e2e-test:
  name: Run local stack e2e test
- runs-on: ubuntu-latest
  strategy:
  matrix:
- include:
- - docker_base_image: "ubuntu:22.04"
- gift_ppa_track: "staging"
- - docker_base_image: "ubuntu:22.04"
- gift_ppa_track: "stable"
-
+ os:
+ - ubuntu-22.04
+ gift_ppa_track: ["staging", "stable"]
+ runs-on: ${{ matrix.os }}
  steps:
  - name: checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  - name: Set up Docker Buildx
  uses: docker/setup-buildx-action@v3
  - name: Build Turbinia API server Docker image
@@ -58,6 +55,10 @@ jobs:
  - name: Patch docker-compose config to use locally build images
  run: |
  sed -i -e 's/#image: "t/image: "t/g' -e 's/image: "u/#image: "u/g' ./docker/local/docker-compose.yml
+ - name: Update pip
+ run: python -m pip install --upgrade pip
+ - name: Install turbinia-client
+ run: pip install turbinia-client
  - name: Run E2E test
  run: |
  chmod +x ./turbinia/e2e/e2e-local.sh

diff --git a/README.md b/README.md
@@ -1,4 +1,5 @@
 # Turbinia
+![Unit tests](https:/google/turbinia/actions/workflows/actions.yml/badge.svg) ![e2e tests](https:/google/turbinia/actions/workflows/e2e.yml/badge.svg)
 
 ## Summary
 

diff --git a/docker/local/docker-compose.yml b/docker/local/docker-compose.yml
@@ -7,22 +7,20 @@ services:
  # See https://docs.docker.com/compose/compose-file/compose-file-v3/#expose
  expose:
  - "6379"
-
  volumes:
  - $PWD/redis-data:/var/lib/redis
-
  environment:
  - REDIS_REPLICATION_MODE=master
 
  turbinia-server:
  #image: "turbinia-server-dev" # Use this for local development and comment out below line
  image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-server:latest" # Latest stable
  container_name: turbinia-server
-
+ depends_on:
+ - redis
  volumes:
  - $PWD/evidence:/evidence
  - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
-
  environment:
  - LC_ALL=C.UTF-8
  - LANG=C.UTF-8
@@ -32,30 +30,33 @@ services:
  #image: "turbinia-api-server-dev" # Use this for local development and comment out below line
  image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-api-server:latest" # Latest stable
  container_name: turbinia-api-server
-
+ depends_on:
+ - redis
  volumes:
  - $PWD/evidence:/evidence
  - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
-
  environment:
  - LC_ALL=C.UTF-8
  - LANG=C.UTF-8
  - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
+ expose:
+ - "8000"
 
  turbinia-worker:
  #image: "turbinia-worker-dev" # Use this for local development and comment out below line
  image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker:latest" # Latest stable
  container_name: turbinia-worker
  privileged: true
-
+ depends_on:
+ - redis
  volumes:
  - $PWD/evidence:/evidence
  - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
-
  environment:
  - LC_ALL=C.UTF-8
  - LANG=C.UTF-8
  - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
+
  # Uncomment below in case you want to run a second worker on the same host.
  # turbinia-worker2:
  # image: "turbinia-worker-dev" # Use this for local development and comment out below line

diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile
@@ -12,6 +12,8 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \
  gpg \
  john \
  john-data \
+ libleveldb1d \
+ libleveldb-dev \
  libssl-dev \
  libterm-readline-gnu-perl \
  libtool \
@@ -41,6 +43,9 @@ RUN add-apt-repository -y ppa:gift/$PPA_TRACK
 RUN apt-get update && apt-get -y install \
  bulk-extractor \
  docker-explorer-tools \
+ libbde-tools \
+ libfsapfs-tools \
+ libluksde-tools \
  sleuthkit \
  libewf-tools \
  && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*

diff --git a/turbinia/api/api_server.py b/turbinia/api/api_server.py
@@ -28,9 +28,6 @@
 from turbinia.api.routes.router import api_router
 from turbinia.api.routes.ui import ui_router
 
-from turbinia.config import logger
-
-logger.setup()
 log = logging.getLogger('turbinia')
 log.setLevel(logging.INFO)
 

diff --git a/turbinia/e2e/e2e-local.sh b/turbinia/e2e/e2e-local.sh
@@ -4,7 +4,8 @@
 # The evidence processed is a prepared raw disk image.
 
 # Set default return value
-RET=0
+RET=1
+set -o posix
 
 echo "Create evidence folder"
 mkdir -p ./evidence
@@ -18,8 +19,8 @@ echo "==> Startup local turbinia docker-compose stack"
 export TURBINIA_EXTRA_ARGS="-d"
 docker-compose -f ./docker/local/docker-compose.yml up -d
 
-echo "==> Sleep for 10s"
-sleep 10s
+echo "==> Sleep for 10 seconds to let containers start"
+sleep 10
 
 echo "==> Show and check running containers"
 containers=( turbinia-server turbinia-worker turbinia-api-server redis )
@@ -36,46 +37,76 @@ do
 done
 echo "All containers up and running!"
 
+echo "==> Getting the turbinia-api-server container IP address"
+API_SERVER=`docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' turbinia-api-server`
+echo "==> Got IP address: $API_SERVER"
+
+echo "==> Generating turbinia-client configuration"
+echo '{
+ "default": {
+ "description": "Local e2e test environment",
+ "API_SERVER_ADDRESS": "http://turbinia-api-server",
+ "API_SERVER_PORT": 8000,
+ "API_AUTHENTICATION_ENABLED": false,
+ "CLIENT_SECRETS_FILENAME": ".client_secrets.json",
+ "CREDENTIALS_FILENAME": ".credentials.json"
+ }
+}' | sed s/turbinia-api-server/$API_SERVER/> ./evidence/.turbinia_api_config.json
+cat ./evidence/.turbinia_api_config.json
+
 echo "==> Show loop device availability in worker"
 docker exec -t turbinia-worker /sbin/losetup -a
 docker exec -t turbinia-worker ls -al /dev/loop*
 
 echo "==> Show evidence volume contents in worker"
 docker exec -t turbinia-worker ls -al /evidence/
 
-echo "==> Show container logs"
-docker logs turbinia-server
-docker logs turbinia-worker
-docker logs turbinia-api-server
-
 echo "==> Create Turbinia request"
-docker exec -t turbinia-server turbiniactl -r 123456789 -P /evidence/e2e-recipe.yaml rawdisk -l /evidence/artifact_disk.dd
+RECIPE_DATA=`cat ./evidence/e2e-recipe.yaml | base64 -w0`
+turbinia-client -p ./evidence submit rawdisk --source_path /evidence/artifact_disk.dd --request_id 123456789 --recipe_data {$RECIPE_DATA} 
 
-echo "==> Sleep for 150 seconds to let Turbinia process evidence"
-sleep 150s
+echo "==> Waiting 5 seconds before polling request status"
+sleep 5
 
-echo "==> Display Turbinia request status"
-docker exec turbinia-server turbiniactl -a status -r 123456789
+echo "==> Polling the API server for request status"
+# Wait until request is complete 
+req_status=$(turbinia-client -p ./evidence status request 123456789 -j | jq -r '.status')
+while [[ $req_status = "running" ]]
+do
+ req_status=$(turbinia-client -p ./evidence status request 123456789 -j | jq -r '.status')
+ if [[ $req_status = "running" ]]
+ then
+ echo "Turbinia request 123456789 is still running. Sleeping for 10 seconds..."
+ sleep 10
+ fi
+done
 
-echo "==> See if any tasks failed"
-FAILED=`docker exec turbinia-server turbiniactl -a status -r 123456789 | awk '/Failed Tasks/,/\* None/' | wc -l`
-if [ "$FAILED" != "2" ]; then
- echo 'Tasks failed!'
- RET=1
+echo "==> Check the status of the request"
+if [ $req_status != "successful" ]
+then
+ echo "Request is not running and the status is not successful!"
+else
+ echo "Request successfully completed"
+ RET=0
 fi
 
+echo "==> Displaying request status"
+turbinia-client -p ./evidence status request 123456789 -j
+
 echo "==> Show Turbinia server logs"
 docker logs turbinia-server
 
 echo "==> Show Turbinia worker logs"
 docker logs turbinia-worker
 
+echo "==> Show Turbinia API server logs"
+docker logs turbinia-api-server
+
 echo "==> Show evidence volume contents in worker"
 docker exec -t turbinia-worker ls -al /evidence/
 docker exec -t turbinia-worker find /evidence -ls
 
 echo "==> Show PlasoParserTask logs"
-for i in cat `docker exec turbinia-server turbiniactl -a status -r 123456789|grep -Eo '*/evidence/123456789/.*PlasoParserTask.*txt'`; do docker exec turbinia-worker cat $i; done
-
+for i in cat `turbinia-client -p ./evidence status request 123456789 -j | jq '.tasks[] | select(.name == "PlasoParserTask") | .saved_paths[]' | grep \.txt | tr -d '"'`; do docker exec turbinia-worker cat $i; done
 
 exit $RET