Merge pull request #800 from facebookresearch/add-tests-for-remote-ex…

…amples

Add tests for remote examples

.github/workflows/cypress-end-to-end-tests.yml

@@ -10,6 +10,7 @@ jobs:
  steps:
  - name: 🔀 Checking out repo
  uses: actions/checkout@v2
+
  - name: 🐍 Installing python
  uses: actions/setup-python@v2
 
@@ -29,8 +30,7 @@ jobs:
  mkdir data
 
  - name: 📂 Set the data directory
- run: |
- mephisto config core.main_data_directory ~/mephisto/data
+ run: mephisto config core.main_data_directory ~/mephisto/data
 
  - name: 📦 Setting up mephisto-task package
  run: |
@@ -43,9 +43,154 @@ jobs:
  uses: cypress-io/[email protected]
  with:
  install: false
+ browser: chrome
  project: ./examples/static_react_task/webapp
  config-file: ./cypress.config.js
  start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
  wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
  command-prefix: yarn dlx
  headless: true
+
+ remote_procedure_template:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: 🔀 Checking out repo
+ uses: actions/checkout@v2
+
+ - name: 🐍 Installing python
+ uses: actions/setup-python@v2
+
+ - name: 🪨 Setup Node
+ uses: actions/setup-node@v1
+ with:
+ node-version: 16
+
+ - name: 🤖 Install Mephisto
+ run: pip install -e .
+
+ - name: 🖋 Create data directory
+ run: |
+ cd ~
+ mkdir mephisto
+ cd mephisto
+ mkdir data
+ - name: 📂 Set the data directory
+ run: mephisto config core.main_data_directory ~/mephisto/data
+
+ - name: 📦 Setting up mephisto-task package
+ run: |
+ cd packages/mephisto-task
+ yarn install
+ yarn build
+ npm link
+
+ - name: ⌛️ Running cypress tests
+ uses: cypress-io/[email protected]
+ with:
+ install: false
+ browser: chrome
+ project: ./examples/remote_procedure/template/webapp
+ config-file: ./cypress.config.js
+ start: python examples/remote_procedure/template/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+ wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+ command-prefix: yarn dlx
+ headless: true
+
+ remote_procedure_mnist:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: 🔀 Checking out repo
+ uses: actions/checkout@v2
+
+ - name: 🐍 Installing python
+ uses: actions/setup-python@v2
+
+ - name: 🪨 Setup Node
+ uses: actions/setup-node@v1
+ with:
+ node-version: 16
+
+ - name: 🤖 Install Mephisto
+ run: |
+ pip install -e .
+ pip install torch pillow numpy
+
+ - name: 🖋 Create data directory
+ run: |
+ cd ~
+ mkdir mephisto
+ cd mephisto
+ mkdir data
+
+ - name: 📂 Set the data directory
+ run: mephisto config core.main_data_directory ~/mephisto/data
+
+ - name: 📦 Setting up mephisto-task package
+ run: |
+ cd packages/mephisto-task
+ yarn install
+ yarn build
+ npm link
+
+ - name: ⌛️ Running cypress tests
+ uses: cypress-io/[email protected]
+ with:
+ install: false
+ browser: chrome
+ project: ./examples/remote_procedure/mnist/webapp
+ config-file: ./cypress.config.js
+ start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+ wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+ command-prefix: yarn dlx
+ headless: true
+
+ remote_procedure_toxicity_detection:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: 🔀 Checking out repo
+ uses: actions/checkout@v2
+
+ - name: 🐍 Installing python
+ uses: actions/setup-python@v2
+
+ - name: 🪨 Setup Node
+ uses: actions/setup-node@v1
+ with:
+ node-version: 16
+
+ - name: 🤖 Install Mephisto
+ run: |
+ pip install -e .
+ pip install detoxify
+
+ - name: 🖋 Create data directory
+ run: |
+ cd ~
+ mkdir mephisto
+ cd mephisto
+ mkdir data
+
+ - name: 📂 Set the data directory
+ run: mephisto config core.main_data_directory ~/mephisto/data
+
+ - name: 📦 Setting up mephisto-task package
+ run: |
+ cd packages/mephisto-task
+ yarn install
+ yarn build
+ npm link
+
+ - name: ⌛️ Running cypress tests
+ uses: cypress-io/[email protected]
+ with:
+ install: false
+ browser: chrome
+ project: ./examples/remote_procedure/toxicity_detection/webapp
+ config-file: ./cypress.config.js
+ start: python examples/remote_procedure/toxicity_detection/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+ wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+ command-prefix: yarn dlx
+ headless: true

examples/remote_procedure/mnist/webapp/cypress.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+ video: false,
+
+ e2e: {
+ baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1",
+ supportFile: false,
+ },
+};

examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js

@@ -0,0 +1,94 @@
+describe("Loads remote_procedure_mnist", () => {
+ it("Makes request for agent", () => {
+ cy.intercept({ pathname: "/request_agent" }).as("agentRequest");
+ cy.visit("/");
+ cy.wait("@agentRequest").then((interception) => {
+ expect(interception.response.statusCode).to.eq(200);
+ });
+ });
+ it("Loads correct react elements", () => {
+ cy.visit("/");
+ cy.get('[data-cy="canvas-container-0"]');
+ cy.get('[data-cy="clear-button-0"]');
+ cy.get('[data-cy="correct-checkbox-0"]');
+ cy.get('[data-cy="correct-text-input-0"]');
+
+ cy.get('[data-cy="canvas-container-1"]');
+ cy.get('[data-cy="clear-button-1"]');
+ cy.get('[data-cy="correct-checkbox-1"]');
+ cy.get('[data-cy="correct-text-input-1"]');
+
+ cy.get('[data-cy="canvas-container-2"]');
+ cy.get('[data-cy="clear-button-2"]');
+ cy.get('[data-cy="correct-checkbox-2"]');
+ cy.get('[data-cy="correct-text-input-2"]');
+
+ cy.get('[data-cy="submit-button"]').as("submitButton");
+ cy.get("@submitButton").should("be.disabled");
+ });
+
+ it("Submitting with three corrected annotations", () => {
+ cy.visit("/");
+ cy.on("window:alert", (txt) => {
+ expect(txt).to.contain("The task has been submitted!");
+ });
+
+ cy.get('[data-cy="clear-button-0"]').as("clearButton0");
+ cy.get('[data-cy="clear-button-1"]').as("clearButton1");
+ cy.get('[data-cy="clear-button-2"]').as("clearButton2");
+
+ // draw 4
+ cy.get('[data-cy="canvas-mouse-down-container-0"]')
+ .trigger("mouseover")
+ .trigger("mousedown", 20, 20)
+ .trigger("mousemove", 40, 150)
+ .trigger("mousemove", 150, 120)
+ .trigger("mousemove", 150, 10)
+ .trigger("mouseup", 150, 220);
+
+ // There is a wait statement here because it takes some time for the model to calculate that it is a 4
+ cy.wait(2000);
+ cy.get('[data-cy="current-annotation-0"]').should("contain.text", "4");
+ cy.get('[data-cy="correct-checkbox-0"]').check();
+ cy.get('[data-cy="correct-text-input-0"]').should("not.exist");
+
+ // draw 1
+ cy.get('[data-cy="canvas-mouse-down-container-1"]')
+ .trigger("mouseover")
+ .trigger("mousedown", 20, 60)
+ .trigger("mousedown", 40, 40)
+ .trigger("mousedown", 150, 30)
+ .trigger("mousedown", 180, 60)
+ .trigger("mousedown", 180, 120)
+ .trigger("mousedown", 150, 135)
+ .trigger("mousedown", 85, 145)
+ .trigger("mousedown", 180, 150)
+ .trigger("mousedown", 180, 210)
+ .trigger("mouseup", 65, 220);
+
+ cy.wait(2000);
+ cy.get('[data-cy="current-annotation-1"]').should("contain.text", "3");
+ cy.get('[data-cy="correct-checkbox-1"]').check();
+ cy.get('[data-cy="correct-text-input-1]').should("not.exist");
+
+ // draw gibberish
+ cy.get('[data-cy="canvas-mouse-down-container-2"]')
+ .trigger("mouseover")
+ .trigger("mousedown", 10, 20)
+ .trigger("mousedown", 120, 200)
+ .trigger("mouseup", 120, 200);
+
+ cy.get('[data-cy="clear-button-2"]').click();
+
+ // draw 7
+ cy.get('[data-cy="canvas-mouse-down-container-2"]')
+ .trigger("mouseover")
+ .trigger("mousedown", 30, 30)
+ .trigger("mousedown", 220, 40)
+ .trigger("mouseup", 40, 200);
+
+ cy.get('[data-cy="correct-text-input-2"]').type("7");
+
+ cy.get('[data-cy="submit-button"]').click({ force: true });
+ });
+});

examples/remote_procedure/mnist/webapp/link_mephisto_task.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+npm link mephisto-task