Merge branch 'improve-cli-for-mephisto-wut' of https://github.com/fac…

…ebookresearch/Mephisto into improve-cli-for-other-mephisto-commands

.github/workflows/cypress-end-to-end-tests.yml

@@ -10,6 +10,7 @@ jobs:
     steps:
       - name: 🔀 Checking out repo
         uses: actions/checkout@v2
+
       - name: 🐍 Installing python
         uses: actions/setup-python@v2
 
@@ -29,8 +30,7 @@ jobs:
           mkdir data
 
       - name: 📂 Set the data directory
-        run: |
-          mephisto config core.main_data_directory ~/mephisto/data
+        run: mephisto config core.main_data_directory ~/mephisto/data
 
       - name: 📦 Setting up mephisto-task package
         run: |
@@ -43,9 +43,154 @@ jobs:
         uses: cypress-io/[email protected]
         with:
           install: false
+          browser: chrome
           project: ./examples/static_react_task/webapp
           config-file: ./cypress.config.js
           start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
           wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
           command-prefix: yarn dlx
           headless: true
+
+  remote_procedure_template:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 🔀 Checking out repo
+        uses: actions/checkout@v2
+
+      - name: 🐍 Installing python
+        uses: actions/setup-python@v2
+
+      - name: 🪨 Setup Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+
+      - name: 🤖 Install Mephisto
+        run: pip install -e .
+
+      - name: 🖋 Create data directory
+        run: |
+          cd ~
+          mkdir mephisto
+          cd mephisto
+          mkdir data
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📦 Setting up mephisto-task package
+        run: |
+          cd packages/mephisto-task
+          yarn install
+          yarn build
+          npm link
+
+      - name: ⌛️ Running cypress tests
+        uses: cypress-io/[email protected]
+        with:
+          install: false
+          browser: chrome
+          project: ./examples/remote_procedure/template/webapp
+          config-file: ./cypress.config.js
+          start: python examples/remote_procedure/template/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+          command-prefix: yarn dlx
+          headless: true
+
+  remote_procedure_mnist:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 🔀 Checking out repo
+        uses: actions/checkout@v2
+
+      - name: 🐍 Installing python
+        uses: actions/setup-python@v2
+
+      - name: 🪨 Setup Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+
+      - name: 🤖 Install Mephisto
+        run: |
+          pip install -e .
+          pip install torch pillow numpy
+
+      - name: 🖋 Create data directory
+        run: |
+          cd ~
+          mkdir mephisto
+          cd mephisto
+          mkdir data
+
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📦 Setting up mephisto-task package
+        run: |
+          cd packages/mephisto-task
+          yarn install
+          yarn build
+          npm link
+
+      - name: ⌛️ Running cypress tests
+        uses: cypress-io/[email protected]
+        with:
+          install: false
+          browser: chrome
+          project: ./examples/remote_procedure/mnist/webapp
+          config-file: ./cypress.config.js
+          start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+          command-prefix: yarn dlx
+          headless: true
+
+  remote_procedure_toxicity_detection:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 🔀 Checking out repo
+        uses: actions/checkout@v2
+
+      - name: 🐍 Installing python
+        uses: actions/setup-python@v2
+
+      - name: 🪨 Setup Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+
+      - name: 🤖 Install Mephisto
+        run: |
+          pip install -e .
+          pip install detoxify
+
+      - name: 🖋 Create data directory
+        run: |
+          cd ~
+          mkdir mephisto
+          cd mephisto
+          mkdir data
+
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📦 Setting up mephisto-task package
+        run: |
+          cd packages/mephisto-task
+          yarn install
+          yarn build
+          npm link
+
+      - name: ⌛️ Running cypress tests
+        uses: cypress-io/[email protected]
+        with:
+          install: false
+          browser: chrome
+          project: ./examples/remote_procedure/toxicity_detection/webapp
+          config-file: ./cypress.config.js
+          start: python examples/remote_procedure/toxicity_detection/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
+          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+          command-prefix: yarn dlx
+          headless: true

.github/workflows/python-test.yml

@@ -22,7 +22,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pytest pytest-cov
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          pip install -e .
       - name: Test with pytest
         run: |
           python -m pytest -v -m "not req_creds" --cov --cov-report=xml

.gitignore

@@ -2,6 +2,7 @@ mephisto/tasks/*
 **/.mypy_cache/*
 **/__pycache__/*
 data/*
+dist/*
 mephisto.egg-info/*
 .DS_Store
 tmp/*

CONTRIBUTING.md

@@ -41,7 +41,7 @@ There is a way to fix this:
     * Make sure to reset the baseUrl back to default("/") if contributing to the repo as the GitHub action will fail otherwise.
 
 ## Local Package Development
-If you are modifying either the `mephisto-task` or `mephisto-worker-experience` packages you probably want to see your changes propagate to the task that you are working on.
+If you are modifying either the `mephisto-task` or `mephisto-worker-addons` packages you probably want to see your changes propagate to the task that you are working on.
 
 The easiest way to do this is to run a task by doing:
 ```bash

docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.md

@@ -6,7 +6,7 @@ import Link from '@docusaurus/Link';
 
 # Check worker quality with Screening Units
 
-Screening units help filter out low-quality work, generally by hiding parts of the validation you're paying attention to behind the Mephisto server. To support this we provide the `ScreenTaskRequred` blueprint mixin.
+Screening units help filter out low-quality work, generally by hiding parts of the validation you're paying attention to behind the Mephisto server. To support this we provide the `ScreenTaskRequired` blueprint mixin.
 
 Screening units are a heuristic-based way to determine, on the first task completion, if a worker has understood the instructions of a task. They can be run either on real data you want annotated (for cases where your heuristics can be run whenever) or on specific 'test' data you believe it's easier to validate on.
 
@@ -33,9 +33,9 @@ def validate_screening_unit(unit: "Unit"):
 shared_state = SharedTaskState(
     ...
     screening_data_factory=False
-    on_unit_submitted=UseGoldUnit.create_validation_function(cfg.mephisto, validate_gold_unit)
+    on_unit_submitted=ScreenTaskRequired.create_validation_function(cfg.mephisto, validate_gold_unit)
 )
-shared_state.qualifications += UseGoldUnit.get_mixin_qualifications(cfg.mephisto, shared_state)
+shared_state.qualifications += ScreenTaskRequired.get_mixin_qualifications(cfg.mephisto, shared_state)
 ...
 ```
 

docs/web/docs/guides/quickstart.md

@@ -20,7 +20,7 @@ $ pip install -e .
 
 ```bash
 # install poetry
-$ curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+$ curl -sSL https://install.python-poetry.org | python3 -
 # from the root dir, install Mephisto:
 $ poetry install
 ```
@@ -34,7 +34,7 @@ Now that you have Mephisto installed, you should have access to the `mephisto` C
 Let's use this CLI tool to set up a data directory via the `mephisto config` command. The data directory is where the results of your crowdsourcing tasks will be stored.
 
 ```bash
-$ mephisto config core.main_data_directory ~/mephisto/data
+$ mephisto config core.main_data_directory ~/Mephisto/data
 ```
 
 Check that everything is set up correctly!

docs/web/docs/guides/tutorials/first_task.md

@@ -44,7 +44,12 @@ If you change to another `worker_id`, however, you can complete two more tasks.
 
 ### 1.3 Reviewing tasks
 
-At this stage, we've collected data, and we'd like to review it. Here we'll do so with the command line interface, using the review script, which is in the same directory:
+By this stage, we've collected data and can now review it.
+
+Generally, to view/export the data, you could write a Python script using the Mephisto [`DataBrowser` class](https://github.com/facebookresearch/Mephisto/blob/main/mephisto/tools/data_browser.py) to access the submitted data.
+
+For your convenience, for the `html-static-task-example` task we've already provided such a script for you in the task folder, called [`examine_results.py`](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/examine_results.py). (This file uses the Mephisto `DataBrowser` class through the helpers in `mephisto.tools.examine_utils`.)
+
 ```bash
 $ python examine_results.py
 Do you want to (r)eview, or (e)xamine data? Default examine. Can put e <end> or e <start> <end> to choose how many to view

examples/remote_procedure/mnist/webapp/cypress.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  video: false,
+
+  e2e: {
+    baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1",
+    supportFile: false,
+  },
+};

examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js

@@ -0,0 +1,92 @@
+describe("Loads remote_procedure_mnist", () => {
+  it("Makes request for agent", () => {
+    cy.intercept({ pathname: "/request_agent" }).as("agentRequest");
+    cy.visit("/");
+    cy.wait("@agentRequest").then((interception) => {
+      expect(interception.response.statusCode).to.eq(200);
+    });
+  });
+  it("Loads correct react elements", () => {
+    cy.get('[data-cy="canvas-container-0"]');
+    cy.get('[data-cy="clear-button-0"]');
+    cy.get('[data-cy="correct-checkbox-0"]');
+    cy.get('[data-cy="correct-text-input-0"]');
+
+    cy.get('[data-cy="canvas-container-1"]');
+    cy.get('[data-cy="clear-button-1"]');
+    cy.get('[data-cy="correct-checkbox-1"]');
+    cy.get('[data-cy="correct-text-input-1"]');
+
+    cy.get('[data-cy="canvas-container-2"]');
+    cy.get('[data-cy="clear-button-2"]');
+    cy.get('[data-cy="correct-checkbox-2"]');
+    cy.get('[data-cy="correct-text-input-2"]');
+
+    cy.get('[data-cy="submit-button"]').as("submitButton");
+    cy.get("@submitButton").should("be.disabled");
+  });
+
+  it("Submitting with three corrected annotations", () => {
+    cy.on("window:alert", (txt) => {
+      expect(txt).to.contain("The task has been submitted!");
+    });
+
+    cy.get('[data-cy="clear-button-0"]').as("clearButton0");
+    cy.get('[data-cy="clear-button-1"]').as("clearButton1");
+    cy.get('[data-cy="clear-button-2"]').as("clearButton2");
+
+    // draw 4
+    cy.get('[data-cy="canvas-mouse-down-container-0"]')
+      .trigger("mouseover")
+      .trigger("mousedown", 20, 20)
+      .trigger("mousemove", 40, 150)
+      .trigger("mousemove", 150, 120)
+      .trigger("mousemove", 150, 10)
+      .trigger("mouseup", 150, 220);
+
+    // There is a wait statement here because it takes some time for the model to calculate that it is a 4
+    cy.wait(1000);
+    cy.get('[data-cy="current-annotation-0"]').should("contain.text", "4");
+    cy.get('[data-cy="correct-checkbox-0"]').check();
+    cy.get('[data-cy="correct-text-input-0"]').should("not.exist");
+
+    // draw 1
+    cy.get('[data-cy="canvas-mouse-down-container-1"]')
+      .trigger("mouseover")
+      .trigger("mousedown", 20, 60)
+      .trigger("mousedown", 40, 40)
+      .trigger("mousedown", 150, 30)
+      .trigger("mousedown", 180, 60)
+      .trigger("mousedown", 180, 120)
+      .trigger("mousedown", 150, 135)
+      .trigger("mousedown", 85, 145)
+      .trigger("mousedown", 180, 150)
+      .trigger("mousedown", 180, 210)
+      .trigger("mouseup", 65, 220);
+
+    cy.wait(1000);
+    cy.get('[data-cy="current-annotation-1"]').should("contain.text", "3");
+    cy.get('[data-cy="correct-checkbox-1"]').check();
+    cy.get('[data-cy="correct-text-input-1]').should("not.exist");
+
+    // draw gibberish
+    cy.get('[data-cy="canvas-mouse-down-container-2"]')
+      .trigger("mouseover")
+      .trigger("mousedown", 10, 20)
+      .trigger("mousedown", 120, 200)
+      .trigger("mouseup", 120, 200);
+
+    cy.get('[data-cy="clear-button-2"]').click();
+
+    // draw 7
+    cy.get('[data-cy="canvas-mouse-down-container-2"]')
+      .trigger("mouseover")
+      .trigger("mousedown", 30, 30)
+      .trigger("mousedown", 220, 40)
+      .trigger("mouseup", 40, 200);
+
+    cy.get('[data-cy="correct-text-input-2"]').type("7");
+
+    cy.get('[data-cy="submit-button"]').click({ force: true });
+  });
+});

examples/remote_procedure/mnist/webapp/link_mephisto_task.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+npm link mephisto-task