Worldbuilder task app refactor

crowdsourcing/custom_world_interactions/collect_narrations/fix_annotation.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from mephisto.abstractions.databases.local_database import LocalMephistoDB
+from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
+from mephisto.data_model.worker import Worker
+from mephisto.data_model.unit import Unit
+from collections import Counter
+
+db = LocalMephistoDB()
+mephisto_data_browser = MephistoDataBrowser(db=db)
+
+DO_REVIEW = True
+
+# task_names = ["objects-interaction-task-pilot-3", "objects-interaction-task-pilot-4", "objects-interaction-task-pilot-5"]
+task_names = ["objects-interaction-task-pilot-5"]
+
+units = []
+for t in task_names:
+    units.extend(mephisto_data_browser.get_units_for_task_name(t))
+
+print(f"prev len: {len(units)}")
+print(Counter([u.get_status() for u in units]))
+
+# workers_to_res = {}
+# worker_name_to_worker = {}
+
+# for unit in units:
+#     status = unit.get_db_status()
+#     # contents = unit["data"]
+#     data = mephisto_data_browser.get_data_from_unit(unit)
+#     outputs = data['data']['outputs']
+#     if "pristine looking bucket, without a mark. Must have" in outputs['rawAction']:
+#         print(outputs)
+#         unit.set_db_status("soft_rejected")
+#         break
+

crowdsourcing/custom_world_interactions/collect_narrations/get_good_workers.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from mephisto.abstractions.databases.local_database import LocalMephistoDB
+from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
+from mephisto.data_model.worker import Worker
+from mephisto.data_model.unit import Unit
+from collections import Counter
+
+db = LocalMephistoDB()
+mephisto_data_browser = MephistoDataBrowser(db=db)
+
+DO_REVIEW = True
+
+units = mephisto_data_browser.get_units_for_task_name("objects-interaction-task-pilot-4")
+
+print(f"prev len: {len(units)}")
+print(Counter([u.get_status() for u in units]))
+# units = [u for u in units if u.get_status() == "completed"]
+# units = [u for u in units if u.get_status() == "completed"]
+# print(f"len: {len(units)}")
+
+workers_to_res = {}
+worker_name_to_worker = {}
+
+for unit in units:
+    worker = unit.get_assigned_agent().get_worker()
+    worker_name = worker.worker_name
+    worker_name_to_worker[worker_name] = worker
+    status = unit.get_db_status()
+    workers_to_res[worker_name] = [status, *workers_to_res.get(worker_name, [])]
+
+# print({k:v for k, v in workers_to_res.items() if len(v) > 1})
+for k, v in workers_to_res.items():
+    if len(v) <= 0:
+        continue
+    counted =  Counter(v)
+    # if counted.get("accepted", 0) >= counted.get('soft_rejected', 0) * 2:
+    if counted.get("accepted", 0) == len(v):
+        worker = worker_name_to_worker[k]
+        worker.grant_qualification("collect_narrations_task_block", 0)
+        print(worker_name_to_worker[k], counted.get("accepted", 0), counted.get('soft_rejected', 0))
+
+

crowdsourcing/custom_world_interactions/collect_narrations/hydra_configs/conf/collect_narrations.yaml

@@ -1,5 +1,7 @@
 #@package _global_
 mephisto:
+  provider:
+    requester_name: alexgurung_sandbox
   blueprint:
     task_source: ${task_dir}/webapp/build/bundle.js
     extra_source_dir: ${task_dir}/webapp/src/static

crowdsourcing/custom_world_interactions/collect_narrations/hydra_configs/conf/find_workers.yaml

@@ -5,15 +5,16 @@ mephisto:
     extra_source_dir: ${task_dir}/webapp/src/static
     units_per_assignment: 1
     block_qualification: collect_narrations_task_block
+    # approve_qualification: collect_narrations_task_allow
   task:
     maximum_units_per_worker: 5
-    max_num_concurrent_units: 100
-    task_name: "objects-interaction-task-pilot-1"
+    max_num_concurrent_units: 10
+    task_name: "objects-interaction-task-pilot-5"
     task_title: "Annotate an interaction between two objects. (pilot)"
     task_description: "In this task you will recieve two object lists. You will be asked to select one from each, and then describe in second person an interaction that would occur if one were to use the objects together."
     task_reward: 0.70
     task_tags: "fantasy,object,interaction"
-  database: 
+  database:
     _database_type: singleton
-num_tasks: 150
+num_tasks: 70
 qualify_new_workers: true

crowdsourcing/custom_world_interactions/collect_narrations/review_results.py

@@ -7,12 +7,25 @@
 from mephisto.abstractions.databases.local_database import LocalMephistoDB
 from mephisto.tools.examine_utils import run_examine_or_review, print_results
 from mephisto.data_model.worker import Worker
+from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
+from collections import Counter
 
 db = LocalMephistoDB()
 
+mephisto_data_browser = MephistoDataBrowser(db=db)
+task_names = ["objects-interaction-task-pilot-5", "objects-interaction-task-pilot-4"]
+
+units = []
+for t in task_names:
+    units.extend(mephisto_data_browser.get_units_for_task_name(t))
+
+print(f"prev len: {len(units)}")
+print(Counter([u.get_status() for u in units]))
+units = [u for u in units if u.get_status() == "completed"]
+print(f"len: {len(units)}")
 
 def format_data_for_printing(data):
-    worker_name = Worker(db, data["worker_id"]).worker_name
+    worker_name = Worker.get(db, data["worker_id"]).worker_name
     contents = data["data"]
     duration = contents["times"]["task_end"] - contents["times"]["task_start"]
     metadata_string = (
@@ -25,7 +38,8 @@ def format_data_for_printing(data):
 
     if contents["inputs"] is not None and contents["outputs"] is not None:
         inputs = contents["inputs"]
-        outputs = contents["outputs"]["final_data"]
+        # outputs = contents["outputs"]["final_data"]
+        outputs = contents["outputs"]
         primary = outputs["primaryObject"]
         secondary = outputs["secondaryObject"]
         primary_object_map = {
@@ -43,7 +57,7 @@ def format_data_for_printing(data):
             f"\t\t{primary}: {primary_object_map[primary]}\n"
             f"\t\t{secondary}: {secondary_object_map[secondary]}\n\n"
         )
-        outputs_string = f"Output:\n\tUse {primary} with {secondary}\n\tAction: {outputs['actionDescription']}\n"
+        outputs_string = f"Output:\n\tUse {primary} with {secondary}\n\n\tRaw Action: {outputs.get('rawAction')}\n\tAction Description: {outputs['actionDescription']}\n"
 
     return f"-------------------\n{metadata_string}{inputs_string}{outputs_string}"
 

crowdsourcing/custom_world_interactions/collect_narrations/run_task.py

@@ -9,11 +9,9 @@
 import shutil
 import subprocess
 from mephisto.operations.operator import Operator
-from mephisto.operations.utils import get_root_dir
 from mephisto.tools.scripts import load_db_and_process_config
-from mephisto.abstractions.blueprint import BlueprintArgs
 from mephisto.abstractions.blueprints.static_react_task.static_react_blueprint import (
-    BLUEPRINT_TYPE,
+    BLUEPRINT_TYPE_STATIC_REACT as BLUEPRINT_TYPE
 )
 from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import (
     SharedStaticTaskState,
@@ -32,7 +30,8 @@
 from typing import List, Any
 
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
-LIGHT_DB_PATH = "~/ParlAI/data/LIGHT/merged.db"
+# LIGHT_DB_PATH = "~/ParlAI/data/LIGHT/merged.db"
+LIGHT_DB_PATH = "~/ParlAI/data/light/environment/db/d3/database3.db"
 PRIMARY_OBJECT_LIST_SIZE = 8
 SECONDARY_OBJECT_LIST_SIZE = 8
 DEFAULT_NUM_TASKS = 20
@@ -41,6 +40,7 @@
 mephisto_data_browser = MephistoDataBrowser(db=db)
 
 defaults = [
+    "_self_",
     {"mephisto/blueprint": BLUEPRINT_TYPE},
     {"mephisto/architect": "local"},
     {"mephisto/provider": "mock"},
@@ -141,15 +141,20 @@ def validate_unit(unit):
     if unit.get_assigned_agent() is None:
         return
 
-    output = mephisto_data_browser.get_data_from_unit(unit)["data"]
+    unit_data = mephisto_data_browser.get_data_from_unit(unit)
+    unit_data = unit_data.get("data")
 
-    if output is None:
+    if unit_data is None:
         return
 
-    data = mephisto_data_browser.get_data_from_unit(unit)["data"]["outputs"][
-        "final_data"
-    ]
-    action_description = data["actionDescription"].strip()
+    data = unit_data.get("outputs")
+
+    if data is None or "actionDescription" not in data:
+        return
+    action_description = data["actionDescription"]
+    if type(action_description) is not str:
+        return
+    action_description = action_description.strip()
 
     if (
         len(action_description) <= 20
@@ -219,7 +224,6 @@ def onboarding_always_valid(onboarding_data):
 
     db, cfg = load_db_and_process_config(cfg)
     operator = Operator(db)
-
     operator.validate_and_run_config(cfg.mephisto, shared_state)
     operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
 

crowdsourcing/custom_world_interactions/collect_narrations/test_get.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from mephisto.abstractions.databases.local_database import LocalMephistoDB
+from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser
+from mephisto.data_model.worker import Worker
+from mephisto.data_model.unit import Unit
+from collections import Counter
+
+db = LocalMephistoDB()
+mephisto_data_browser = MephistoDataBrowser(db=db)
+
+DO_REVIEW = True
+
+units = mephisto_data_browser.get_units_for_task_name("objects-interaction-task-pilot-5")
+
+print(f"prev len: {len(units)}")
+print(Counter([u.get_status() for u in units]))
+units = [u for u in units if u.get_status() == "completed"]
+print(f"len: {len(units)}")