[Draft] LIGHT-QA paper code merge

projects/lightqa/crowdsourcing/acute_eval/.gitignore

@@ -0,0 +1,4 @@
+task_config/pairings_files/
+task_config/acute_results/
+block_list.txt
+webapp/package-lock.json

projects/lightqa/crowdsourcing/acute_eval/acute_eval_blueprint.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from dataclasses import dataclass, field
+from typing import ClassVar, Type, TYPE_CHECKING, Dict, Any
+
+from mephisto.operations.registry import register_mephisto_abstraction
+from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import (
+    FastAcuteBlueprint,
+    FastAcuteBlueprintArgs,
+)
+from parlai_internal.projects.light.lightqa.crowdsourcing.acute_eval.acute_eval_builder import (
+    LightQAAcuteBuilder,
+)
+
+if TYPE_CHECKING:
+    from mephisto.abstractions.blueprint import TaskBuilder
+
+LIGHTQA_FAST_BLUEPRINT_TYPE = "lightqa_fast_acute_eval"
+
+
+@dataclass
+class LightQAFastAcuteBlueprintArgs(FastAcuteBlueprintArgs):
+    _blueprint_type: str = LIGHTQA_FAST_BLUEPRINT_TYPE
+    additional_task_description_prefix: str = field(
+        default="Thats the default",
+        metadata={
+            "help": "Additional text to show on the left pane in the beginning of the text."
+        },
+    )
+    dialogues_input_dir: str = field(
+        default="",
+        metadata={"help": "Input directory from which to load the dialogue files."},
+    )
+    add_history: bool = field(
+        default=False,
+        metadata={"help": "Add a history to the chat."},
+    )
+    add_knowledge: bool = field(
+        default=True,
+        metadata={"help": "Add knowledge to the chat."},
+    )
+
+
+@register_mephisto_abstraction()
+class LightQAFastAcuteBlueprint(FastAcuteBlueprint):
+    """
+    Subclass of FastAcuteEvalBlueprint with params for fast ACUTE runs.
+    """
+
+    TaskBuilderClass: ClassVar[Type["TaskBuilder"]] = LightQAAcuteBuilder
+    ArgsClass = LightQAFastAcuteBlueprintArgs
+    BLUEPRINT_TYPE = LIGHTQA_FAST_BLUEPRINT_TYPE
+
+    def get_frontend_args(self) -> Dict[str, Any]:
+        """
+        Specifies what options within a task_config should be forwarded to the client
+        for use by the task's frontend.
+        """
+        return {
+            **super().get_frontend_args(),
+            **{
+                "additional_task_description_prefix": self.args.blueprint.additional_task_description_prefix,
+            },
+        }

projects/lightqa/crowdsourcing/acute_eval/acute_eval_builder.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+
+from parlai.crowdsourcing.tasks.acute_eval.acute_eval_builder import AcuteEvalBuilder
+
+
+class LightQAAcuteBuilder(AcuteEvalBuilder):
+    """
+    Subclass of AcuteEvalBuilder to specify a local frontend build directory.
+    """
+
+    ACUTE_TASK_DIR = os.path.dirname(os.path.abspath(__file__))
+    FRONTEND_SOURCE_DIR = os.path.join(ACUTE_TASK_DIR, "webapp")
+    FRONTEND_BUILD_DIR = os.path.join(FRONTEND_SOURCE_DIR, "build")

projects/lightqa/crowdsourcing/acute_eval/hydra_configs/conf/conf.yaml

@@ -0,0 +1,22 @@
+#@package _global_
+mephisto:
+  blueprint:
+    acute_eval_type: knowledgeable
+    subtasks_per_unit: 4
+    root_dir: ${task_dir}/task_config
+    randomize_conversations: True
+    task: nq_open
+    matchups_per_pair: 1
+  task:
+    allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
+    maximum_units_per_worker: 1  # important for replicability and reliability
+    task_description: "Evaluate quality of conversations through comparison."
+    task_name: acute_eval_${current_time}
+    task_reward: 0.5
+    task_tags: "chat,evaluation,comparison,conversation"
+    task_title: "Which Conversational Partner is Better?"
+    additional_task_description_prefix: "On the right you see two chat windows with the same structure: First, a longer context from a Wikipedia article, followed by a question about this topic."
+mturk:
+  worker_blocklist_paths: "parlai_standard"

projects/lightqa/crowdsourcing/acute_eval/run.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import datetime
+import json
+from dataclasses import dataclass, field
+from typing import List, Any, Dict, Optional
+
+import hydra
+from mephisto.operations.operator import Operator
+from mephisto.operations.hydra_config import register_script_config
+from mephisto.tools.scripts import load_db_and_process_config
+from omegaconf import DictConfig
+from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
+from parlai.crowdsourcing.tasks.acute_eval.fast_eval import (
+    ACUTE_EVAL_TYPES,
+    FastAcuteExecutor,
+)
+
+from parlai_internal.projects.light.lightqa.crowdsourcing.acute_eval.util import (
+    TASK_DIRECTORY,
+    download_parlai_worker_blocklist,
+)
+from parlai_internal.projects.light.lightqa.crowdsourcing.acute_eval.acute_eval_blueprint import (
+    LIGHTQA_FAST_BLUEPRINT_TYPE,
+)
+from parlai_internal.crowdsourcing.block_list import ALLOW_LIST_QUAL_ID
+from parlai_internal.projects.light.lightqa.crowdsourcing.acute_eval.acute_eval_blueprint import (
+    LightQAFastAcuteBlueprint,
+)
+
+"""
+Read parlai/crowdsourcing/README.md to learn how to launch
+crowdsourcing tasks with this script.
+"""
+
+ACUTE_EVAL_TYPES = {
+    **ACUTE_EVAL_TYPES,
+    **{
+        "knowledgeable": {
+            "question": "If you had to say that one speaker is more knowledgeable and one is more ignorant, who is more knowledgeable?",
+            "s1_choice": "<Speaker 1> is more knowledgeable",
+            "s2_choice": "<Speaker 2> is more knowledgeable",
+        }
+    },
+}
+
+
+class LightFastAcuteExecutor(FastAcuteExecutor):
+    def __init__(self, args: DictConfig):
+        self.args = args
+        self.fast_acute_args = self.args.mephisto.blueprint
+
+        # models + task
+        self.task: str = self.fast_acute_args.task
+        self.pairings_filepath = None
+
+        # question config for running ACUTE
+        self.question_config: Dict[str, str] = ACUTE_EVAL_TYPES[
+            self.fast_acute_args.acute_eval_type
+        ]
+
+        self.run_id = self.args.mephisto.task.task_name
+
+        if args.mturk.worker_blocklist_paths == "parlai_standard":
+            args.mturk.worker_blocklist_paths = download_parlai_worker_blocklist()
+
+    def _load_pairings_file(self):
+        """
+        Load pairings file.
+
+        Need to set:
+        - self.pairings_filepath
+        - self.models
+        - self.combos
+        """
+        self.pairings_filepath = os.path.join(
+            self.fast_acute_args.root_dir, "pairings_files", "pairings.jsonl"
+        )
+        self._load_models()
+
+    def _load_models(self):
+        with open(self.pairings_filepath, "r") as f:
+            convs = [json.loads(line) for line in f]
+        self.models = list(
+            set([model for conv in convs for model in conv["speakers_to_eval"]])
+        )
+        combos = list(
+            set([",".join(sorted(conv["speakers_to_eval"])) for conv in convs])
+        )
+        self.combos = [tuple(combo.split(",")) for combo in combos]
+        self.models.sort()
+
+    def run_acute_eval(self):
+        """
+        Run ACUTE Eval.
+        """
+        self.set_up_acute_eval()
+        db, cfg = load_db_and_process_config(self.args)
+        print(f"*** RUN ID: {cfg.mephisto.task.task_name} ***")
+        operator = Operator(db)
+        shared_state = LightQAFastAcuteBlueprint.SharedStateClass()
+        shared_state.mturk_specific_qualifications = [
+            {
+                "QualificationTypeId": ALLOW_LIST_QUAL_ID,
+                "Comparator": "Exists",
+                "ActionsGuarded": "DiscoverPreviewAndAccept",
+            },
+        ]
+        operator.validate_and_run_config(
+            run_config=cfg.mephisto, shared_state=shared_state
+        )
+        operator.wait_for_runs_then_shutdown(
+            skip_input=True, log_rate=cfg.monitoring_log_rate
+        )
+
+
+defaults = [
+    {"mephisto/blueprint": LIGHTQA_FAST_BLUEPRINT_TYPE},
+    {"mephisto/architect": "local"},
+    {"mephisto/provider": "mock"},
+    {"conf": "conf_fast"},
+]
+
+
+@dataclass
+class LightQAScriptConfig(MTurkRunScriptConfig):
+    defaults: List[Any] = field(default_factory=lambda: defaults)
+    task_dir: str = TASK_DIRECTORY
+    monitoring_log_rate: int = field(
+        default=30,
+        metadata={
+            "help": "Frequency in seconds of logging the monitoring of the crowdsourcing task"
+        },
+    )
+
+
+register_script_config(name="lightqa_scriptconfig", module=LightQAScriptConfig)
+
+
+@hydra.main(config_path="hydra_configs", config_name="lightqa_scriptconfig")
+def main(cfg: DictConfig) -> None:
+
+    runner = LightFastAcuteExecutor(cfg)
+    runner.run_acute_eval()
+    runner.analyze_results()
+
+
+if __name__ == "__main__":
+    main()

projects/lightqa/crowdsourcing/acute_eval/util.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+from typing import List, Optional
+from parlai_internal.crowdsourcing.block_list import WORKER_BLOCK_LIST
+
+TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+
+
+def download_parlai_worker_blocklist(file_name: Optional[str] = None) -> str:
+    """
+    Download the standard ParlAI blocklist for MTurk.
+
+    Return the download path.
+    """
+    if not file_name:
+        file_name = f"{TASK_DIRECTORY}/block_list.txt"
+    os.makedirs("/".join(file_name.split("/")[:-1]), exist_ok=True)
+    with open(file_name, "w") as f:
+        f.write("\n".join(WORKER_BLOCK_LIST))
+    return file_name

projects/lightqa/crowdsourcing/acute_eval/webapp/.babelrc

@@ -0,0 +1,4 @@
+{
+    "presets": ["@babel/env", "@babel/preset-react"],
+    "plugins": ["@babel/plugin-proposal-class-properties"],
+}

projects/lightqa/crowdsourcing/acute_eval/webapp/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "parlai-mephisto-task-compiler",
+  "version": "1.0.0",
+  "description": "",
+  "main": "webpack.config.js",
+  "scripts": {
+    "dev": "webpack --mode development -q"
+  },
+  "keywords": [],
+  "author": "",
+  "dependencies": {
+    "bootstrap": "^4.3.1",
+    "jquery": "^3.0.0",
+    "mephisto-task": "^1.0.10",
+    "react": "16.13.1",
+    "react-bootstrap": "^0.32.4",
+    "react-dom": "16.13.1",
+    "react-table": "^6.8.6"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.1.0",
+    "@babel/core": "^7.1.0",
+    "@babel/plugin-proposal-class-properties": "^7.1.0",
+    "@babel/preset-env": "^7.1.0",
+    "@babel/preset-react": "^7.0.0",
+    "babel-loader": "^8.0.2",
+    "css-loader": "^1.0.0",
+    "style-loader": "^0.23.0",
+    "url-loader": "^2.0.1",
+    "webpack": "^4.19.1",
+    "webpack-cli": "^3.1.1"
+  }
+}

projects/lightqa/crowdsourcing/acute_eval/webapp/src/app.jsx

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+import React from "react";
+import ReactDOM from "react-dom";
+import {
+  TaskDescription,
+  BaseFrontend,
+} from "./components/core_components.jsx";
+import { useMephistoTask, getBlockedExplanation } from "mephisto-task";
+
+/* ================= Application Components ================= */
+
+function MainApp() {
+  const {
+    blockedReason,
+    taskConfig,
+    isPreview,
+    isLoading,
+    initialTaskData,
+    handleSubmit,
+  } = useMephistoTask();
+
+  if (blockedReason !== null) {
+    return <h1>{getBlockedExplanation(blockedReason)}</h1>;
+  }
+  if (isPreview) {
+    return <TaskDescription task_config={taskConfig} is_cover_page={true} />;
+  }
+  if (isLoading) {
+    return <div>Initializing...</div>;
+  }
+  if (initialTaskData === null) {
+    return <h1>Gathering data...</h1>;
+  }
+
+  return (
+    <div>
+      <BaseFrontend
+        task_data={initialTaskData}
+        task_config={taskConfig}
+        onSubmit={handleSubmit}
+      />
+    </div>
+  );
+}
+
+ReactDOM.render(<MainApp />, document.getElementById("app"));