Reviewer with multiple characteristics

README.md

@@ -104,6 +104,17 @@ export AZURE_OPENAI_KEY=... # Your Azure OpenAI key here
 
 A demo can be found in `notebooks/demo.ipynb`
 
+### Customizing your own environment
+
+You can add a new setting in `agentreview/experiment_config.py`, then add the setting as a new entry to the `all_settings` dictionary:
+
+```python
+all_settings = {
+    "BASELINE": baseline_setting,
+    "benign_Rx1": benign_Rx1_setting,
+    ...
+    "your_setting_name": your_setting
+```
 
 ## Framework Overview
 

agentreview/arguments.py

@@ -1,18 +1,22 @@
 import argparse
 import logging
 import os
-import sys
+
+logger = logging.getLogger(__name__)
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description="Argument parser for configuring OpenAI API and experiment settings")
 
     # Authentication details for OpenAI API
     parser.add_argument(
-        "--openai_key", type=str, default=None, help="API key to authenticate with OpenAI. Can be set via this argument or through the OPENAI_API_KEY environment variable."
+        "--openai_key", type=str, default=None,
+        help="API key to authenticate with OpenAI. Can be set via this argument or through the OPENAI_API_KEY environment variable."
     )
 
     parser.add_argument(
-        "--deployment", type=str, default=None, help="For Azure OpenAI: the deployment name to be used when calling the API."
+        "--deployment", type=str, default=None,
+        help="For Azure OpenAI: the deployment name to be used when calling the API."
     )
 
     parser.add_argument(
@@ -21,10 +25,10 @@ def parse_args():
     )
 
     parser.add_argument(
-        "--endpoint", type=str, default=None, help="For Azure OpenAI: custom endpoint to access the API. Should be in the format 'https://<your-endpoint>.openai.azure.com'."
+        "--endpoint", type=str, default=None,
+        help="For Azure OpenAI: custom endpoint to access the API. Should be in the format 'https://<your-endpoint>.openai.azure.com'."
     )
 
-
     parser.add_argument(
         "--api_version", type=str, default="2023-05-15", help="API version to be used for making requests. Required "
                                                               "for Azure OpenAI clients."
@@ -37,7 +41,8 @@ def parse_args():
     )
 
     parser.add_argument(
-        "--conference", type=str, default="ICLR2023", help="Conference name where the papers are being evaluated, e.g., 'ICLR2023'."
+        "--conference", type=str, default="ICLR2023",
+        help="Conference name where the papers are being evaluated, e.g., 'ICLR2023'."
     )
 
     parser.add_argument(
@@ -47,22 +52,20 @@ def parse_args():
     parser.add_argument(
         "--experiment_name",
         type=str, default=None, required=False,
-        choices=[
-            "BASELINE", "benign_Rx1", "malicious_Rx1", "malicious_Rx2", "malicious_Rx3", "unknowledgeable_Rx1",
-            "knowledgeable_Rx1", "responsible_Rx1", "irresponsible_Rx1", "irresponsible_Rx2", "irresponsible_Rx3",
-            "inclusive_ACx1", "authoritarian_ACx1", "conformist_ACx1", "no_numeric_ratings"],
         help="Specifies the name of the experiment to run. Choose from predefined experiment types based on the reviewer and AC behavior or experiment configuration."
     )
 
     parser.add_argument(
-        "--overwrite", action="store_true", help="If set, existing results or output files will be overwritten without prompting."
+        "--overwrite", action="store_true",
+        help="If set, existing results or output files will be overwritten without prompting."
     )
     parser.add_argument(
         "--skip_logging", action="store_true", help="If set, we do not log the messages in the console."
     )
 
     parser.add_argument(
-        "--num_papers_per_area_chair", type=int, default=10, help="The number of papers each area chair is assigned for evaluation."
+        "--num_papers_per_area_chair", type=int, default=10,
+        help="The number of papers each area chair is assigned for evaluation."
     )
 
     # Model configuration
@@ -83,19 +86,20 @@ def parse_args():
     )
 
     parser.add_argument(
-        "--visual_dir", type=str, default="outputs/visual", help="Directory where visualization files (such as graphs and plots) will be stored."
+        "--visual_dir", type=str, default="outputs/visual",
+        help="Directory where visualization files (such as graphs and plots) will be stored."
     )
 
     # System configuration
     parser.add_argument(
-        "--device", type=str, default='cuda', help="The device to be used for processing (e.g., 'cuda' for GPU acceleration or 'cpu' for standard processing)."
+        "--device", type=str, default='cuda',
+        help="The device to be used for processing (e.g., 'cuda' for GPU acceleration or 'cpu' for standard processing)."
     )
 
     parser.add_argument(
         "--data_dir", type=str, default='data', help="Directory where input data (e.g., papers) are stored."
     )
 
-
     parser.add_argument(
         "--acceptance_rate", type=float, default=0.32,
         help="Percentage of papers to accept. We use 0.32, the average acceptance rate for ICLR 2020 - 2023"
@@ -126,6 +130,17 @@ def parse_args():
                                                       "OPENAI_API_KEY environment variable.")
             raise ValueError("OpenAI key is missing.")
 
+    EXISTING_EXPERIMENT_SETTINGS = [
+        "BASELINE", "benign_Rx1", "malicious_Rx1", "malicious_Rx2", "malicious_Rx3", "unknowledgeable_Rx1",
+        "knowledgeable_Rx1", "responsible_Rx1", "irresponsible_Rx1", "irresponsible_Rx2", "irresponsible_Rx3",
+        "inclusive_ACx1", "authoritarian_ACx1", "conformist_ACx1", "no_numeric_ratings"]
+
+    if args.experiment_name not in EXISTING_EXPERIMENT_SETTINGS:
+        logger.warning(f"Experiment name '{args.experiment_name}' is not recognized. "
+                       f"This can happen if you are customizing your own experiment settings. "
+                       f"Otherwise, please choose from the following: "
+                       f"{EXISTING_EXPERIMENT_SETTINGS}")
+
     if args.openai_client_type == "azure_openai":
         if os.environ.get('AZURE_OPENAI_KEY') is None:
             assert isinstance(args.openai_key, str), ("Please specify the `--openai_key` argument OR set the "

agentreview/experiment_config.py

@@ -224,6 +224,26 @@ no_numeric_ratings_setting = {
     }
 }
 
+malicious_and_irresponsible_Rx1_setting = {
+    "AC": [
+        "BASELINE"
+    ],
+
+    "reviewer": [
+        "malicious irresponsible",
+        "BASELINE",
+        "BASELINE"
+    ],
+
+    "author": [
+        "BASELINE"
+    ],
+    "global_settings":{
+        "provides_numeric_rating": ['reviewer', 'ac'],
+        "persons_aware_of_authors_identities": []
+    }
+}
+
 
 # All experimental settings.
 # Customize your own by adding new settings to this dict.
@@ -231,14 +251,15 @@ all_settings = {
     "BASELINE": baseline_setting,
     "benign_Rx1": benign_Rx1_setting,
     "malicious_Rx1": malicious_Rx1_setting,
-    "knowledgeable_Rx1_setting": knowledgeable_Rx1_setting,
-    "unknowledgeable_Rx1_setting": unknowledgeable_Rx1_setting,
-    "responsible_Rx1_setting": responsible_Rx1_setting,
-    "irresponsible_Rx1_setting": irresponsible_Rx1_setting,
+    "knowledgeable_Rx1": knowledgeable_Rx1_setting,
+    "unknowledgeable_Rx1": unknowledgeable_Rx1_setting,
+    "responsible_Rx1": responsible_Rx1_setting,
+    "irresponsible_Rx1": irresponsible_Rx1_setting,
     "conformist_ACx1": conformist_ACx1_setting,
     "authoritarian_ACx1": authoritarian_ACx1_setting,
     "inclusive_ACx1": inclusive_ACx1_setting,
     "no_numeric_ratings": no_numeric_ratings_setting,
+    "malicious_and_irresponsible_Rx1": malicious_and_irresponsible_Rx1_setting,
 
 }
 

agentreview/paper_review_settings.py

@@ -66,36 +66,29 @@ def get_reviewer_setting_from_reviewer_type(reviewer_type: str):
     }
 
     # Intention
-    if reviewer_type == "benign":
+    if "benign" in reviewer_type:
         reviewer_setting["is_benign"] = True
-    elif reviewer_type == "malicious":
+    if "malicious" in reviewer_type:
         reviewer_setting["is_benign"] = False
 
     # Knowledgeability
-    elif reviewer_type == "knowledgeable":
+    if "knowledgeable" in reviewer_type:
         reviewer_setting["is_knowledgeable"] = True
-    elif reviewer_type == "unknowledgeable":
+    if "unknowledgeable" in reviewer_type:
         reviewer_setting["is_knowledgeable"] = False
 
     # Commitment
-    elif reviewer_type == "responsible":
+    if "responsible" in reviewer_type:
         reviewer_setting["is_responsible"] = True
-    elif reviewer_type == "irresponsible":
+    if "irresponsible" in reviewer_type:
         reviewer_setting["is_responsible"] = False
 
-    elif reviewer_type in ["BASELINE"]:
-        pass
-
-    elif reviewer_type in ["authors_are_famous"]:
+    if reviewer_type in ["authors_are_famous"]:
         reviewer_setting["knows_authors"] = "famous"
 
     elif reviewer_type in ["authors_are_unfamous"]:
         reviewer_setting["knows_authors"] = "unfamous"
 
-
-    else:
-        raise ValueError(f"Unknown reviewer type: {reviewer_type}")
-
     return reviewer_setting
 
 

agentreview/role_descriptions.py

@@ -199,7 +199,7 @@ def get_reviewer_description(is_benign: bool = None, is_knowledgeable: bool = No
                                  "critically analyze the methodologies, and carefully consider the paper's "
                                  "contribution to the field. ")
 
-    desc_lazy_reviewer = ("As a lazy reviewer, your reviews tend to be superficial and hastily done. You do not like "
+    desc_irresponsible_reviewer = ("As a lazy reviewer, your reviews tend to be superficial and hastily done. You do not like "
                           "to discuss in the reviewer-AC discussion. "
                           "Your assessments might overlook critical details, lack depth in analysis, "
                           "fail to recognize novel contributions, "
@@ -210,7 +210,7 @@ def get_reviewer_description(is_benign: bool = None, is_knowledgeable: bool = No
         if is_responsible:
             desc = desc_responsible_reviewer
         else:
-            desc = desc_lazy_reviewer
+            desc = desc_irresponsible_reviewer
 
         bio += f"Responsibility: {desc}\n\n"