constants.py

import os

"""
This file is a Singleton instance that holds project wide constants
Note: This file should NOT be used for experiment configuration, rather these are constants that will rarely, if ever, be changed.
I.e. a reference to the root directory of the project

Use Example
>>> from constants import ROOT_DIR
"""

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
DEMONSTRATION_DIR = os.path.join(ROOT_DIR, 'data', "demonstrations")

# ONLINE RLHF OUTPUTS
ONLINE_EXP_ROOT = os.path.expanduser("~/Documents/VMAS_Experiments/")

DEMO_FILENAMES = {
    "obs": "obs.csv",
    "act": "actions.csv",
    "info": "info.csv",
    "reward": "rewards.csv",
    "done": "done.csv",
    "config": "config.yml",
}

BASE_ENV_CONFIG = {
    "device": "cpu",
    "continuous_actions": True,
    "max_steps": 500,
    "scenario_config": {
        "n_agents": 3,
    },
}

BASE_PPO_CONFIG = {
    "framework": "torch",
    "num_gpus": 0,
    "num_workers": 0,
    "train_batch_size": 60000,
    "rollout_fragment_length": 125,
    "sgd_minibatch_size": 4096,
    "explore": False,
    "env_config": BASE_ENV_CONFIG,
}

BASE_MULTI_AGENT_CONFIG = {
    "activation_fn": "relu",
    "heterogeneous": False,
    "comm_radius": 0.35,
    "pos_start": 0,
    "pos_dim": 2,
    "vel_start": 2,
    "vel_dim": 2,
    "trainer": "MultiPPOTrainer",
    "share_action_value": False,
    "use_beta": False,
    "add_agent_index": False,
    "aggr": "add",
    "gnn_type": "MatPosConv",
}

ALGO_CONFIG = {
    "CPPO": {"use_mlp": True, "centralized_critic": False, "share_observations": False, "heterogeneous": False},
    "GAPPO": {"use_mlp": False, "centralized_critic": True, "share_observations": True, "heterogeneous": False},
    "MAPPO": {"use_mlp": False, "centralized_critic": True, "share_observations": False, "heterogeneous": False},
    "GPPO": {"use_mlp": False, "centralized_critic": False, "share_observations": True, "heterogeneous": False},
    "IPPO": {"use_mlp": False, "centralized_critic": False, "share_observations": False, "heterogeneous": False},
}

DEFAULT_AGENTS_COUNT = {
    "balance": 3,
    "transport": 3,
    "buzz_wire": 2,
    "football": 3,
    "navigation": 3,
    "wheel": 3,
    "diffdrive_navigation": 3,
    "diffdrive_pusher": 3
}

GT_REWARD_KEY = {
    "balance": "custom_metrics/agent_0/gt_reward_mean",
    "transport": "custom_metrics/agent_0/gt_reward_mean",
    "buzz_wire": "custom_metrics/agent_0/gt_reward_mean",
    "football": "custom_metrics/agent_blue_0/gt_reward_mean",
    "navigation": "custom_metrics/agent_0/gt_reward_mean",
    "wheel": "custom_metrics/agent_0/gt_reward_mean",
    "diffdrive_navigation": "custom_metrics/agent_0/gt_reward_mean"
}

ONLINE_RLHF_EXPERIMENT_CONFIG = {
    "MAX_CYCLES": 100,
    "EPISODE_LENGTH": 500,
    "REWARD_ITERS_PER_CYCLE": 3,
    "INIT_TRAIN_DEMOS": 10,
    "INIT_TEST_DEMOS": 5,
    "TRAIN_DEMOS_PER_CYCLE": 5,
    "TEST_DEMOS_PER_CYCLE": 2,
    "TRAIN_PREFERENCES_PER_CYCLE": 50,
    "TEST_PREFERENCES_PER_CYCLE": 15,
    "MAX_PREFS": 5000,
    "POLICY_STEPS_PER_CYCLE": int(5e5),
    "MAX_RL_TIMEOUT_SEC": 300,
    "SEGMENT_LENGTH": 20,
}

KEYBOARD_DEMO_INSTRUCTIONS = """
You can change the agent to control by pressing TAB.
You can reset the environment by pressing R.
You can control agent actions with the arrow keys and M/N (left/right control the first action, up/down control the second, M/N controls the third)

If you have more than 1 agent, you can control another one with W,A,S,D and Q,E in the same way.
and switch the agent using LSHIFT.
"""

POLICY_DEMO_INSTRUCTIONS = """
An AI agent (policy) will now execute a policy to collect demonstrations.
To terminate, use Ctrl+C in the terminal or close the simulator GUI.
"""