colosseum.experiment.utils

  1import os
  2from glob import glob
  3from typing import Dict, List, Optional, Type, Union
  4
  5import gin
  6import yaml
  7from tqdm import tqdm
  8
  9from colosseum import config
 10from colosseum.agent.agents.base import BaseAgent
 11from colosseum.agent.utils import sample_n_agent_hyperparameters
 12from colosseum.experiment import ExperimentConfig
 13from colosseum.experiment.folder_structuring import (
 14    get_mdp_agent_gin_configs,
 15    get_experiment_config,
 16)
 17from colosseum.mdp import BaseMDP
 18from colosseum.utils import ensure_folder
 19from colosseum.utils.miscellanea import get_colosseum_agent_classes
 20from colosseum.utils.miscellanea import get_colosseum_mdp_classes
 21
 22
 23def apply_gin_config(gin_config_files_paths: List[str]):
 24    """
 25    binds the gin files configuration to the corresponding objects.
 26    """
 27
 28    import gin
 29
 30    gin.clear_config()
 31
 32    get_colosseum_mdp_classes()
 33    get_colosseum_agent_classes()
 34
 35    for config_file in gin_config_files_paths:
 36        gin.parse_config_file(config_file)
 37
 38
 39def check_same_experiment(folder_1: str, folder_2: str, exclude_config: bool = False):
 40    """
 41    checks whether all the files from folder_1 perfectly match the configuration files in folder_2.
 42    """
 43
 44    # Check same experiment configuration
 45    if not exclude_config:
 46        with open(ensure_folder(folder_1) + "experiment_config.yml", "r") as f:
 47            config_1 = yaml.load(f, yaml.Loader)
 48        with open(ensure_folder(folder_2) + "experiment_config.yml", "r") as f:
 49            config_2 = yaml.load(f, yaml.Loader)
 50        if config_1 != config_2:
 51            return False
 52
 53    # Check same MDP configs
 54    mdp_configs_1 = set(os.listdir(ensure_folder(folder_1) + "mdp_configs"))
 55    mdp_configs_2 = set(os.listdir(ensure_folder(folder_2) + "mdp_configs"))
 56    if mdp_configs_1 != mdp_configs_2:
 57        return False
 58    for mdp_config in mdp_configs_1:
 59        with open(ensure_folder(folder_1) + "mdp_configs" + os.sep + mdp_config) as f:
 60            mdp_config_1 = f.read()
 61        with open(ensure_folder(folder_2) + "mdp_configs" + os.sep + mdp_config) as f:
 62            mdp_config_2 = f.read()
 63        if mdp_config_1 != mdp_config_2:
 64            return False
 65
 66    # Check same agent configs
 67    if "agents_configs" in os.listdir(ensure_folder(folder_1)):
 68        agent_configs_1 = set(os.listdir(ensure_folder(folder_1) + "agents_configs"))
 69        agent_configs_2 = set(os.listdir(ensure_folder(folder_2) + "agents_configs"))
 70        if agent_configs_1 != agent_configs_2:
 71            return False
 72        for agent_config in agent_configs_1:
 73            with open(
 74                ensure_folder(folder_1) + "agents_configs" + os.sep + agent_config
 75            ) as f:
 76                agent_config_1 = f.read()
 77            with open(
 78                ensure_folder(folder_2) + "agents_configs" + os.sep + agent_config
 79            ) as f:
 80                agent_config_2 = f.read()
 81            if agent_config_1 != agent_config_2:
 82                return False
 83
 84    return True
 85
 86
 87def check_experiments_to_run_folders_formatting(
 88    experiment_configs: Optional[Dict[str, ExperimentConfig]] = None
 89):
 90    """
 91    checks whether there are problems in the structure of the experiments folders.
 92    """
 93    experiment_folders = glob(config.get_experiments_folder() + "**")
 94    assert (
 95        len(experiment_folders) > 0
 96    ), f"No experiment folder found in {config.get_experiment_to_run_folder()}."
 97
 98    for experiment_folder in experiment_folders:
 99        check_experiment_folder(experiment_folder, experiment_configs)
100
101
102def check_experiment_folder(
103    experiment_folder: str,
104    experiment_config: Union[str, ExperimentConfig] = None,
105):
106    """
107    checks the integrity of the experiment folder.
108    """
109
110    assert os.path.isdir(
111        experiment_folder
112    ), f"The file {experiment_folder} is not a directory, please remove it."
113
114    assert "agents_configs" in os.listdir(
115        experiment_folder
116    ), f"The experiment folder {experiment_folder} is missing the agents_configs folder."
117    assert "mdp_configs" in os.listdir(
118        experiment_folder
119    ), f"The experiment folder {experiment_folder} is missing the mdp_configs folder."
120    b_config_file = "experiment_config.yml" in os.listdir(experiment_folder)
121
122    experiment_name = experiment_folder[experiment_folder.rfind(os.sep) + 1 :]
123    assert (
124        experiment_config is not None or b_config_file
125    ), "The experiment configuration should be either provided as a .yml file or as an ExperimentConfig object."
126    if b_config_file:
127        config_file = ensure_folder(experiment_folder) + "experiment_config.yml"
128        with open(config_file, "r") as f:
129            experiment_config = yaml.load(f, yaml.Loader)
130        assert (
131            "n_seeds" in experiment_config.keys()
132        ), f"n_seeds missing from {config_file}."
133        assert (
134            "n_steps" in experiment_config.keys()
135        ), f"n_steps missing from {config_file}."
136        assert (
137            "max_interaction_time_s" in experiment_config.keys()
138        ), f"max_interaction_time_s missing from {config_file}."
139        assert (
140            "log_performance_indicators_every" in experiment_config.keys()
141        ), f"log_performance_indicators_every missing from {config_file}."
142    else:
143        assert (
144            type(experiment_config) == ExperimentConfig
145        ), "The experiment configuration should be given as an ExperimentConfig object."
146
147    from colosseum.experiment.folder_structuring import remove_corrupted_log_files
148
149    remove_corrupted_log_files(experiment_folder, ExperimentConfig(**experiment_config))
150
151
152def instantiate_gin_files(
153    dest_folder: str,
154    agent_classes: List[Type["BaseAgent"]],
155    mdp_classes: List[Type["BaseMDP"]],
156    n_samples_agents: int,
157    n_samples_mdps: int,
158    seed: int,
159) -> List[str]:
160    """
161    produces and instantiates gin files from samples of the given MDP and Agent classes in the destination folder.
162    Returns
163    -------
164    List[str]
165        The file paths of the gin files
166    """
167    os.makedirs(dest_folder, exist_ok=True)
168
169    # Store gin files for MDPs
170    gin_files = []
171    for mdp_class in mdp_classes:
172        fp = (
173            ensure_folder(dest_folder)
174            + "mdp_configs"
175            + os.sep
176            + mdp_class.__name__
177            + ".gin"
178        )
179        gin_files.append(fp)
180        with open(fp, "w") as f:
181            f.write(
182                "\n\n".join(
183                    mdp_class.produce_gin_file_from_mdp_parameters(
184                        mdp_hyperparameters, mdp_class.__name__, i
185                    )
186                    for i, mdp_hyperparameters in enumerate(
187                        mdp_class.sample_parameters(n_samples_mdps, seed)
188                    )
189                )
190            )
191
192    # Store gin file for agents
193    for agent_class in agent_classes:
194        fp = (
195            ensure_folder(dest_folder)
196            + "agents_configs"
197            + os.sep
198            + agent_class.__name__
199            + ".gin"
200        )
201        gin_files.append(fp)
202        with open(fp, "w") as f:
203            f.write(
204                "\n\n".join(
205                    agent_class.produce_gin_file_from_parameters(
206                        agent_hyperparameter, i
207                    )
208                    for i, agent_hyperparameter in enumerate(
209                        sample_n_agent_hyperparameters(
210                            n_samples_agents, agent_class, seed
211                        )
212                    )
213                )
214            )
215
216    return gin_files
217
218
219def instantiate_mdps_from_experiment_folder(
220    experiment_folder: str, exclude_horizon_from_parameters=False
221) -> List["BaseMDP"]:
222    """
223    Returns
224    -------
225    List["BaseMDP"]
226        The MDP instances corresponding to the experiment folder in input.
227    """
228    (
229        mdp_classes_scopes,
230        agent_classes_scopes,
231        gin_config_files_paths,
232    ) = get_mdp_agent_gin_configs(experiment_folder)
233    exp_config = get_experiment_config(experiment_folder, None)
234
235    loop = mdp_classes_scopes.items()
236    if config.VERBOSE_LEVEL != 0:
237        loop = tqdm(loop, desc=os.path.basename(experiment_folder))
238
239    mdps = []
240    for mdp_class, mdp_scopes in loop:
241        for mdp_scope in mdp_scopes:
242            apply_gin_config(gin_config_files_paths)
243            with gin.config_scope(mdp_scope):
244                for seed in range(exp_config.n_seeds):
245                    mdps.append(
246                        mdp_class(
247                            seed=seed,
248                            exclude_horizon_from_parameters=exclude_horizon_from_parameters,
249                        )
250                    )
251
252    return mdps