colosseum.analysis.utils

  1import os
  2import re
  3from glob import glob
  4from typing import List, Tuple
  5
  6import numpy as np
  7import pandas as pd
  8
  9from colosseum import config
 10from colosseum.utils import ensure_folder
 11from colosseum.utils.formatter import clear_agent_mdp_class_name
 12
 13
 14def get_formatted_name(mdp_or_agent_class_name: str, mdp_or_agent_prms: str) -> str:
 15    """
 16    provides a nice formatting for the name of an agent or MDP class and its corresponding gin config index.
 17
 18    Parameters
 19    ----------
 20    mdp_or_agent_class_name : str
 21        The str containing the name of an agent or MDP class.
 22    mdp_or_agent_prms : str
 23        The str containing the gin config prms of an agent or MDP.
 24
 25    Returns
 26    -------
 27    str
 28        A nicely formatted name for the MDP and the agent specification in input.
 29    """
 30    return (
 31        clear_agent_mdp_class_name(mdp_or_agent_class_name)
 32        + f" ({1 + int(re.findall('[0-9]+', mdp_or_agent_prms)[0])})"
 33    )
 34
 35
 36def format_indicator_name(indicator: str) -> str:
 37    """
 38    provides a nice formatting to the indicator code name in input.
 39
 40    Parameters
 41    ----------
 42    indicator : str
 43        The indicator code name.
 44
 45    Returns
 46    -------
 47    str
 48        A nicely formatted string.
 49    """
 50    return indicator.replace("_", " ").replace("normalized", "norm.").capitalize()
 51
 52
 53def get_available_mdps_agents_prms_and_names(
 54    experiment_folder: str,
 55) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]:
 56    """
 57    retrieves the gin configurations for MDPs and agents from the experiment folder.
 58
 59    Parameters
 60    ----------
 61    experiment_folder : str
 62        The folder where the experiments are.
 63
 64    Returns
 65    -------
 66    Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]
 67        The tuples of gin config parameters and MDPs/agents class names in the given experiment folder.
 68    """
 69    logs_folders = os.listdir(f"{ensure_folder(experiment_folder)}logs{os.sep}")
 70    available_mdps, available_agents = set(), set()
 71    for logs_folder in logs_folders:
 72        mdp_code, agent_code = logs_folder.split(config.EXPERIMENT_SEPARATOR_MDP_AGENT)
 73        available_mdps.add(
 74            tuple(mdp_code.split(config.EXPERIMENT_SEPARATOR_PRMS)[::-1])
 75        )
 76        available_agents.add(
 77            tuple(agent_code.split(config.EXPERIMENT_SEPARATOR_PRMS)[::-1])
 78        )
 79    return sorted(available_mdps, key=lambda x: "".join(x)), sorted(
 80        available_agents, key=lambda x: "".join(x)
 81    )
 82
 83
 84def get_logs_data(
 85    experiment_folder: str,
 86    mdp_class_name: str,
 87    mdp_prm: str,
 88    agent_class_name: str,
 89    agent_prm: str,
 90) -> Tuple[pd.DataFrame, int]:
 91    """
 92    retrieves the logging data of the experiment folder for the given mdp class, mdp gin config, agent class, and agent
 93    gin config.
 94
 95    Parameters
 96    ----------
 97    experiment_folder : str
 98        The folder that contains the experiment logs, MDP configurations and agent configurations.
 99    mdp_class_name : str
100        The name of the MDP class.
101    mdp_prm : str
102        The gin configuration parameter of the MDP.
103    agent_class_name
104        The name of the agent class.
105    agent_prm : str
106        The gin configuration parameter of the agent.
107
108    Returns
109    -------
110    pd.DataFrame
111        The logging data stored as a pd.DataFrame.
112    int
113        The number of seeds of used in the experiment.
114    """
115
116    mdp_code = mdp_prm + config.EXPERIMENT_SEPARATOR_PRMS + mdp_class_name
117    agent_code = agent_prm + config.EXPERIMENT_SEPARATOR_PRMS + agent_class_name
118    log_seed_files = glob(
119        f"{experiment_folder}{os.sep}logs{os.sep}{mdp_code}*{agent_code}{os.sep}*.csv"
120    )
121    assert len(log_seed_files), f"No logs files found for {mdp_code}___{agent_code}"
122    dfs = []
123    for log_seed_file in log_seed_files:
124        seed = int(log_seed_file[-10])
125        df = pd.read_csv(log_seed_file)
126        df.loc[:, "seed"] = seed
127        df.loc[-1] = {
128            c: seed
129            if c == "seed"
130            else (df.loc[0, c] if c == "steps_per_second" else 0.0)
131            for c in df.columns
132        }
133        dfs.append(df.sort_index())
134    return pd.concat(dfs).reset_index(drop=True), len(log_seed_files)
135
136
137def get_n_failed_interactions(
138    experiment_folder: str,
139    mdp_class_name: str,
140    mdp_prm: str,
141    agent_class_name: str,
142    agent_prm: str,
143) -> int:
144    """
145    retrieves the number of times the agent config has failed to complete the total number of interaction in time in the
146    given MDP config.
147
148    Parameters
149    ----------
150    experiment_folder : str
151        The folder that contains the experiment logs, MDP configurations and agent configurations.
152    mdp_class_name : str
153        The name of the MDP class.
154    mdp_prm : str
155        The gin configuration parameter of the MDP.
156    agent_class_name
157        The name of the agent class.
158    agent_prm : str
159        The gin configuration parameter of the agent.
160
161    Returns
162    -------
163    int
164        The number of time the agent broke the computational time limit.
165    """
166
167    mdp_code = mdp_prm + config.EXPERIMENT_SEPARATOR_PRMS + mdp_class_name
168    agent_code = agent_prm + config.EXPERIMENT_SEPARATOR_PRMS + agent_class_name
169    time_exceeded_file = (
170        f"{ensure_folder(experiment_folder)}logs{os.sep}{mdp_code}"
171        f"{config.EXPERIMENT_SEPARATOR_MDP_AGENT}{agent_code}{os.sep}time_exceeded.txt"
172    )
173    if os.path.isfile(time_exceeded_file):
174        with open(time_exceeded_file, "r") as f:
175            failed = set(f.readlines())
176        return len(failed)
177    return 0
178
179
180def add_time_exceed_sign_to_plot(
181    ax,
182    df: pd.DataFrame,
183    color: str,
184    indicator: str,
185    n_seeds: int,
186    experiment_folder,
187    mdp_prm: str,
188    agent_prm: str,
189):
190    """
191    adds the '}' symbol to the plot in the time corresponding to the average time the agent broke the computational time
192    limit.
193
194    Parameters
195    ----------
196    ax : plt.Axes
197        The ax object where the symbol will be put.
198    df : pd.DataFrame
199        The logging data of the experiment for a given agent config and MDP config.
200    color : str
201        The code name for the color of the symbol.
202    indicator : str
203        is a string representing the performance indicator that is shown in the plot. Check `MDPLoop.get_indicators()`
204        to get a list of the available indicators. By default, the 'normalized_cumulative_regret' is used.
205    n_seeds : int
206        The total number of seed used in the experiment.
207    experiment_folder : str
208        The path of the directory containing the experiment results.
209    mdp_prm : str
210        The gin configuration parameter of the MDP.
211    agent_prm : str
212        The gin configuration parameter of the agent.
213    """
214
215    time_exceeded_file = (
216        f"{ensure_folder(experiment_folder)}logs{os.sep}{mdp_prm}"
217        f"{config.EXPERIMENT_SEPARATOR_MDP_AGENT}{agent_prm}{os.sep}time_exceeded.txt"
218    )
219    if os.path.isfile(time_exceeded_file):
220        with open(time_exceeded_file, "r") as f:
221            failed = set(f.readlines())
222
223        # Getting the average time step at which the training stopped for the different seeds
224        mean_time_step = 0
225        for fail in failed:
226            time_step = int(fail[fail.find("(") + 1 : fail.find(")")])
227            mean_time_step += time_step / n_seeds
228
229        # Plot the stopped training symbol on the plot for the given measure
230        mean_time_step = df.loc[:, "steps"].tolist()[
231            np.argmin(np.abs(df.loc[:, "steps"] - mean_time_step))
232        ]
233        ax.text(
234            mean_time_step,
235            df[df.loc[:, "steps"] == mean_time_step].loc[:, indicator].mean(),
236            "}",
237            fontdict=dict(size=27),
238            verticalalignment="center",
239            color=color,
240        )