diff --git a/.gitignore b/.gitignore index 03321b53b..0967e2314 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,6 @@ code_carbon.db # Local file emissions*.csv* tests/test_data/rapl/* + +#asciinema +*.cast diff --git a/carbonserver/carbonserver/api/infra/repositories/repository_organizations.py b/carbonserver/carbonserver/api/infra/repositories/repository_organizations.py index 22a9b60e9..c8510940a 100644 --- a/carbonserver/carbonserver/api/infra/repositories/repository_organizations.py +++ b/carbonserver/carbonserver/api/infra/repositories/repository_organizations.py @@ -39,7 +39,14 @@ def add_organization(self, organization: OrganizationCreate) -> Organization: description=organization.description, api_key=generate_api_key(), ) - + existing_organization = ( + session.query(SqlModelOrganization).filter(SqlModelOrganization.name == organization.name).first() + ) + if existing_organization: + raise HTTPException( + status_code=404,detail=f"the organization name {organization.name} already exists" + ) + session.add(db_organization) session.commit() session.refresh(db_organization) diff --git a/carbonserver/carbonserver/api/infra/repositories/repository_projects.py b/carbonserver/carbonserver/api/infra/repositories/repository_projects.py index 4cfdcbe16..216ae6a40 100644 --- a/carbonserver/carbonserver/api/infra/repositories/repository_projects.py +++ b/carbonserver/carbonserver/api/infra/repositories/repository_projects.py @@ -24,6 +24,16 @@ def add_project(self, project: ProjectCreate): description=project.description, team_id=project.team_id, ) + existing_project = ( + session.query(SqlModelProject) + .filter(SqlModelProject.name == project.name) + .filter(SqlModelProject.team_id == project.team_id) + .first() + ) + if existing_project: + raise HTTPException( + status_code=404,detail=f"the project name {project.name} of team {project.team_id} already exists" + ) session.add(db_project) session.commit() diff --git a/carbonserver/carbonserver/api/infra/repositories/repository_teams.py b/carbonserver/carbonserver/api/infra/repositories/repository_teams.py index 9e79f24d0..53e7891ae 100644 --- a/carbonserver/carbonserver/api/infra/repositories/repository_teams.py +++ b/carbonserver/carbonserver/api/infra/repositories/repository_teams.py @@ -28,6 +28,18 @@ def add_team(self, team: TeamCreate) -> Team: api_key=generate_api_key(), organization_id=team.organization_id, ) + existing_team = ( + session.query(SqlModelTeam) + .filter(SqlModelTeam.name == team.name) + .filter(SqlModelTeam.organization_id == team.organization_id) + .first() + ) + if existing_team: + raise HTTPException( + status_code=404,detail=f"the team name {team.name} of organization {team.organization_id} already exists" + ) + + session.add(db_team) session.commit() session.refresh(db_team) diff --git a/carbonserver/carbonserver/database/scripts/spcc_purgeduplicatedata.sql b/carbonserver/carbonserver/database/scripts/spcc_purgeduplicatedata.sql new file mode 100644 index 000000000..33f2b0d1b --- /dev/null +++ b/carbonserver/carbonserver/database/scripts/spcc_purgeduplicatedata.sql @@ -0,0 +1,149 @@ +/* + spname : spcc_purgedata + goal : create a tempory table to display records where there are any information on runs and emissions + delete all records linked by their key referenced uuid from an organizations unused. + date : 20240222 + version : 01 + comment : 01 : initialize procedure spcc_purgeduplicatedata + created by : MARC ALENCON + modified by : MARC ALENCON +*/ + +/* Comments & Comments : + sample for camille organization to delete : + Delete from public.experiments + where project_id in ('6a121901-5fa6-4e37-9ad8-8ec86941feb5'); + + delete from public.projects + where team_id='d8e80b93-50f8-42fc-9280-650954415dbb'; + + delete from teams + where organization_id='92570ce9-1f90-4904-b9e6-80471963b740'; + + delete from organizations + where id ='92570ce9-1f90-4904-b9e6-80471963b740'; +*/ + +CREATE OR REPLACE PROCEDURE public.spcc_purgeduplicatedata() +LANGUAGE 'plpgsql' +AS $BODY$ + +BEGIN -- Start of transaction + +-- Création de la table temporaire +CREATE TEMP TABLE temp_table ( + nb int , + orga_id uuid , + team_id uuid , + project_id uuid , + experiment_id uuid , + run_id uuid , + emission_id uuid +); + + +-- get distinct id from tables experiments , projects , teams , organizations +-- Insertion des données de la table source vers la table temporaire +INSERT INTO temp_table (nb, orga_id,team_id,project_id,experiment_id,run_id,emission_id ) +SELECT count(*) as nb , o.id as orga_id,t.id as team_id, p.id as project_id, e.id as experiment_id , r.id as run_id , em.id as emission_id +from public.organizations o +left outer join public.teams t on o.id=t.organization_id +left outer join public.projects p on t.id = p.team_id +left outer join public.experiments e on p.id = e.project_id +left outer join public.runs r on e.id = r.experiment_id +left outer join public.emissions em on r.id = em.run_id +where r.id is null and em.id is null +group by o.id,t.id, p.id,e.id,r.id ,em.id; + + +/* +select count(*) from temp_table -- 752 +select count(*) from experiments -- 1376 / 653 +select count(*) from projects -- 41 /21 +select count(*) from teams -- 26 /15 +select count(*) from organizations --25 /12 +*/ + +DO $$ +DECLARE + row_data RECORD; + -- variables techniques . + a_count integer; + v_state TEXT; + v_msg TEXT; + v_detail TEXT; + v_hint TEXT; + v_context TEXT; + + +BEGIN + GET DIAGNOSTICS a_count = ROW_COUNT; + RAISE NOTICE '------- START -------'; + FOR row_data IN SELECT orga_id, team_id,project_id,experiment_id,run_id,emission_id FROM temp_table LOOP + + a_count = a_count +1; + + --RAISE NOTICE '------- START -------'; + RAISE NOTICE 'The rows affected by A=%',a_count; + RAISE NOTICE 'Delete experiments which contains any runs affected'; + delete FROM public.experiments e + where e.id not in ( select r.experiment_id + from runs r + ) + and e.project_id =row_data.project_id; + + + RAISE NOTICE '--------------'; + RAISE NOTICE 'Delete projects which contains any experiments affected'; + + delete FROM public.projects p + where p.id not in ( select e.project_id + from experiments e + ) + and p.team_id =row_data.team_id; + + + RAISE NOTICE '--------------'; + RAISE NOTICE 'Delete teams which contains any project affected '; + DELETE from teams t + where t.id not in (select p.team_id from projects p) + and t.organization_id =row_data.orga_id; + + + + RAISE NOTICE '--------------'; + RAISE NOTICE 'Delete organizations which contains any teams affected'; + DELETE from organizations o + where o.id not in (select t.organization_id from teams t ) + and o.id = row_data.orga_id; + + END LOOP; + RAISE NOTICE '-------- END ------'; +EXCEPTION + WHEN others THEN + + -- Handling error:Cancelled transaction when error produced + ROLLBACK; + get stacked diagnostics + v_state = returned_sqlstate, + v_msg = message_text, + v_detail = pg_exception_detail, + v_hint = pg_exception_hint, + v_context = pg_exception_context; + RAISE NOTICE E'Got exception : + state : % + message : % + detail : % + hint : % + context : % ', v_state, v_msg, v_detail, v_hint, v_context ; +END $$; + +DROP TABLE temp_table; + +COMMIT; -- end of transaction +END; +$BODY$; + + + + diff --git a/codecarbon/__init__.py b/codecarbon/__init__.py index 23a5aff39..f602f2635 100644 --- a/codecarbon/__init__.py +++ b/codecarbon/__init__.py @@ -10,3 +10,4 @@ ) __all__ = ["EmissionsTracker", "OfflineEmissionsTracker", "track_emissions"] +__app_name__ = "codecarbon" diff --git a/codecarbon/cli/cli_utils.py b/codecarbon/cli/cli_utils.py index 3539c12cf..82b58a205 100644 --- a/codecarbon/cli/cli_utils.py +++ b/codecarbon/cli/cli_utils.py @@ -1,9 +1,29 @@ import configparser from pathlib import Path +from typing import Optional +import typer +from rich.prompt import Confirm -def get_api_endpoint(): - p = Path.cwd().resolve() / ".codecarbon.config" + +def get_config(path: Optional[Path] = None): + p = path or Path.cwd().resolve() / ".codecarbon.config" + + if p.exists(): + config = configparser.ConfigParser() + config.read(str(p)) + if "codecarbon" in config.sections(): + d = dict(config["codecarbon"]) + return d + + else: + raise FileNotFoundError( + "No .codecarbon.config file found in the current directory." + ) + + +def get_api_endpoint(path: Optional[Path] = None): + p = path or Path.cwd().resolve() / ".codecarbon.config" if p.exists(): config = configparser.ConfigParser() config.read(str(p)) @@ -11,11 +31,14 @@ def get_api_endpoint(): d = dict(config["codecarbon"]) if "api_endpoint" in d: return d["api_endpoint"] + else: + with p.open("a") as f: + f.write("api_endpoint=https://api.codecarbon.io\n") return "https://api.codecarbon.io" -def get_existing_local_exp_id(): - p = Path.cwd().resolve() / ".codecarbon.config" +def get_existing_local_exp_id(path: Optional[Path] = None): + p = path or Path.cwd().resolve() / ".codecarbon.config" if p.exists(): config = configparser.ConfigParser() config.read(str(p)) @@ -25,8 +48,9 @@ def get_existing_local_exp_id(): return d["experiment_id"] -def write_local_exp_id(exp_id): - p = Path.cwd().resolve() / ".codecarbon.config" +def write_local_exp_id(exp_id, path: Optional[Path] = None): + p = path or Path.cwd().resolve() / ".codecarbon.config" + config = configparser.ConfigParser() if p.exists(): config.read(str(p)) @@ -37,3 +61,38 @@ def write_local_exp_id(exp_id): with p.open("w") as f: config.write(f) + + +def overwrite_local_config(config_name, value, path: Optional[Path] = None): + p = path or Path.cwd().resolve() / ".codecarbon.config" + + config = configparser.ConfigParser() + if p.exists(): + config.read(str(p)) + if "codecarbon" not in config.sections(): + config.add_section("codecarbon") + + config["codecarbon"][config_name] = value + with p.open("w") as f: + config.write(f) + + +def create_new_config_file(): + typer.echo("Creating new config file") + file_path = typer.prompt( + "Where do you want to put your config file ?", + type=str, + default="./.codecarbon.config", + ) + file_path = Path(file_path) + if not file_path.parent.exists(): + create = Confirm.ask( + "Parent folder does not exist do you want to create it (and parents) ?" + ) + if create: + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + with open(file_path, "w") as f: + f.write("[codecarbon]\n") + typer.echo(f"Config file created at {file_path}") + return file_path diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index 9031703f6..5f8927f10 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -1,84 +1,306 @@ -import sys import time +from pathlib import Path +from typing import Optional -import click +import questionary +import typer +from rich import print +from rich.prompt import Confirm +from typing_extensions import Annotated -from codecarbon import EmissionsTracker +from codecarbon import __app_name__, __version__ from codecarbon.cli.cli_utils import ( + create_new_config_file, get_api_endpoint, + get_config, get_existing_local_exp_id, - write_local_exp_id, + overwrite_local_config, ) from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone -from codecarbon.core.schemas import ExperimentCreate +from codecarbon.core.schemas import ( + ExperimentCreate, + OrganizationCreate, + ProjectCreate, + TeamCreate, +) +from codecarbon.emissions_tracker import EmissionsTracker DEFAULT_PROJECT_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" +DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" +codecarbon = typer.Typer() -@click.group() -def codecarbon(): - pass +def _version_callback(value: bool) -> None: + if value: + print(f"{__app_name__} v{__version__}") + raise typer.Exit() -@codecarbon.command("init", short_help="Create an experiment id in a public project.") -def init(): - experiment_id = get_existing_local_exp_id() - new_local = False - if experiment_id is None: - api = ApiClient(endpoint_url=get_api_endpoint()) - experiment = ExperimentCreate( - timestamp=get_datetime_with_timezone(), - name="Code Carbon user test", - description="Code Carbon user test with default project", - on_cloud=False, - project_id=DEFAULT_PROJECT_ID, - country_name="France", - country_iso_code="FRA", - region="france", + +@codecarbon.callback() +def main( + version: Optional[bool] = typer.Option( + None, + "--version", + "-v", + help="Show the application's version and exit.", + callback=_version_callback, + is_eager=True, + ), +) -> None: + return + + +def show_config(path: Path = Path("./.codecarbon.config")) -> None: + d = get_config(path) + api_endpoint = get_api_endpoint(path) + api = ApiClient(endpoint_url=api_endpoint) + print("Current configuration : \n") + print("Config file content : ") + print(d) + try: + if "organization_id" not in d: + print( + "No organization_id in config, follow setup instruction to complete your configuration file!", + ) + else: + org = api.get_organization(d["organization_id"]) + + if "team_id" not in d: + print( + "No team_id in config, follow setup instruction to complete your configuration file!", + ) + else: + team = api.get_team(d["team_id"]) + if "project_id" not in d: + print( + "No project_id in config, follow setup instruction to complete your configuration file!", + ) + else: + project = api.get_project(d["project_id"]) + if "experiment_id" not in d: + print( + "No experiment_id in config, follow setup instruction to complete your configuration file!", + ) + else: + experiment = api.get_experiment(d["experiment_id"]) + print("\nExperiment :") + print(experiment) + print("\nProject :") + print(project) + print("\nTeam :") + print(team) + print("\nOrganization :") + print(org) + except Exception as e: + raise ValueError( + f"Your configuration is invalid, please run `codecarbon config --init` first! (error: {e})" ) - experiment_id = api.add_experiment(experiment) - write_local_exp_id(experiment_id) - new_local = True - - click.echo( - "\nWelcome to CodeCarbon, here is your experiment id:\n" - + click.style(f"{experiment_id}", fg="bright_green") - + ( - "" - if new_local - else " (from " - + click.style("./.codecarbon.config", fg="bright_blue") - + ")\n" + + +@codecarbon.command("config", short_help="Generate or show config") +def config( + init: Annotated[ + bool, typer.Option(help="Initialise or modify configuration") + ] = None, + show: Annotated[bool, typer.Option(help="Show configuration details")] = None, +): + """ + Initialize CodeCarbon, this will prompt you for configuration of Organisation/Team/Project/Experiment. + """ + if show: + show_config() + elif init: + print("Welcome to CodeCarbon configuration wizard") + default_path = Path("./.codecarbon.config") + + if default_path.exists(): + print("Existing config file found :") + show_config(default_path) + + use_config = questionary_prompt( + "Use existing ./.codecarbon.config to configure or create a new file somwhere else ? ", + ["./.codecarbon.config", "Create New Config"], + default="./.codecarbon.config", + ) + + if use_config == "./.codecarbon.config": + modify = Confirm.ask("Do you want to modify the existing config file ?") + if modify: + print(f"Modifying existing config file {default_path}:") + file_path = default_path + else: + print(f"Using already existing config file {default_path}") + return + else: + file_path = create_new_config_file() + else: + file_path = create_new_config_file() + + api_endpoint = get_api_endpoint(file_path) + api_endpoint = typer.prompt( + f"Current API endpoint is {api_endpoint}. Press enter to continue or input other url", + type=str, + default=api_endpoint, ) - ) - if new_local: - click.echo( - "\nCodeCarbon automatically added this id to your local config: " - + click.style("./.codecarbon.config", fg="bright_blue") - + "\n" + overwrite_local_config("api_endpoint", api_endpoint, path=file_path) + api = ApiClient(endpoint_url=api_endpoint) + organizations = api.get_list_organizations() + org = questionary_prompt( + "Pick existing organization from list or Create new organization ?", + [org["name"] for org in organizations] + ["Create New Organization"], + default="Create New Organization", ) + if org == "Create New Organization": + org_name = typer.prompt( + "Organization name", default="Code Carbon user test" + ) + org_description = typer.prompt( + "Organization description", default="Code Carbon user test" + ) + if org_name in organizations: + print( + f"Organization {org_name} already exists, using it for this experiment." + ) + organization = [orga for orga in organizations if orga["name"] == org][ + 0 + ] + else: + organization_create = OrganizationCreate( + name=org_name, + description=org_description, + ) + organization = api.create_organization(organization=organization_create) + print(f"Created organization : {organization}") + else: + organization = [orga for orga in organizations if orga["name"] == org][0] + overwrite_local_config("organization_id", organization["id"], path=file_path) + teams = api.list_teams_from_organization(organization["id"]) -@codecarbon.command( - "monitor", short_help="Run an infinite loop to monitor this machine." -) -@click.option( - "--measure_power_secs", default=10, help="Interval between two measures. (10)" -) -@click.option( - "--api_call_interval", - default=30, - help="Number of measures before calling API. (30).", -) -@click.option( - "--api/--no-api", default=True, help="Choose to call Code Carbon API or not. (yes)" -) -def monitor(measure_power_secs, api_call_interval, api): + team = questionary_prompt( + "Pick existing team from list or create new team in organization ?", + [team["name"] for team in teams] + ["Create New Team"], + default="Create New Team", + ) + if team == "Create New Team": + team_name = typer.prompt("Team name", default="Code Carbon user test") + team_description = typer.prompt( + "Team description", default="Code Carbon user test" + ) + team_create = TeamCreate( + name=team_name, + description=team_description, + organization_id=organization["id"], + ) + team = api.create_team( + team=team_create, + ) + print(f"Created team : {team}") + else: + team = [t for t in teams if t["name"] == team][0] + overwrite_local_config("team_id", team["id"], path=file_path) + + projects = api.list_projects_from_team(team["id"]) + project = questionary_prompt( + "Pick existing project from list or Create new project ?", + [project["name"] for project in projects] + ["Create New Project"], + default="Create New Project", + ) + if project == "Create New Project": + project_name = typer.prompt("Project name", default="Code Carbon user test") + project_description = typer.prompt( + "Project description", default="Code Carbon user test" + ) + project_create = ProjectCreate( + name=project_name, + description=project_description, + team_id=team["id"], + ) + project = api.create_project(project=project_create) + print(f"Created project : {project}") + else: + project = [p for p in projects if p["name"] == project][0] + overwrite_local_config("project_id", project["id"], path=file_path) + + experiments = api.list_experiments_from_project(project["id"]) + experiment = questionary_prompt( + "Pick existing experiment from list or Create new experiment ?", + [experiment["name"] for experiment in experiments] + + ["Create New Experiment"], + default="Create New Experiment", + ) + if experiment == "Create New Experiment": + print("Creating new experiment") + exp_name = typer.prompt( + "Experiment name :", default="Code Carbon user test" + ) + exp_description = typer.prompt( + "Experiment description :", + default="Code Carbon user test ", + ) + + exp_on_cloud = Confirm.ask("Is this experiment running on the cloud ?") + if exp_on_cloud is True: + cloud_provider = typer.prompt( + "Cloud provider (AWS, GCP, Azure, ...)", default="AWS" + ) + cloud_region = typer.prompt( + "Cloud region (eu-west-1, us-east-1, ...)", default="eu-west-1" + ) + else: + cloud_provider = None + cloud_region = None + country_name = typer.prompt("Country name :", default="France") + country_iso_code = typer.prompt("Country ISO code :", default="FRA") + region = typer.prompt("Region :", default="france") + experiment_create = ExperimentCreate( + timestamp=get_datetime_with_timezone(), + name=exp_name, + description=exp_description, + on_cloud=exp_on_cloud, + project_id=project["id"], + country_name=country_name, + country_iso_code=country_iso_code, + region=region, + cloud_provider=cloud_provider, + cloud_region=cloud_region, + ) + experiment = api.create_experiment(experiment=experiment_create) + + else: + experiment = [e for e in experiments if e["name"] == experiment][0] + + overwrite_local_config("experiment_id", experiment["id"], path=file_path) + show_config(file_path) + print( + "Consult [link=https://mlco2.github.io/codecarbon/usage.html#configuration]configuration documentation[/link] for more configuration options" + ) + + +@codecarbon.command("monitor", short_help="Monitor your machine's carbon emissions.") +def monitor( + measure_power_secs: Annotated[ + int, typer.Argument(help="Interval between two measures.") + ] = 10, + api_call_interval: Annotated[ + int, typer.Argument(help="Number of measures between API calls.") + ] = 30, + api: Annotated[ + bool, typer.Option(help="Choose to call Code Carbon API or not") + ] = True, +): + """Monitor your machine's carbon emissions. + + Args: + measure_power_secs (Annotated[int, typer.Argument, optional): Interval between two measures. Defaults to 10. + api_call_interval (Annotated[int, typer.Argument, optional): Number of measures before calling API. Defaults to 30. + api (Annotated[bool, typer.Option, optional): Choose to call Code Carbon API or not. Defaults to True. + """ experiment_id = get_existing_local_exp_id() if api and experiment_id is None: - click.echo("ERROR: No experiment id, call 'codecarbon init' first.") - sys.exit(1) - click.echo("CodeCarbon is going in an infinite loop to monitor this machine.") + print("ERROR: No experiment id, call 'codecarbon init' first.", err=True) + print("CodeCarbon is going in an infinite loop to monitor this machine.") with EmissionsTracker( measure_power_secs=measure_power_secs, api_call_interval=api_call_interval, @@ -87,3 +309,16 @@ def monitor(measure_power_secs, api_call_interval, api): # Infinite loop while True: time.sleep(300) + + +def questionary_prompt(prompt, list_options, default): + value = questionary.select( + prompt, + list_options, + default, + ).ask() + return value + + +if __name__ == "__main__": + codecarbon() diff --git a/codecarbon/cli/test_cli_utils.py b/codecarbon/cli/test_cli_utils.py new file mode 100644 index 000000000..c67e95722 --- /dev/null +++ b/codecarbon/cli/test_cli_utils.py @@ -0,0 +1,32 @@ +from pathlib import Path +from unittest.mock import patch + +from typer.testing import CliRunner + +from codecarbon.cli.cli_utils import create_new_config_file + + +def test_create_new_config_file(): + runner = CliRunner() + + # Mock the typer.prompt function + with patch("codecarbon.cli.cli_utils.typer.prompt") as mock_prompt: + mock_prompt.return_value = "./.codecarbon.config" + + result = runner.invoke(create_new_config_file) + + assert result.exit_code == 0 + assert "Config file created at" in result.stdout + + # Verify that the prompt was called with the correct arguments + mock_prompt.assert_called_once_with( + "Where do you want to put your config file ?", + type=str, + default="./.codecarbon.config", + ) + + # Verify that the file was created + file_path = Path("./.codecarbon.config") + assert file_path.exists() + assert file_path.is_file() + assert file_path.read_text() == "[codecarbon]\n" diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 15539d8b6..db735ba2f 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -14,7 +14,14 @@ import arrow import requests -from codecarbon.core.schemas import EmissionCreate, ExperimentCreate, RunCreate +from codecarbon.core.schemas import ( + EmissionCreate, + ExperimentCreate, + OrganizationCreate, + ProjectCreate, + RunCreate, + TeamCreate, +) from codecarbon.external.logger import logger # from codecarbon.output import EmissionsData @@ -53,6 +60,108 @@ def __init__( if self.experiment_id is not None: self._create_run(self.experiment_id) + def get_list_organizations(self): + """ + List all organizations + """ + url = self.url + "/organizations" + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def create_organization(self, organization: OrganizationCreate): + """ + Create an organization + """ + payload = dataclasses.asdict(organization) + url = self.url + "/organization" + r = requests.post(url=url, json=payload, timeout=2) + if r.status_code != 201: + self._log_error(url, payload, r) + return None + return r.json() + + def get_organization(self, organization_id): + """ + Get an organization + """ + url = self.url + "/organization/" + organization_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def list_teams_from_organization(self, organization_id): + """ + List all teams + """ + url = self.url + "/teams/organization/" + organization_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def create_team(self, team: TeamCreate): + """ + Create a team + """ + payload = dataclasses.asdict(team) + url = self.url + "/team" + r = requests.post(url=url, json=payload, timeout=2) + if r.status_code != 201: + self._log_error(url, payload, r) + return None + return r.json() + + def get_team(self, team_id): + """ + Get a team + """ + url = self.url + "/team/" + team_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def list_projects_from_team(self, team_id): + """ + List all projects + """ + url = self.url + "/projects/team/" + team_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def create_project(self, project: ProjectCreate): + """ + Create a project + """ + payload = dataclasses.asdict(project) + url = self.url + "/project" + r = requests.post(url=url, json=payload, timeout=2) + if r.status_code != 201: + self._log_error(url, payload, r) + return None + return r.json() + + def get_project(self, project_id): + """ + Get a project + """ + url = self.url + "/project/" + project_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + def add_emission(self, carbon_emission: dict): assert self.experiment_id is not None self._previous_call = time.time() @@ -149,7 +258,24 @@ def _create_run(self, experiment_id): except Exception as e: logger.error(e, exc_info=True) - def add_experiment(self, experiment: ExperimentCreate): + def list_experiments_from_project(self, project_id: str): + """ + List all experiments for a project + """ + url = self.url + "/experiments/project/" + project_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() + + def set_experiment(self, experiment_id: str): + """ + Set the experiment id + """ + self.experiment_id = experiment_id + + def create_experiment(self, experiment: ExperimentCreate): """ Create an experiment, used by the CLI, not the package. ::experiment:: The experiment to create. @@ -160,13 +286,26 @@ def add_experiment(self, experiment: ExperimentCreate): if r.status_code != 201: self._log_error(url, payload, r) return None - self.experiment_id = r.json()["id"] - return self.experiment_id + return r.json() + + def get_experiment(self, experiment_id): + """ + Get an experiment by id + """ + url = self.url + "/experiment/" + experiment_id + r = requests.get(url=url, timeout=2) + if r.status_code != 200: + self._log_error(url, {}, r) + return None + return r.json() def _log_error(self, url, payload, response): - logger.error( - f"ApiClient Error when calling the API on {url} with : {json.dumps(payload)}" - ) + if len(payload) > 0: + logger.error( + f"ApiClient Error when calling the API on {url} with : {json.dumps(payload)}" + ) + else: + logger.error(f"ApiClient Error when calling the API on {url}") logger.error( f"ApiClient API return http code {response.status_code} and answer : {response.text}" ) diff --git a/codecarbon/core/schemas.py b/codecarbon/core/schemas.py index 2c6043d8d..d8969dfa1 100644 --- a/codecarbon/core/schemas.py +++ b/codecarbon/core/schemas.py @@ -79,3 +79,47 @@ class ExperimentCreate(ExperimentBase): class Experiment(ExperimentBase): id: str + + +@dataclass +class OrganizationBase: + name: str + description: str + + +class OrganizationCreate(OrganizationBase): + pass + + +class Organization(OrganizationBase): + id: str + + +@dataclass +class TeamBase: + name: str + description: str + organization_id: str + + +class TeamCreate(TeamBase): + pass + + +class Team(TeamBase): + id: str + + +@dataclass +class ProjectBase: + name: str + description: str + team_id: str + + +class ProjectCreate(ProjectBase): + pass + + +class Project(ProjectBase): + id: str diff --git a/dashboard/data/data_functions.py b/dashboard/data/data_functions.py index 25b0e7d6f..3c69be703 100644 --- a/dashboard/data/data_functions.py +++ b/dashboard/data/data_functions.py @@ -138,3 +138,8 @@ def get_project_list(organization_id) -> pd.DataFrame: projects_to_add = pd.DataFrame.from_dict(load_team_projects(i)) projects = pd.concat([projects, projects_to_add]) return projects + + +def get_team_list(organization_id) -> pd.DataFrame: + teams = pd.DataFrame.from_dict(load_organization_teams(organization_id)) + return teams diff --git a/dashboard/data/data_loader.py b/dashboard/data/data_loader.py index e19d66000..5fc07f500 100644 --- a/dashboard/data/data_loader.py +++ b/dashboard/data/data_loader.py @@ -7,12 +7,20 @@ import requests +from codecarbon.core.config import get_hierarchical_config + API_PATH = os.getenv("CODECARBON_API_URL") if API_PATH is None: - # API_PATH = "http://carbonserver.cleverapps.io" - API_PATH = "https://api.codecarbon.io" -# API_PATH = "http://localhost:8008" # export CODECARBON_API_URL=http://localhost:8008 -# API_PATH = "http://carbonserver.cleverapps.io" + conf = get_hierarchical_config() + if "api_endpoint" in conf: + API_PATH = conf["api_endpoint"] + + else: + API_PATH = "http://localhost:8008" + # API_PATH = "http://carbonserver.cleverapps.io" + # API_PATH = "https://api.codecarbon.io" + # export CODECARBON_API_URL=http://localhost:8008 + # API_PATH = "http://carbonserver.cleverapps.io" USER = "jessica" PSSD = "fake-super-secret-token" @@ -217,3 +225,4 @@ def load_run_infos(run_id: str, **kwargs) -> tuple: """ path = f"{API_PATH}/run/{run_id}" return path, kwargs + return path, kwargs diff --git a/dashboard/layout/app.py b/dashboard/layout/app.py index 97e30d36d..a43846a97 100644 --- a/dashboard/layout/app.py +++ b/dashboard/layout/app.py @@ -44,6 +44,8 @@ # Define application app = dash.Dash( __name__, + pages_folder='pages', + use_pages=True, external_stylesheets=[dbc.themes.BOOTSTRAP], meta_tags=[ {"name": "viewport", "content": "width=device-width, initial-scale=1.0"} @@ -59,304 +61,32 @@ def serve_layout(): return dbc.Container( [ - dbc.Row([components.get_header(), components.get_global_summary()]), - html.Div( - [ # hold project level information - html.Img(src=""), - dbc.Row( - dbc.Col( - [ - html.H5( - "Organization :", - ), - dcc.Dropdown( - id="org-dropdown", - options=[ - {"label": orgName, "value": orgId} - for orgName, orgId in zip( - df_org.name, df_org.id - ) - ], - clearable=False, - value=orga_id, - # value=df_org.id.unique().tolist()[0], - # value="Select your organization", - style={"color": "black"}, - # clearable=False, - ), - html.H5( - "Project :", - ), - dbc.RadioItems( - id="projectPicked", - options=[ - {"label": projectName, "value": projectId} - for projectName, projectId in zip( - df_project.name, df_project.id - ) - ], - value=( - df_project.id.unique().tolist()[-1] - if len(df_project) > 0 - else "No projects in this organization !" - ), - inline=True, - # label_checked_class_name="text-primary", - # input_checked_class_name="border border-primary bg-primary", + dbc.Row( + [ components.get_header() ] #, components.get_global_summary() ] + ), + dbc.Row([ + dbc.Navbar( + dbc.Container([ + dbc.Nav([ + dbc.NavLink(page['name'] , href=page['path'])\ + for page in dash.page_registry.values() + if not page['path'].startswith("/app") + + ]), + + + ], fluid=True ), - ], - width={"size": 6, "offset": 4}, - ) - ), - dbc.Row( - [ - # holding pieCharts - dbc.Col( - dbc.Spinner(dcc.Graph(id="pieCharts", config=config)) - ), - dbc.Col( - [ - dbc.CardGroup( - [ - components.get_household_equivalent(), - components.get_car_equivalent(), - components.get_tv_equivalent(), - ] - ), - ] - ), - ], - ), - ], - className="shadow", - ), - html.Div( # holding experiment related graph - dbc.Row( - [ - dbc.Col( - dcc.Graph(id="barChart", clickData=None, config=config) - ), # holding barChart - dbc.Col( - dbc.Spinner( - dcc.Graph( - id="bubbleChart", - clickData=None, - hoverData=None, - figure={}, - config=config, - ) - ) + #style={"border":"solid" , "border-color":"#CDCDCD"}, + dark=False, + color="#CDCDCD", + + ), ] - ), - className="shadow", - ), - html.Div( # holding run level graph - dbc.Row( - [ - # holding line chart - dbc.Col( - dbc.Spinner(dcc.Graph(id="lineChart", config=config)), - width=6, - ), - dbc.Col( - dbc.Spinner( - html.Table( - [ - html.Tr([html.Th("Metadata", colSpan=2)]), - html.Tr( - [ - html.Td("O.S."), - html.Td( - id="OS", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Python Version"), - html.Td( - id="python_version", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Number of C.P.U."), - html.Td( - id="CPU_count", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("C.P.U. model"), - html.Td( - id="CPU_model", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Number of G.P.U."), - html.Td( - id="GPU_count", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("G.P.U. model"), - html.Td( - id="GPU_model", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Longitude"), - html.Td( - id="longitude", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Latitude"), - html.Td( - id="latitude", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Region"), - html.Td( - id="region", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Provider"), - html.Td( - id="provider", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("RAM total size"), - html.Td( - id="ram_tot", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - html.Tr( - [ - html.Td("Tracking mode"), - html.Td( - id="tracking_mode", - style={ - "padding-top": "2px", - "padding-bottom": "2px", - "text-align": "right", - }, - ), - ] - ), - ] - ) - ) - ), - ] - ), - className="shadow", - ), - # holding carbon emission map - html.Br(), - dcc.Dropdown( - id="slct_kpi", - options=[ - { - "label": "Global Carbon Intensity", - "value": "Global Carbon Intensity", - }, - {"label": "My Project Emissions", "value": "My Project Emissions"}, - ], - multi=False, - value="Global Carbon Intensity", - style={"width": "50%", "color": "black"}, - clearable=False, - ), - html.Div(id="output_container", children=[]), - dcc.Graph(id="my_emission_map", figure={}, config=config), - html.Div( - [ - html.Span("Powered by "), - html.A( - "Clever Cloud", - href="https://www.clever-cloud.com/", - target="_blank", - ), - html.Span("."), - ], - className="sponsor", ), + dbc.Row( + dash.page_container + ) ] ) diff --git a/dashboard/layout/assets/calculation.png b/dashboard/layout/assets/calculation.png new file mode 100644 index 000000000..07a629f46 Binary files /dev/null and b/dashboard/layout/assets/calculation.png differ diff --git a/dashboard/layout/callbacks.py b/dashboard/layout/callbacks.py index ba202df87..536068827 100644 --- a/dashboard/layout/callbacks.py +++ b/dashboard/layout/callbacks.py @@ -2,7 +2,7 @@ import pandas as pd import plotly.express as px import plotly.graph_objects as go -from dash.dependencies import Input, Output +from dash.dependencies import Input, Output ,State from data.data_functions import ( get_experiment, get_experiment_runs, @@ -16,15 +16,169 @@ get_run_emissions, get_run_info, get_run_sums, + get_team_list, ) from layout.app import app, df_mix from layout.template import darkgreen, vividgreen from plotly.subplots import make_subplots +from dash import Input, Output,State , no_update +from dash.exceptions import PreventUpdate + + +import json +import os +import requests +import time + + +API_PATH = os.getenv("CODECARBON_API_URL") +if API_PATH is None: + #API_PATH = "https://api.codecarbon.io" + API_PATH = "http://localhost:8008" +USER = "jessica" +PSSD = "fake-super-secret-token" + + + # callback section: connecting the components # ************************************************************************ # ************************************************************************ +# @app.callback( +# Output('body-div', 'children'), +# #Output('err','children'), +# Input('show-secret', 'n_clicks'), + +# State('input_organame','value'), +# State('input_orgadesc','value'), +# #prevent_initial_call=True, +# running=[(Output("show-secret", "disabled"), True, False)] +# ) +# def update_output(n_clicks,input_organame,input_orgadesc): +# try: +# #time.sleep(5) +# if n_clicks is None: +# raise PreventUpdate +# if not input_organame and not input_orgadesc: +# return [""] +# else: +# print(input_organame) +# print(input_orgadesc) +# return [f'votre saisie est {input_organame} and {input_orgadesc} '] +# except Exception as e: +# return no_update, ''.format(e.args) + +# def checkText(input_organame): +# for char in input_organame: +# if not char.isalpha(): +# return False +# return True + +@app.callback( + Output("Orgacard_to_hidden", "style"), + Output("Output_text","style"), + Output("Output_data","children"), + Output("Teamcard_to_hidden","style"), + [Input('submit_btn','n_clicks')], + [State("input_organame", "value"), + State("input_orgadesc", "value")], + +) +def toggle_card_visibility(n_clicks,input_organame,input_orgadesc): + try: + if n_clicks is None: + raise PreventUpdate + if not input_organame or not input_orgadesc: + return {"display":"block"},{"display":"none"}, [""] ,{"display":"none"} + else: + Output_data= save_organization(input_organame,input_orgadesc) + print(Output_data) + print(n_clicks) + print(input_organame) + print(input_orgadesc) + return {"display":"none"},{"display":"block"},[Output_data],{"display":"block"} + except Exception as e: + return {"display":"block"},{"display":"block"}, e.args,{"display":"none"} + +def save_organization(input_organame, input_orgadesc) -> str: + try: + path = f"{API_PATH}/organization" + print(path) + payload = {'name': input_organame , 'description' : input_orgadesc} + response = requests.post(path, json=payload) + message = "" + if response.status_code == 201: + return f'You have entered "{input_organame}" and "{input_orgadesc}" into the database' + else: + if response.status_code == 405: + return f'You have entered "{response.status_code}" and reason : "{response.reason}" ' + else: + return f'You have entered error : "{response.status_code}" and reason : "{response.reason}" for path {path} and payload {payload}' + except: + return f'none' + +####################### + +# Refresh organizations list +# @app.callback( +# [ +# Output(component_id="dropdown-div", component_property="options"), +# Output(component_id="dropdown-div", component_property="value"), +# ], +# [Input("url-location", "pathname")], +# ) +# def refresh_org_list(url): +# df_org = get_organization_list() +# org_id = df_org.id.unique().tolist()[1] +# options = [ +# {"label": orgName, "value": orgId} +# for orgName, orgId in zip(df_org.name, df_org.id) +# ] +# return options, org_id + + + +# def update_dropdown(n_clicks, value1, value2): +# if n_clicks > 0 and value1 and value2: +# # Charger les options à partir de la base de données +# options = get_options_from_database() +# # Retourner la liste déroulante avec les options chargées +# return dcc.Dropdown(id='dropdown', options=options, placeholder='Sélectionnez une option') +# else: +# return None + + + + + +#@app.callback( +# [ +# Output(component_id="teamPicked", component_property="options"), +# # Output(component_id="projectPicked", component_property="value"), +# ], +# [ +# Input(component_id="org-dropdown", component_property="value"), +# ], +#) +#def update_team_from_organization(value): +# orga_id = value +# df_team = get_team_list(orga_id) +# if len(df_team) > 0: +# # project_id = df_project.id.unique().tolist()[0] + # project_name = df_project.name.unique().tolist()[0] + # options = [ + # {"label": teamName, "value": teamId} + # for teamName, teamId in zip(df_team.name, df_team.id) + # ] + # else: + # # project_id = None + # # project_name = "No Project !!!" + # options = [] + + # return [options] + + # indicators # ------------------------------------------------------------------------- diff --git a/dashboard/layout/components.py b/dashboard/layout/components.py index 18f31f8d4..476994d8e 100644 --- a/dashboard/layout/components.py +++ b/dashboard/layout/components.py @@ -1,5 +1,4 @@ from datetime import date, timedelta - import dash_bootstrap_components as dbc from dash import dcc, html @@ -13,6 +12,31 @@ def get_header(): [html.Img(src="/assets/logo.png")], href="https://codecarbon.io" ), html.P("Track and reduce CO2 emissions from your computing"), + # dcc.Location(id="url-location", refresh=False), + # dcc.DatePickerRange( + # id="periode", + # day_size=39, + # month_format="MMMM Y", + # end_date_placeholder_text="MMMM Y", + # display_format="DD/MM/YYYY", + # # should be calculated from today() like minus 1 week + # start_date=date(2021, 1, 1), + # min_date_allowed=date(2021, 1, 1), + # max_date_allowed=date.today() + timedelta(days=1), + # initial_visible_month=date.today(), + # end_date=date.today() + timedelta(days=1), + #), + ], + xs=12, + sm=12, + md=12, + lg=5, + xl=5, + ) +#### add + def get_daterange(self): + return dbc.Col( + [ dcc.Location(id="url-location", refresh=False), dcc.DatePickerRange( id="periode", @@ -27,13 +51,19 @@ def get_header(): initial_visible_month=date.today(), end_date=date.today() + timedelta(days=1), ), - ], - xs=12, - sm=12, - md=12, - lg=5, - xl=5, + + + ] + + ) + + + + + + +#### def get_global_summary(self): return dbc.Col( @@ -56,7 +86,7 @@ def get_global_summary(self): html.P("kWh", className="text-center"), ] ) - ], + ],style={"color":"white"} ), dbc.Card( [ @@ -76,7 +106,7 @@ def get_global_summary(self): ), ] ) - ], + ],style={"color":"white"} ), dbc.Card( [ @@ -96,7 +126,7 @@ def get_global_summary(self): ), ] ) - ], + ],style={"color":"white"} ), ], className="shadow", diff --git a/dashboard/layout/pages/admin.py b/dashboard/layout/pages/admin.py new file mode 100644 index 000000000..1a013d8aa --- /dev/null +++ b/dashboard/layout/pages/admin.py @@ -0,0 +1,173 @@ +import dash +from dash import html,dcc +import dash_bootstrap_components as dbc +from data.data_functions import get_organization_list, get_project_list, get_team_list + + +dash.register_page(__name__, path='/admin', name="Admin",order=1) + + +##################### Get Data ########################### + +df_org = get_organization_list() +orga_id = df_org.id.unique().tolist()[1] +df_project = get_project_list(orga_id) +df_team = get_team_list(orga_id) + + +##################### PAGE LAYOUT ########################### +def layout(): + return html.Div( + id="app_container", + children= + [ + html.Div( + id="sub_title", + children=[ + html.Br(), + html.H2( "Organization setting", + style={"text-align":"center"}, + + + ), + ] + + ), + + html.Br(), + dbc.Row( + [ + dbc.Col([ + dbc.Card( + id="Orgacard_to_hidden", + children= + [ + dbc.CardHeader("Formulaire",style={"color":"#CDCDCD"}), + dbc.CardBody( + children= + [ + html.H5( + "Create an Organization :", style={"color":"white"} + ), + html.Hr(), + dcc.Input(id="input_organame", type="text", placeholder="Name"), + html.Br(), + dcc.Input(id="input_orgadesc", type="text", placeholder="Description" ), + html.Br(), + html.Br(), + dbc.Button("submit", id="submit_btn", color="primary", n_clicks=0 ), + ], + style={"height": "10%", "border-radius": "10px", "border":"solid" , "border-color":"#CDCDCD" + }, + className="shadow", + #value="on" + ) + + + + ],style={"display":"block"} #make the card visible starting + ), + dbc.Card( + id="Output_text", + children= + [ + dbc.CardHeader("Result",style={"color":"#CDCDCD"} ), + dbc.CardBody( + [ + + html.Div(id='Output_data'), + + ],style={"color":"#CDCDCD"} + + ), + + + + ],style={"display":"none","color":"#CDCDCD"} + ), + + + + + ]), + + + ] + + + + ), + html.Br(), + dbc.Col([ + dbc.Card( + id="Teamcard_to_hidden", + children= + [ + + dbc.CardBody( + [ + + html.H5( + "Create a Team :", style={"color":"white"} + ), + + dbc.Label("organization selected : ",width=10 ,style={"color": "white"}, ), + #dcc.Dropdown(id='dropdown-div'), + dcc.Dropdown( + #id="dropdown-div", + options=[ + {"label": orgName, "value": orgId} + for orgName, orgId in zip( + df_org.name, df_org.id + ) + ], + clearable=False, + value=orga_id, + style={"color": "black"}, + ), + #html.Div(id='output2'), + html.Br(), + + dcc.Input(id="input_teamname", type="text", placeholder="Name", debounce=True ), + html.Br(), + dcc.Input(id="input_teamdesc", type="text", placeholder="Description" , debounce=True ), + html.Br(), + html.Br(), + dbc.Button("submit", id="submit_btn_team", color="primary", n_clicks=0), + ], + style={"height": "10%", "border-radius": "10px", "border":"solid" , "border-color":"#CDCDCD" + }, + className="shadow" + ), + ],style={"display":"none"} + ) + + ]), + + + + + + + # html.Div([ + + # dcc.Input(id="input_organame", type="text", placeholder="Name"), + # html.Br(), + # dcc.Input(id="input_orgadesc", type="text", placeholder="Description"), + # dbc.Button("submit", id="show-secret", color="primary",n_clicks=0), + # #html.Div(id='err', style={'color':'red'}), + # html.Div(id='body-div', children='Enter a value and press submit') + + # ]) + + + + + ] + + ), + + + + + diff --git a/dashboard/layout/pages/codecarbon.py b/dashboard/layout/pages/codecarbon.py new file mode 100644 index 000000000..732181eac --- /dev/null +++ b/dashboard/layout/pages/codecarbon.py @@ -0,0 +1,368 @@ +#import dash +#from dash import html +import dash +import dash_bootstrap_components as dbc +import pandas as pd +from dash import dcc, html +from data.data_functions import get_organization_list, get_project_list +from dashboard.layout.components import Components + + +dash.register_page(__name__, path='/codecarbon', name="Codecarbon", order=2) + + +# Set configuration (prevent default plotly modebar to appears, disable zoom on figures, set a double click reset ~ not working that good IMO ) +config = { + "displayModeBar": True, + "scrollZoom": False, + "doubleClick": "reset", + "displaylogo": False, + "modeBarButtonsToRemove": [ + "zoom", + "pan", + "select", + "zoomIn", + "zoomOut", + "autoScale", + "lasso2d", + ], +} + + +# App +# ******************************************************************************* +# ******************************************************************************* + + +# Get organizations ans associated projects +df_org = get_organization_list() +orga_id = df_org.id.unique().tolist()[1] +df_project = get_project_list(orga_id) + +# Load WorldElectricityMix +df_mix = pd.read_csv("./WorldElectricityMix.csv") + + + +components = Components() + + +##################### PAGE LAYOUT ########################### +layout = html.Div(children=[ + html.Br(), + html.H2("Code carbon Dashboard", style={"text-align":"center"},), + + html.Div( + [ # hold project level information + html.Img(src=""), + dbc.Row( + components.get_global_summary(), + # components.get_daterange(), + + ), + dbc.Row( + components.get_daterange(), + + ), + dbc.Row( + dbc.Col( + [ + html.Br(), + html.H5( + "Organization :", + ), + dcc.Dropdown( + id="org-dropdown", + options=[ + {"label": orgName, "value": orgId} + for orgName, orgId in zip( + df_org.name, df_org.id + ) + ], + clearable=False, + value=orga_id, + # value=df_org.id.unique().tolist()[0], + # value="Select your organization", + style={"color": "black"}, + # clearable=False, + ), + html.H5( + "Project :", + ), + dbc.RadioItems( + id="projectPicked", + options=[ + {"label": projectName, "value": projectId} + for projectName, projectId in zip( + df_project.name, df_project.id + ) + ], + value=df_project.id.unique().tolist()[-1] + if len(df_project) > 0 + else "No projects in this organization !", + inline=True, + # label_checked_class_name="text-primary", + # input_checked_class_name="border border-primary bg-primary", + ), + ], + width={"size": 6, "offset": 4}, + ) + ), + dbc.Row( + [ + # holding pieCharts + dbc.Col( + dbc.Spinner(dcc.Graph(id="pieCharts", config=config)) + ), + dbc.Col( + [ + dbc.CardGroup( + [ + components.get_household_equivalent(), + components.get_car_equivalent(), + components.get_tv_equivalent(), + ] + ), + ] + ), + ], + ), + ], + className="shadow", + ), + html.Div( # holding experiment related graph + dbc.Row( + [ + dbc.Col( + dcc.Graph(id="barChart", clickData=None, config=config) + ), # holding barChart + dbc.Col( + dbc.Spinner( + dcc.Graph( + id="bubbleChart", + clickData=None, + hoverData=None, + figure={}, + config=config, + ) + ) + ), + ] + ), + className="shadow", + ), + html.Div( # holding run level graph + dbc.Row( + [ + # holding line chart + dbc.Col( + dbc.Spinner(dcc.Graph(id="lineChart", config=config)), + width=6, + ), + dbc.Col( + dbc.Spinner( + html.Table( + [ + html.Tr([html.Th("Metadata", colSpan=2)]), + html.Tr( + [ + html.Td("O.S."), + html.Td( + id="OS", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Python Version"), + html.Td( + id="python_version", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Number of C.P.U."), + html.Td( + id="CPU_count", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("C.P.U. model"), + html.Td( + id="CPU_model", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Number of G.P.U."), + html.Td( + id="GPU_count", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("G.P.U. model"), + html.Td( + id="GPU_model", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Longitude"), + html.Td( + id="longitude", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Latitude"), + html.Td( + id="latitude", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Region"), + html.Td( + id="region", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Provider"), + html.Td( + id="provider", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("RAM total size"), + html.Td( + id="ram_tot", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + html.Tr( + [ + html.Td("Tracking mode"), + html.Td( + id="tracking_mode", + style={ + "padding-top": "2px", + "padding-bottom": "2px", + "text-align": "right", + }, + ), + ] + ), + ] + ) + ) + ), + ] + ), + className="shadow", + ), + # holding carbon emission map + html.Br(), + dcc.Dropdown( + id="slct_kpi", + options=[ + { + "label": "Global Carbon Intensity", + "value": "Global Carbon Intensity", + }, + {"label": "My Project Emissions", "value": "My Project Emissions"}, + ], + multi=False, + value="Global Carbon Intensity", + style={"width": "50%", "color": "black"}, + clearable=False, + ), + html.Div(id="output_container", children=[]), + dcc.Graph(id="my_emission_map", figure={}, config=config), + html.Div( + [ + html.Span("Powered by "), + html.A( + "Clever Cloud", + href="https://www.clever-cloud.com/", + target="_blank", + ), + html.Span("."), + ], + className="sponsor", + ), + + + + + ] + ) + + \ No newline at end of file diff --git a/dashboard/layout/pages/home.py b/dashboard/layout/pages/home.py new file mode 100644 index 000000000..4b4e46372 --- /dev/null +++ b/dashboard/layout/pages/home.py @@ -0,0 +1,75 @@ +import dash +from dash import html,dcc +import dash_bootstrap_components as dbc + +dash.register_page(__name__, path='/', name="Home", order=0) + + +from data.data_functions import get_organization_list, get_project_list, get_team_list + +##################### Get Data ########################### + +df_org = get_organization_list() +orga_id = df_org.id.unique().tolist()[1] +df_project = get_project_list(orga_id) +df_team = get_team_list(orga_id) +##################### PAGE LAYOUT ########################### +def layout(): + return html.Div(children=[ + html.Br(), + html.H2(" About CodeCarbon ", + style={"text-align":"center"}, + #className="my-1" + ), + html.Hr(), + dbc.Row( + [ + dbc.Col([ + html.H3("Description: ",style={"text-align":"center"}, ), + + ]), + dbc.Col([ + html.H6( + "We created a Python package that estimates your hardware electricity power consumption (GPU + CPU + RAM) and we apply to it the carbon intensity of the region where the computing is done." + ) + + ]) + + ], justify="center", + ), + html.Hr(), + dbc.Row([ + dbc.Col([ + dbc.Card( + [ + html.Img(src="/assets/calculation.png"), + ], + ) + + + ]) + + + + ]), + html.Hr(), + dbc.Row( + [ + dbc.Col([ + html.H6([ + "We explain more about this calculation in the ", + dcc.Link(" Methodology ", href="https://mlco2.github.io/codecarbon/methodology.html#", target="_blank"), + "section of the documentation. Our hope is that this package will be used widely for estimating the carbon footprint of computing, and for establishing best practices with regards to the disclosure and reduction of this footprint." + + ]) + + + ]) + + ], justify="center", + ), + + ] + ) + + \ No newline at end of file diff --git a/docs/_sources/usage.rst.txt b/docs/_sources/usage.rst.txt index e9dafe568..52ad93b5d 100644 --- a/docs/_sources/usage.rst.txt +++ b/docs/_sources/usage.rst.txt @@ -16,12 +16,26 @@ Command line If you want to track the emissions of a computer without having to modify your code, you can use the command line interface: +Create a minimal configuration file (just follow the prompts) .. code-block:: console + + codecarbon config --init - codecarbon monitor --no-api +Start monitoring the emissions of the computer +.. code-block:: console + + codecarbon monitor You have to stop the monitoring manually with ``Ctrl+C``. +In the following example you will see how to use the CLI to monitor all the emissions of you computer and sending everything +to an API running on localhost:8008 (that you can start with the docke-compose) +.. raw:: html + + + + + Implementing CodeCarbon in your code allows you to track the emissions of a specific block of code. Explicit Object diff --git a/docs/_static/pygments.css b/docs/_static/pygments.css index 691aeb82d..0d49244ed 100644 --- a/docs/_static/pygments.css +++ b/docs/_static/pygments.css @@ -17,6 +17,7 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #A00000 } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */ .highlight .gr { color: #FF0000 } /* Generic.Error */ .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ .highlight .gi { color: #00A000 } /* Generic.Inserted */ diff --git a/docs/api.html b/docs/api.html index 6fc0cf945..27530aec9 100644 --- a/docs/api.html +++ b/docs/api.html @@ -1,14 +1,12 @@ - + CodeCarbon API — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/comet.html b/docs/comet.html index 459117b3b..07ed9f02b 100644 --- a/docs/comet.html +++ b/docs/comet.html @@ -1,14 +1,12 @@ - + Comet Integration — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/edit/usage.rst b/docs/edit/usage.rst index e9dafe568..6510fd0fd 100644 --- a/docs/edit/usage.rst +++ b/docs/edit/usage.rst @@ -16,12 +16,28 @@ Command line If you want to track the emissions of a computer without having to modify your code, you can use the command line interface: +Create a minimal configuration file (just follow the prompts) + .. code-block:: console - codecarbon monitor --no-api + codecarbon config --init + +Start monitoring the emissions of the computer +.. code-block:: console + + codecarbon monitor You have to stop the monitoring manually with ``Ctrl+C``. +In the following example you will see how to use the CLI to monitor all the emissions of you computer and sending everything +to an API running on localhost:8008 (that you can start with the docke-compose) + +.. raw:: html + + + + + Implementing CodeCarbon in your code allows you to track the emissions of a specific block of code. Explicit Object diff --git a/docs/examples.html b/docs/examples.html index 76ebeeee2..5d4d38041 100644 --- a/docs/examples.html +++ b/docs/examples.html @@ -1,14 +1,12 @@ - + Examples — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/faq.html b/docs/faq.html index 06792d78c..1c6eb15f6 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -1,14 +1,12 @@ - + Frequently Asked Questions — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/genindex.html b/docs/genindex.html index 16e62aac7..428e18372 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -1,13 +1,11 @@ - + Index — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/index.html b/docs/index.html index ae51971dc..fe185fd00 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,14 +1,12 @@ - + CodeCarbon — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/installation.html b/docs/installation.html index 1e4c56f20..f4386d55b 100644 --- a/docs/installation.html +++ b/docs/installation.html @@ -1,14 +1,12 @@ - + Installing CodeCarbon — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/methodology.html b/docs/methodology.html index ee1212bf1..b2191b9fe 100644 --- a/docs/methodology.html +++ b/docs/methodology.html @@ -1,14 +1,12 @@ - + Methodology — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/model_examples.html b/docs/model_examples.html index ebe1fdb3b..e3d9cfc48 100644 --- a/docs/model_examples.html +++ b/docs/model_examples.html @@ -1,14 +1,12 @@ - + Model Comparisons — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/motivation.html b/docs/motivation.html index c1c16cc6a..ca88a5c96 100644 --- a/docs/motivation.html +++ b/docs/motivation.html @@ -1,14 +1,12 @@ - + Motivation — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/objects.inv b/docs/objects.inv index 0251f17b6..3ca2bac60 100644 Binary files a/docs/objects.inv and b/docs/objects.inv differ diff --git a/docs/output.html b/docs/output.html index 3c3445a75..72ac89dd3 100644 --- a/docs/output.html +++ b/docs/output.html @@ -1,14 +1,12 @@ - + Output — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/parameters.html b/docs/parameters.html index 4ead0433d..0bf6ba1b0 100644 --- a/docs/parameters.html +++ b/docs/parameters.html @@ -1,14 +1,12 @@ - + Parameters — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/search.html b/docs/search.html index 1b492a54c..bd518185a 100644 --- a/docs/search.html +++ b/docs/search.html @@ -1,13 +1,11 @@ - + Search — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/docs/usage.html b/docs/usage.html index d08332d34..2acf6cb2b 100644 --- a/docs/usage.html +++ b/docs/usage.html @@ -1,14 +1,12 @@ - + Quickstart — CodeCarbon 2.3.4 documentation - - - - + + @@ -122,10 +120,23 @@

Online Mode

Command line

If you want to track the emissions of a computer without having to modify your code, you can use the command line interface:

-
codecarbon monitor --no-api
-
-
+

Create a minimal configuration file (just follow the prompts) +.. code-block:: console

+
+

codecarbon config –init

+
+

Start monitoring the emissions of the computer +.. code-block:: console

+
+

codecarbon monitor

+

You have to stop the monitoring manually with Ctrl+C.

+

In the following example you will see how to use the CLI to monitor all the emissions of you computer and sending everything +to an API running on localhost:8008 (that you can start with the docke-compose) +.. raw:: html

+
+

<script src=”https://asciinema.org/a/bJMOlPe5F4mFLY0Rl6fiJSOp3.js” id=”asciicast-bJMOlPe5F4mFLY0Rl6fiJSOp3” async></script>

+

Implementing CodeCarbon in your code allows you to track the emissions of a specific block of code.

diff --git a/docs/visualize.html b/docs/visualize.html index 8007bd041..06c33468c 100644 --- a/docs/visualize.html +++ b/docs/visualize.html @@ -1,14 +1,12 @@ - + Visualize — CodeCarbon 2.3.4 documentation - - - - + + diff --git a/setup.py b/setup.py index 37b607519..52308f51a 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,9 @@ "py-cpuinfo", "rapidfuzz", "click", + "typer", + "questionary", + "rich", "prometheus_client", ] diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 000000000..c28c88308 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,114 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from codecarbon import __app_name__, __version__ +from codecarbon.cli.main import codecarbon + +# MOCK API CLIENT + + +@patch("codecarbon.cli.main.ApiClient") +class TestApp(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + self.mock_api_client = MagicMock() + self.mock_api_client.get_list_organizations.return_value = [ + {"id": "1", "name": "test org Code Carbon"} + ] + self.mock_api_client.list_teams_from_organization.return_value = [ + {"id": "1", "name": "test team Code Carbon"} + ] + + self.mock_api_client.list_projects_from_team.return_value = [ + {"id": "1", "name": "test project Code Carbon"} + ] + self.mock_api_client.list_experiments_from_project.return_value = [ + {"id": "1", "name": "test experiment Code Carbon"} + ] + self.mock_api_client.create_organization.return_value = { + "id": "1", + "name": "test org Code Carbon", + } + self.mock_api_client.create_team.return_value = { + "id": "1", + "name": "test team Code Carbon", + } + self.mock_api_client.create_project.return_value = { + "id": "1", + "name": "test project Code Carbon", + } + self.mock_api_client.create_experiment.return_value = { + "id": "1", + "name": "test experiment Code Carbon", + } + + def test_app(self, MockApiClient): + result = self.runner.invoke(codecarbon, ["--version"]) + self.assertEqual(result.exit_code, 0) + self.assertIn(__app_name__, result.stdout) + self.assertIn(__version__, result.stdout) + + @patch("codecarbon.cli.main.Confirm.ask") + @patch("codecarbon.cli.main.questionary_prompt") + def test_init_no_local_new_all(self, mock_prompt, mock_confirm, MockApiClient): + temp_dir = os.getenv("RUNNER_TEMP", tempfile.gettempdir()) + temp_codecarbon_config = tempfile.NamedTemporaryFile( + mode="w+t", delete=False, dir=temp_dir + ) + + MockApiClient.return_value = self.mock_api_client + mock_prompt.side_effect = [ + "Create New Organization", + "Create New Team", + "Create New Project", + "Create New Experiment", + ] + mock_confirm.side_effect = [True, False, False, False] + + result = self.runner.invoke( + codecarbon, + ["config", "--init"], + input=f"{temp_codecarbon_config.name}\n", + ) + self.assertEqual(result.exit_code, 0) + self.assertIn( + "Creating new experiment", + result.stdout, + ) + self.assertIn( + "Consult configuration documentation for more configuration options", + result.stdout, + ) + + @patch("codecarbon.cli.main.Path") + @patch("codecarbon.cli.main.questionary_prompt") + def test_init_use_local(self, mock_prompt, mock_path, MockApiClient): + temp_dir = os.getenv("RUNNER_TEMP", tempfile.gettempdir()) + + temp_codecarbon_config = tempfile.NamedTemporaryFile( + mode="w+t", delete=False, dir=temp_dir + ) + mock_path.return_value = Path(temp_codecarbon_config.name) + test_data = "[codecarbon]\nexperiment_id = 12345" + temp_codecarbon_config.write(test_data) + temp_codecarbon_config.seek(0) + mock_prompt.return_value = "./.codecarbon.config" + result = self.runner.invoke(codecarbon, ["config", "--init"], input="n") + self.assertEqual(result.exit_code, 0) + self.assertIn( + "Using already existing config file ", + result.stdout, + ) + + def custom_questionary_side_effect(*args, **kwargs): + default_value = kwargs.get("default") + return MagicMock(return_value=default_value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tox.ini b/tox.ini index 1f2c8f779..34db1f7bf 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ deps = pytest -rrequirements-dev.txt -rrequirements-test.txt +passenv = RUNNER_TEMP commands = pip install -e . @@ -32,3 +33,4 @@ python = 3.9: py39 3.10: py310 3.11: py311 +