Skip to content

Commit b04d698

Browse files
Added redisbench-admin deploy subtool (#351)
* Add the hability to override GIT_REPO and GIT_ORG * added redisbench-admin deploy subtool
1 parent 8ac448c commit b04d698

File tree

7 files changed

+191
-14
lines changed

7 files changed

+191
-14
lines changed

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redisbench-admin"
3-
version = "0.8.7"
3+
version = "0.8.8"
44
description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
55
authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
66
readme = "README.md"

redisbench_admin/cli.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
from redisbench_admin import __version__
1515
from redisbench_admin.compare.args import create_compare_arguments
1616
from redisbench_admin.compare.compare import compare_command_logic
17+
from redisbench_admin.deploy.args import create_deploy_arguments
18+
from redisbench_admin.deploy.deploy import deploy_command_logic
1719
from redisbench_admin.export.args import create_export_arguments
1820
from redisbench_admin.export.export import export_command_logic
1921
from redisbench_admin.extract.args import create_extract_arguments
@@ -91,7 +93,8 @@ def main():
9193
parser = create_grafana_api_arguments(parser)
9294
elif requested_tool == "--version":
9395
print_version(project_name, project_version)
94-
sys.exit(0)
96+
elif requested_tool == "deploy":
97+
parser = create_deploy_arguments(parser)
9598
elif requested_tool == "--help":
9699
print_help(project_name, project_version)
97100
sys.exit(0)
@@ -100,6 +103,7 @@ def main():
100103
"compare",
101104
"run-local",
102105
"run-remote",
106+
"deploy",
103107
"export",
104108
"extract",
105109
"watchdog",
@@ -150,6 +154,8 @@ def main():
150154
watchdog_command_logic(args, project_name, project_version)
151155
if requested_tool == "compare":
152156
compare_command_logic(args, project_name, project_version)
157+
if requested_tool == "deploy":
158+
deploy_command_logic(args, project_name, project_version)
153159
if requested_tool == "grafana-api":
154160
grafana_api_command_logic(args, project_name, project_version)
155161

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Apache License Version 2.0
2+
#
3+
# Copyright (c) 2021., Redis Labs Modules
4+
# All rights reserved.
5+
#

redisbench_admin/deploy/args.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Apache License Version 2.0
2+
#
3+
# Copyright (c) 2021., Redis Labs Modules
4+
# All rights reserved.
5+
#
6+
import os
7+
8+
DEFAULT_PRIVATE_KEY = "/tmp/benchmarks.redislabs.pem"
9+
TERRAFORM_BIN_PATH = os.getenv("TERRAFORM_BIN_PATH", "terraform")
10+
INFRA_TIMEOUT_SECS = int(os.getenv("INFRA_TIMEOUT_SECS", "1200"))
11+
GIT_ORG = os.getenv("GIT_ORG", None)
12+
GIT_REPO = os.getenv("GIT_REPO", None)
13+
14+
15+
def create_deploy_arguments(parser):
16+
parser.add_argument(
17+
"--private_key",
18+
required=False,
19+
default=DEFAULT_PRIVATE_KEY,
20+
type=str,
21+
help="Use this key for ssh connections.",
22+
)
23+
parser.add_argument("--terraform_bin_path", type=str, default=TERRAFORM_BIN_PATH)
24+
parser.add_argument("--infra_timeout_secs", type=int, default=INFRA_TIMEOUT_SECS)
25+
parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
26+
parser.add_argument("--github_repo", type=str, default=GIT_REPO)
27+
parser.add_argument("--github_org", type=str, default=GIT_ORG)
28+
parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
29+
parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
30+
parser.add_argument("--inventory_git", required=True, type=str)
31+
parser.add_argument("--inventory_local_dir", default=None, type=str)
32+
parser.add_argument("--set_env_vars_json", default="", type=str)
33+
parser.add_argument("--setup_name_sufix", type=str, default="")
34+
parser.add_argument(
35+
"--destroy", help="destroy the current env", action="store_true"
36+
)
37+
return parser

redisbench_admin/deploy/deploy.py

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Apache License Version 2.0
2+
#
3+
# Copyright (c) 2021., Redis Labs Modules
4+
# All rights reserved.
5+
#
6+
import json
7+
import logging
8+
import os
9+
10+
from redisbench_admin.run.git import git_vars_crosscheck
11+
from redisbench_admin.run.ssh import ssh_pem_check
12+
from redisbench_admin.utils.remote import (
13+
fetch_remote_setup_git_url,
14+
setup_remote_environment,
15+
)
16+
from python_terraform import Terraform
17+
18+
from redisbench_admin.utils.utils import EC2_PRIVATE_PEM
19+
20+
21+
def deploy_command_logic(args, project_name, project_version):
22+
logging.info(
23+
"Using: {project_name} {project_version}".format(
24+
project_name=project_name, project_version=project_version
25+
)
26+
)
27+
terraform_bin_path = args.terraform_bin_path
28+
tf_github_org = args.github_org
29+
tf_github_actor = args.github_actor
30+
tf_github_repo = args.github_repo
31+
tf_github_sha = args.github_sha
32+
tf_github_branch = args.github_branch
33+
infra_timeout_secs = args.infra_timeout_secs
34+
35+
(
36+
tf_github_actor,
37+
tf_github_branch,
38+
tf_github_org,
39+
tf_github_repo,
40+
tf_github_sha,
41+
) = git_vars_crosscheck(
42+
tf_github_actor, tf_github_branch, tf_github_org, tf_github_repo, tf_github_sha
43+
)
44+
45+
private_key = args.private_key
46+
ssh_pem_check(EC2_PRIVATE_PEM, private_key)
47+
inventory_git = args.inventory_git
48+
inventory_local_dir = args.inventory_local_dir
49+
destroy = args.destroy
50+
if inventory_local_dir is not None:
51+
if os.path.isdir(inventory_local_dir) is False:
52+
os.mkdir(inventory_local_dir)
53+
(
54+
remote_setup,
55+
deployment_type,
56+
remote_id,
57+
) = fetch_remote_setup_git_url(inventory_git, inventory_local_dir, destroy)
58+
tf = Terraform(
59+
working_dir=remote_setup,
60+
terraform_bin_path=terraform_bin_path,
61+
)
62+
tf_setup_name_sufix = "{}-{}".format(args.setup_name_sufix, tf_github_sha)
63+
tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
64+
terraform_backend_key = "benchmarks/infrastructure/{}.tfstate".format(tf_setup_name)
65+
tf_triggering_env = "redisbench-admin-deploy"
66+
logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
67+
if args.destroy is False:
68+
(tf_return_code, _, _, _, _, _, _,) = setup_remote_environment(
69+
tf,
70+
tf_github_sha,
71+
tf_github_actor,
72+
tf_setup_name,
73+
tf_github_org,
74+
tf_github_repo,
75+
tf_triggering_env,
76+
infra_timeout_secs,
77+
)
78+
logging.info("Deploy terraform return code {}".format(tf_return_code))
79+
env_json = tf.output()
80+
output_dict = {}
81+
for k, v in env_json.items():
82+
k = k.upper()
83+
if type(v["value"]) == list:
84+
output_dict[k] = v["value"][0]
85+
else:
86+
output_dict[k] = v["value"]
87+
88+
logging.info("JSON env variables {}".format(output_dict))
89+
if args.set_env_vars_json != "":
90+
with open(args.set_env_vars_json, "w") as json_fd:
91+
json.dump(output_dict, json_fd)
92+
else:
93+
_, _, _ = tf.init(
94+
capture_output=True,
95+
backend_config={"key": terraform_backend_key},
96+
)
97+
logging.info("Refreshing remote state")
98+
_, _, _ = tf.refresh()
99+
logging.info("Triggering destroy")
100+
output = tf.destroy(capture_output=True)
101+
logging.info("Finished destroying the remote env. Output:")
102+
for line in output[1].split("\n"):
103+
print(line)

redisbench_admin/run/args.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,17 +87,17 @@ def common_run_args(parser):
8787
default="defaults.yml",
8888
help="specify the defaults file containing spec topologies, common metric extractions,etc...",
8989
)
90-
parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
91-
parser.add_argument("--github_repo", type=str, default=GIT_REPO)
92-
parser.add_argument("--github_org", type=str, default=GIT_ORG)
93-
parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
9490
parser.add_argument(
9591
"--required-module",
9692
default=None,
9793
action="append",
9894
help="path to the module file. "
9995
"You can use `--required-module` more than once",
10096
)
97+
parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
98+
parser.add_argument("--github_repo", type=str, default=GIT_REPO)
99+
parser.add_argument("--github_org", type=str, default=GIT_ORG)
100+
parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
101101
parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
102102
parser.add_argument("--triggering_env", type=str, default=TRIGGERING_ENV)
103103
parser.add_argument(

redisbench_admin/utils/remote.py

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,39 @@ def fetch_remote_id_from_config(
504504
return setup
505505

506506

507+
def fetch_remote_setup_git_url(git_url, temporary_dir=None, destroy=False):
508+
setup_type = None
509+
setup = None
510+
branch = "master"
511+
path = "/"
512+
513+
tree_pos = git_url.find("tree")
514+
if tree_pos > 0:
515+
repo = git_url[: tree_pos - 1]
516+
remaining = git_url[tree_pos + 5 :].split("/")
517+
if len(remaining) > 0:
518+
branch = remaining[0]
519+
if len(remaining) > 1:
520+
path = path + "/".join(remaining[1:])
521+
522+
terraform_working_dir = common_tf(branch, path, repo, temporary_dir, destroy)
523+
return terraform_working_dir, setup_type, setup
524+
525+
526+
def common_tf(branch, path, repo, temporary_dir=None, destroy=False):
527+
if temporary_dir is None:
528+
temporary_dir = tempfile.mkdtemp()
529+
if destroy is False:
530+
logging.info(
531+
"Fetching infrastructure definition from git repo {}/{} (branch={}). Using local dir {} to store state".format(
532+
repo, path, branch, temporary_dir
533+
)
534+
)
535+
git.Repo.clone_from(repo, temporary_dir, branch=branch, depth=1)
536+
terraform_working_dir = temporary_dir + path
537+
return terraform_working_dir
538+
539+
507540
def fetch_remote_setup_from_config(
508541
remote_setup_config,
509542
repo="https://github.com/RedisLabsModules/testing-infrastructure.git",
@@ -518,14 +551,7 @@ def fetch_remote_setup_from_config(
518551
setup = remote_setup_property["setup"]
519552
# fetch terraform folder
520553
path = "/terraform/{}-{}".format(setup_type, setup)
521-
temporary_dir = tempfile.mkdtemp()
522-
logging.info(
523-
"Fetching infrastructure definition from git repo {}/{} (branch={})".format(
524-
repo, path, branch
525-
)
526-
)
527-
git.Repo.clone_from(repo, temporary_dir, branch=branch, depth=1)
528-
terraform_working_dir = temporary_dir + path
554+
terraform_working_dir = common_tf(branch, path, repo)
529555
return terraform_working_dir, setup_type, setup
530556

531557

0 commit comments

Comments
 (0)