Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding HtxBootme test for Nic Devices on OpTest #880

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Adding HtxBootme test for Nic Devices on OpTest #880

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
261 changes: 246 additions & 15 deletions testcases/OpTestHtxBootme.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,7 @@ def setUp(self):
self.htx_rpm_link=self.conf.args.htx_rpm_link

self.ssh_host = OpTestSSH(self.host_ip, self.host_user, self.host_password)
self.ssh_host.set_system(self.conf.system())

self.ssh_host.set_system(self.conf.system())
path = "/usr/lpp/htx/mdt/%s" % self.mdt_file
res = self.ssh_host.run_command("if [ -f %s ];then echo 'true';else echo 'false';fi" % path)
if 'false' in res:
Expand Down Expand Up @@ -111,11 +110,12 @@ def install_latest_htx_rpm(self):
if distro_pattern in htx_rpm]
distro_specific_htx_versions.sort(reverse=True)
self.latest_htx_rpm = distro_specific_htx_versions[0]

if "error:" in self.con.run_command('rpm -ivh --nodeps %s%s '
'--force' % (self.htx_rpm_link,
self.latest_htx_rpm
), timeout=180):
cmd_wget = ('wget %s%s --no-check-certificate'
% (self.htx_rpm_link,
self.latest_htx_rpm))
cmd_install = ('rpm -ivh %s --force' % self.latest_htx_rpm)
if ("ERROR:" in self.con.run_command(cmd_wget, timeout=180) or
"error:" in self.con.run_command(cmd_install, timeout=180)):
self.fail("Installion of rpm failed")

def setup_htx(self):
Expand All @@ -132,14 +132,14 @@ def setup_htx(self):
packages.extend(['libncurses6', 'gcc-c++',
'ncurses-devel', 'tar', 'wget'])
else:
self.fail("Test not supported in %s" % host_distro_name)
self.fail("Test not supported in %s" % self.host_distro_name)
if self.host_distro_name == 'rhel':
self.installer = "yum install"
elif self.host_distro_name == 'sles':
self.installer = "zypper install"
log.debug("Installing packages")
for pkg in packages:
self.con.run_command("%s %s -y" % (self.installer, pkg))
self.con.run_command("%s -y %s" % (self.installer, pkg))

ins_htx = self.con.run_command_ignore_fail('rpm -qa | grep htx')
if ins_htx:
Expand Down Expand Up @@ -205,18 +205,19 @@ def htx_bootme_test(self):
if not self.wait_for_reboot_completion(self.cv_HOST.ip):
log.debug("Failed to confirm system reboot within the timeout period. Check the system manually.")
break
time.sleep(15)
self.con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
time.sleep(10)
for j in range(5):
cmd = 'htxcmdline -query -mdt %s' % self.mdt_file
res = self.con.run_command_ignore_fail(cmd, timeout=60)
if ("/usr/lpp/htx/mdt/mdt.all") in res[2]:
if ("/usr/lpp/htx/mdt/") in res[2]:
break
else:
time.sleep(10)
log.debug("Mdt start is still in progress")
self.con.run_command(cmd)
htxerr_file = self.ssh_host.run_command('wc -c {}'.format("/tmp/htx/htxerr"))
htxerr_file = self.con.run_command('wc -c {}'.format("/tmp/htx/htxerr"))
if int(htxerr_file[0].split()[0]) != 0:
self.fail("check error logs for exact error and failure")
log.info("Reboot cycle %s completed successfully" % (i+1))
Expand Down Expand Up @@ -284,24 +285,24 @@ def htx_stop(self):
self.suspend_all_block_device()

log.debug("shutting down the %s ", self.mdt_file)
cmd = 'htxcmdline -shutdown -mdt %s' % self.mdt_file
self.con.run_command(cmd)
cmd_shutdown = 'htxcmdline -shutdown -mdt %s' % self.mdt_file
self.con.run_command(cmd_shutdown)

cmd = '/usr/lpp/htx/etc/scripts/htx.d status'
daemon_state = self.con.run_command(cmd)
if 'running' in daemon_state[-1]:
self.con.run_command('/usr/lpp/htx/etc/scripts/htxd_shutdown')

if self.current_test_case == "HtxBootme_NicDevices":
self.ssh.run_command(cmd_shutdown)
self.ip_restore_host()
self.ip_restore_peer()

def teardown(self):
"""
close the session to the console
"""
if self.console_thread.isAlive():
self.console_thread.console_terminate()
self.console_thread.console_terminate()

class HtxBootme_AllMdt(OpTestHtxBootmeIO, unittest.TestCase):
"""
Expand Down Expand Up @@ -453,3 +454,233 @@ def get_mpath_from_dm(self, dm_id):
for mpath in mpaths:
if dm_id in mpath:
return mpath.split()[-1]

class HtxBootme_NicDevices(OpTestHtxBootmeIO, unittest.TestCase):
"""
The Test case is to run htx bootme on Network device net.mdt
"""
def setUp(self):
super(HtxBootme_NicDevices, self).setUp()

self.current_test_case="HtxBootme_NicDevices"
self.host_intfs = []
self.peer_ip = self.conf.args.peer_public_ip
self.peer_user = self.conf.args.peer_user
self.peer_password = self.conf.args.peer_password
devices = self.conf.args.htx_host_interfaces
if devices:
interfaces = self.ssh_host.run_command('ls /sys/class/net')
if interfaces:
interfaces = interfaces[0].split()
for device in devices.split(" "):
if device in interfaces:
self.host_intfs.append(device)
else:
self.fail("Please check the network device")
self.peer_intfs = self.conf.args.peer_interfaces.split(" ")
self.mdt_file = self.conf.args.mdt_file
self.query_cmd = "htxcmdline -query -mdt %s" % self.mdt_file

self.ssh = OpTestSSH(self.peer_ip, self.peer_user, self.peer_password)
self.ssh.set_system(self.conf.system())

# Flush out the ip addresses on host and peer before starting the test
self.ip_restore_host()
self.ip_restore_peer()

# Get distro details of peer lpar
self.get_peer_distro()
self.get_peer_distro_version()

def get_peer_distro(self):
"""
Get the distro name that is installed on peer lpar
"""
res = self.ssh.run_command("cat /etc/os-release")
if "Ubuntu" in res[0] or "Ubuntu" in res[1]:
self.peer_distro = "ubuntu"
elif 'Red Hat' in res[0] or 'Red Hat' in res[1]:
self.peer_distro = "rhel"
elif 'SLES' in res[0] or 'SLES' in res[1]:
self.peer_distro = "sles"
else:
self.peer_distro = "unknown"

def get_peer_distro_version(self):
"""
Get the distro version that is installed on peer lpar
"""
res = self.ssh.run_command("cat /etc/os-release")
for line in res:
if 'VERSION_ID' in line:
self.peer_distro_version = line.split('=')[1].strip('"').split('.')[0]

def update_host_peer_names(self):
"""
Update hostname & IP of both Host & Peer in /etc/hosts on both machines.
"""

res = self.ssh_host.run_command("nslookup %s" % self.host_ip)
self.host_name = re.search(r'name = (.+)\.', res[0]).group(1)
res = self.ssh.run_command("nslookup %s" % self.peer_ip)
self.peer_name = re.search(r'name = (.+)\.', res[0]).group(1)

self.hosts_file = "/etc/hosts"

log.info("Updating hostname of both Host & Peer in %s file", self.hosts_file)

self.delete_unwanted_entries()

# Update for Host
existing_entries_host = set(self.ssh_host.run_command(f"cat {self.hosts_file}"))
for ip, name in [(self.host_ip, self.host_name)]:
line = f"{ip} {name}"
if line not in existing_entries_host:
log.info("Adding missing entry on Host: %s", line)
self.ssh_host.run_command(f'echo "{line}" | sudo tee -a {self.hosts_file}')
else:
log.info("Entry exists on Host: %s", line)

# Update for Peer
existing_entries_peer = set(self.ssh.run_command(f"cat {self.hosts_file}"))
for ip, name in [(self.peer_ip, self.peer_name)]:
line = f"{ip} {name}"
if line not in existing_entries_peer:
log.info("Adding missing entry on Peer: %s", line)
self.ssh.run_command(f'echo "{line}" | sudo tee -a {self.hosts_file}')
else:
log.info("Entry exists on Peer: %s", line)

def delete_unwanted_entries(self):
"""
Deletes entries from /etc/hosts that match 'netXX.XX' pattern based on host and peer IPs.
"""
host_last_two = ".".join(self.host_ip.split(".")[-2:])
peer_last_two = ".".join(self.peer_ip.split(".")[-2:])

pattern_host = rf"net{host_last_two}"
pattern_peer = rf"net{peer_last_two}"

sed_command = f"sudo sed -i.bak -E '/{pattern_host}/d; /{pattern_peer}/d' {self.hosts_file}"

# Run on host
log.debug("Running on Host : %s" % sed_command)
self.ssh_host.run_command(sed_command)

# Run on peer
log.debug("Running on Peer : %s" % sed_command)
self.ssh.run_command(sed_command)

log.info("Deletion complete.")

def htx_configure_net(self):
"""
The function is to setup network topology for htx run
on both host and peer.
The build_net multisystem <hostname/IP> command
configures the netwrok interfaces on both host and peer Lpars with
some random net_ids and check pingum and also
starts the htx deamon for net.mdt
There is no need to explicitly start the htx deamon, create/select
and activate for net.mdt
"""
log.debug("Setting up the Network configuration on Host and Peer")

cmd = "build_net multisystem %s" % self.peer_ip

# try up to 3 times if the command fails to set the network interfaces
for i in range(3):
output = self.con.run_command(cmd, timeout=900)
if "All networks ping Ok" in output:
log.debug("Htx configuration was successful on host and peer")
break
output = self.con.run_command('pingum')
if "All networks ping Ok" not in output:
self.fail("Failed to set htx configuration on host and peer")

def start_htx_run(self):
super(HtxBootme_NicDevices, self).start_htx_run()

self.update_host_peer_names()
self.htx_configure_net()
log.debug("Running the HTX for %s on Host", self.mdt_file)
cmd = "htxcmdline -run -mdt %s" % self.mdt_file
self.con.run_command(cmd)

log.debug("Running the HTX for %s on Peer", self.mdt_file)
self.ssh.run_command(cmd)

def install_latest_htx_rpm(self):
super(HtxBootme_NicDevices, self).install_latest_htx_rpm()

if self.host_distro_name == "SuSE":
self.host_distro_name = "sles"
elif self.peer_distro == "SuSE":
self.peer_distro = "sles"
host_distro_pattern = "%s%s" % (
self.host_distro_name,
self.host_distro_version)
peer_distro_pattern = "%s%s" % (
self.peer_distro,
self.peer_distro_version)
patterns = [host_distro_pattern, peer_distro_pattern]
for pattern in patterns:
temp_string = subprocess.run(
"curl --silent %s" % (self.htx_rpm_link),
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=30)
matching_htx_versions = re.findall(
r"(?<=\>)htx\w*[-]\d*[-]\w*[.]\w*[.]\w*", str(temp_string))
distro_specific_htx_versions = [htx_rpm
for htx_rpm
in matching_htx_versions
if pattern in htx_rpm]
distro_specific_htx_versions.sort(reverse=True)
self.latest_htx_rpm = distro_specific_htx_versions[0]

cmd_wget = ('wget %s%s --no-check-certificate'
% (self.htx_rpm_link,
self.latest_htx_rpm))
cmd_install = ('rpm -ivh %s --force' % self.latest_htx_rpm)
if host_distro_pattern == peer_distro_pattern:
if ("ERROR:" in self.con.run_command(cmd_wget, timeout=180) or
"error:" in self.con.run_command(cmd_install, timeout=180)):
self.fail("Installion of rpm failed")
if ("ERROR:" in self.ssh.run_command(cmd_wget, timeout=180) or
"error:" in self.ssh.run_command(cmd_install, timeout=180)):
self.fail("Unable to install the package %s %s"
" on peer machine" % (self.htx_rpm_link,
self.latest_htx_rpm))
break

if pattern == host_distro_pattern:
if ("ERROR:" in self.con.run_command(cmd_wget, timeout=180) or
"error:" in self.con.run_command(cmd_install, timeout=180)):
self.fail("Installion of rpm failed")

if pattern == peer_distro_pattern:
if ("ERROR:" in self.ssh.run_command(cmd_wget, timeout=180) or
"error:" in self.ssh.run_command(cmd_install, timeout=180)):
self.fail("Unable to install the package %s %s"
" on peer machine" % (self.htx_rpm_link,
self.latest_htx_rpm))

def ip_restore_host(self):
'''
restoring ip for host
'''
for interface in self.host_intfs:
cmd = "ip addr flush %s" % interface
self.con.run_command(cmd)
cmd = "ip link set dev %s up" % interface
self.con.run_command(cmd)

def ip_restore_peer(self):
'''
config ip for peer
'''
for interface in self.peer_intfs:
cmd = "ip addr flush %s" % interface
self.ssh.run_command(cmd)
cmd = "ip link set dev %s up" % interface
self.ssh.run_command(cmd)