diff --git a/ansible/ci/update_timestamps.yml b/ansible/ci/update_timestamps.yml new file mode 100644 index 000000000..e9a455a1e --- /dev/null +++ b/ansible/ci/update_timestamps.yml @@ -0,0 +1,16 @@ +- hosts: localhost + tasks: + - name: Get latest timestamps from sources + latest_timestamps: + repos_dict: "{{ appliances_pulp_repos }}" + content_url: "https://ark.stackhpc.com/pulp/content" + register: _result + + - name: Overwrite repo timestamps with latest + ansible.builtin.copy: + dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml" + content: "{{ repo_template | to_nice_yaml(indent=2) }}" + backup: true + vars: + repo_template: + appliances_pulp_repos: "{{ _result.timestamps }}" diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py new file mode 100644 index 000000000..6407ef049 --- /dev/null +++ b/ansible/library/latest_timestamps.py @@ -0,0 +1,76 @@ +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: latest_timestamps +short_description: Gets the latest set of snapshots from Pulp +version_added: "1.0.0" +description: Gets the latest set of snapshots from given source URLs and returns dictionary to replace 'appliances_repo_timestamps' with +author: + - William Tripp + - Steve Brasier +''' + +EXAMPLES = r''' +- name: Get latest timestamps + latest_timestamps: + repos_dict: "{{ appliances_repo_timestamp_sources }}" + content_url: "https://ark.stackhpc.com/pulp/content" + register: result +''' + +RETURN = r''' +latest_dict: + description: Dictionary with updated timestamps + type: dict + returned: always +changed_timestamps: + description: List of repos that have updated timestamps + type: str[] + returned: always +''' + +from ansible.module_utils.basic import AnsibleModule +import requests +from bs4 import BeautifulSoup + +def run_module(): + module_args = dict( + repos_dict=dict(type='dict', required=True), + content_url=dict(type='str', required=True) + ) + + result = dict( + changed=False, + original_message='', + message='' + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + timestamps = dict(module.params['repos_dict']) + for repo in timestamps: + for version in timestamps[repo]: + + html_txt = requests.get( + url= module.params['content_url'] + '/' + timestamps[repo][version]['path'] + ).text + timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html + timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags + latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed + timestamps[repo][version]['timestamp'] = latest_timestamp + + result['timestamps'] = dict(sorted(timestamps.items())) + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/environments/.stackhpc/ansible.cfg b/environments/.stackhpc/ansible.cfg index 26587e33f..6261f3149 100644 --- a/environments/.stackhpc/ansible.cfg +++ b/environments/.stackhpc/ansible.cfg @@ -10,6 +10,7 @@ inventory = ../common/inventory,inventory collections_path = ../../ansible/collections roles_path = ../../ansible/roles filter_plugins = ../../ansible/filter_plugins +library = ../../ansible/library [ssh_connection] ssh_args = -o ServerAliveInterval=10 -o ControlMaster=auto -o ControlPath=~/.ssh/%r@%h-%p -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null diff --git a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json index 01f61fa0a..ea8d46e4a 100644 --- a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json @@ -1,6 +1,6 @@ { "cluster_image": { - "RL8": "openhpc-RL8-250331-1627-cccd6c9c", - "RL9": "openhpc-RL9-250331-1627-cccd6c9c" + "RL8": "openhpc-RL8-250401-1100-9a3cffdb", + "RL9": "openhpc-RL9-250401-1100-9a3cffdb" } } diff --git a/environments/common/inventory/group_vars/all/defaults.yml b/environments/common/inventory/group_vars/all/defaults.yml index d96eed501..27a4ee0e6 100644 --- a/environments/common/inventory/group_vars/all/defaults.yml +++ b/environments/common/inventory/group_vars/all/defaults.yml @@ -98,75 +98,3 @@ appliances_extra_packages_default: appliances_extra_packages_other: [] appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}" - -###################### ark repo timestamps ################################################### - -appliances_pulp_repos: - baseos: - '9.4': - timestamp: 20241115T011711 - path: rocky/9.4/BaseOS/x86_64/os - '9.5': - timestamp: 20241216T013503 - path: rocky/9.5/BaseOS/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/BaseOS/x86_64/os - appstream: - '9.4': - timestamp: 20241112T003151 - path: rocky/9.4/AppStream/x86_64/os - '9.5': - timestamp: 20241217T005008 - path: rocky/9.5/AppStream/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/AppStream/x86_64/os - crb: - '9.4': - timestamp: 20241115T003133 - path: rocky/9.4/CRB/x86_64/os - '9.5': - timestamp: 20241217T005008 - path: rocky/9.5/CRB/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/PowerTools/x86_64/os - extras: - '9.4': - timestamp: 20241118T002802 - path: rocky/9.4/extras/x86_64/os - '9.5': - timestamp: 20241218T004632 - path: rocky/9.5/extras/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/extras/x86_64/os - epel: - '9': - timestamp: 20241213T010218 - path: epel/9/Everything/x86_64 - '8': - timestamp: 20241216T235733 - path: epel/8/Everything/x86_64 - openhpc_base: - '8': - path: OpenHPC/2/EL_8 - timestamp: 20241218T154614 - '9': - path: OpenHPC/3/EL_9 - timestamp: 20241218T154614 - openhpc_updates: - '8': - path: OpenHPC/2/updates/EL_8 - timestamp: 20241218T154614 - '9': - path: OpenHPC/3/updates/EL_9 - timestamp: 20241218T154614 - ceph: - '8': - timestamp: 20231104T015751 - path: centos/8-stream/storage/x86_64/ceph-quincy - '9': - timestamp: 20240923T233036 - path: centos/9-stream/storage/x86_64/ceph-reef diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml new file mode 100644 index 000000000..f073a9ac6 --- /dev/null +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -0,0 +1,69 @@ +appliances_pulp_repos: + appstream: + '8.10': + path: rocky/8.10/AppStream/x86_64/os + timestamp: 20250328T030013 + '9.4': + path: rocky/9.4/AppStream/x86_64/os + timestamp: 20241112T003151 + '9.5': + path: rocky/9.5/AppStream/x86_64/os + timestamp: 20250328T031822 + baseos: + '8.10': + path: rocky/8.10/BaseOS/x86_64/os + timestamp: 20250328T030013 + '9.4': + path: rocky/9.4/BaseOS/x86_64/os + timestamp: 20241115T011711 + '9.5': + path: rocky/9.5/BaseOS/x86_64/os + timestamp: 20250326T030636 + ceph: + '8': + path: centos/8-stream/storage/x86_64/ceph-quincy + timestamp: 20231104T015751 + '9': + path: centos/9-stream/storage/x86_64/ceph-reef + timestamp: 20240923T233036 + crb: + '8.10': + path: rocky/8.10/PowerTools/x86_64/os + timestamp: 20250328T030013 + '9.4': + path: rocky/9.4/CRB/x86_64/os + timestamp: 20241115T003133 + '9.5': + path: rocky/9.5/CRB/x86_64/os + timestamp: 20250325T031428 + epel: + '8': + path: epel/8/Everything/x86_64 + timestamp: 20250326T000103 + '9': + path: epel/9/Everything/x86_64 + timestamp: 20250326T000103 + extras: + '8.10': + path: rocky/8.10/extras/x86_64/os + timestamp: 20250327T030422 + '9.4': + path: rocky/9.4/extras/x86_64/os + timestamp: 20241118T002802 + '9.5': + path: rocky/9.5/extras/x86_64/os + timestamp: 20250328T031822 + openhpc_base: + '8': + path: OpenHPC/2/EL_8 + timestamp: 20241218T154614 + '9': + path: OpenHPC/3/EL_9 + timestamp: 20241218T154614 + openhpc_updates: + '8': + path: OpenHPC/2/updates/EL_8 + timestamp: 20241218T154614 + '9': + path: OpenHPC/3/updates/EL_9 + timestamp: 20241218T154614 diff --git a/requirements.txt b/requirements.txt index 872ee9516..bfb186180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ selinux # this is a shim to avoid having to use --system-site-packages, you stil netaddr matplotlib pulp-cli==0.23.2 +beautifulsoup4==4.13.3