Skip to content

Commit d10dd38

Browse files
authored
fix(init): proper floating tag support (#2459)
This change adds a `pullPolicy` configuration to better support floating tags for OCI artifacts. This variable is similar to `forceDownload` but is intended to bring behavior that is more consistent with other platforms that deal with image containers. The possible settings for `pullPolicy` are: - `Always`: install-dynamic-plugins.py will compare the image digest in the remote registry and download the artifact if it has changed, regardless if the plugin has been downloaded already. - `IfNotPresent`: install-dynamic-plugins.py will not compare image digests and instead only download the artifact if it is not present already in the dynamic-plugins-root folder. These settings have also been applied to the NPM downloading method, however `Always` will always download the remote artifact with no digest check. The existing `forceDownload` option is still functional, however this setting will take precedence. Eventually `forceDownload` may be removed. This change also fixes an issue where if only the URL is changed for an OCI artifact (say changing it from 1.0 to 1.1) the artifact is unintentionally deleted during the cleanup phase of the script. Signed-off-by: Stan Lewis <[email protected]>
1 parent 176def0 commit d10dd38

File tree

1 file changed

+89
-36
lines changed

1 file changed

+89
-36
lines changed

docker/install-dynamic-plugins.py

+89-36
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#
1515

1616
import copy
17+
from enum import StrEnum
1718
import hashlib
1819
import json
1920
import os
@@ -62,10 +63,21 @@
6263
# - merge the plugin-specific configuration fragment in a global configuration file named `app-config.dynamic-plugins.yaml`
6364
#
6465

66+
class PullPolicy(StrEnum):
67+
IF_NOT_PRESENT = 'IfNotPresent'
68+
ALWAYS = 'Always'
69+
# NEVER = 'Never' not needed
70+
6571
class InstallException(Exception):
6672
"""Exception class from which every exception in this library will derive."""
6773
pass
6874

75+
RECOGNIZED_ALGORITHMS = (
76+
'sha512',
77+
'sha384',
78+
'sha256',
79+
)
80+
6981
def merge(source, destination, prefix = ''):
7082
for key, value in source.items():
7183
if isinstance(value, dict):
@@ -81,11 +93,6 @@ def merge(source, destination, prefix = ''):
8193

8294
return destination
8395

84-
RECOGNIZED_ALGORITHMS = (
85-
'sha512',
86-
'sha384',
87-
'sha256',
88-
)
8996

9097
class OciDownloader:
9198
def __init__(self, destination: str):
@@ -99,9 +106,10 @@ def __init__(self, destination: str):
99106
self.destination = destination
100107

101108
def skopeo(self, command):
102-
rv = subprocess.run([self._skopeo] + command, check=True)
109+
rv = subprocess.run([self._skopeo] + command, check=True, capture_output=True)
103110
if rv.returncode != 0:
104111
raise InstallException(f'Error while running skopeo command: {rv.stderr}')
112+
return rv.stdout
105113

106114
def get_plugin_tar(self, image: str) -> str:
107115
if image not in self.image_to_tarball:
@@ -153,6 +161,15 @@ def download(self, package: str) -> str:
153161
shutil.rmtree(plugin_directory, ignore_errors=True, onerror=None)
154162
self.extract_plugin(tar_file=tar_file, plugin_path=plugin_path)
155163
return plugin_path
164+
165+
def digest(self, package: str) -> str:
166+
(image, plugin_path) = package.split('!')
167+
image_url = image.replace('oci://', 'docker://')
168+
output = self.skopeo(['inspect', image_url])
169+
data = json.loads(output)
170+
# OCI artifact digest field is defined as "hash method" ":" "hash"
171+
digest = data['Digest'].split(':')[1]
172+
return f"{digest}"
156173

157174
def verify_package_integrity(plugin: dict, archive: str, working_directory: str) -> None:
158175
package = plugin['package']
@@ -314,22 +331,24 @@ def main():
314331
# add a hash for each plugin configuration to detect changes
315332
for plugin in allPlugins.values():
316333
hash_dict = copy.deepcopy(plugin)
334+
# remove elements that shouldn't be tracked for installation detection
317335
hash_dict.pop('pluginConfig', None)
318336
hash = hashlib.sha256(json.dumps(hash_dict, sort_keys=True).encode('utf-8')).hexdigest()
319337
plugin['hash'] = hash
320338

321-
# create a dict installed_plugins of all installed plugins in dynamicPluginsRoot
322-
installed_plugins = {}
339+
# create a dict of all currently installed plugins in dynamicPluginsRoot
340+
plugin_path_by_hash = {}
323341
for dir_name in os.listdir(dynamicPluginsRoot):
324342
dir_path = os.path.join(dynamicPluginsRoot, dir_name)
325343
if os.path.isdir(dir_path):
326344
hash_file_path = os.path.join(dir_path, 'dynamic-plugin-config.hash')
327345
if os.path.isfile(hash_file_path):
328346
with open(hash_file_path, 'r') as hash_file:
329347
hash_value = hash_file.read().strip()
330-
installed_plugins[hash_value] = dir_name
331-
348+
plugin_path_by_hash[hash_value] = dir_name
349+
332350
oci_downloader = OciDownloader(dynamicPluginsRoot)
351+
333352
# iterate through the list of plugins
334353
for plugin in allPlugins.values():
335354
package = plugin['package']
@@ -338,28 +357,64 @@ def main():
338357
print('\n======= Skipping disabled dynamic plugin', package, flush=True)
339358
continue
340359

341-
plugin_already_installed = False
342-
if plugin['hash'] in installed_plugins:
343-
force_download = plugin.get('forceDownload', False)
344-
if force_download:
345-
print('\n======= Forcing download of already installed dynamic plugin', package, flush=True)
346-
else:
347-
print('\n======= Skipping download of already installed dynamic plugin', package, flush=True)
348-
plugin_already_installed = True
349-
# remove the hash from installed_plugins so that we can detect plugins that have been removed
350-
installed_plugins.pop(plugin['hash'])
351-
352-
if not plugin_already_installed:
353-
print('\n======= Installing dynamic plugin', package, flush=True)
354-
355-
package_is_oci = package.startswith('oci://')
360+
# Stores the relative path of the plugin directory once downloaded
356361
plugin_path = ''
357-
if package_is_oci and not plugin_already_installed:
362+
if package.startswith('oci://'):
363+
# The OCI downloader
358364
try:
365+
pull_policy = plugin.get('pullPolicy', PullPolicy.ALWAYS if ':latest!' in package else PullPolicy.IF_NOT_PRESENT)
366+
367+
if plugin['hash'] in plugin_path_by_hash and pull_policy == PullPolicy.IF_NOT_PRESENT:
368+
print('\n======= Skipping download of already installed dynamic plugin', package, flush=True)
369+
plugin_path_by_hash.pop(plugin['hash'])
370+
continue
371+
372+
if plugin['hash'] in plugin_path_by_hash and pull_policy == PullPolicy.ALWAYS:
373+
digest_file_path = os.path.join(dynamicPluginsRoot, plugin_path_by_hash.pop(plugin['hash']), 'dynamic-plugin-image.hash')
374+
local_image_digest = None
375+
if os.path.isfile(digest_file_path):
376+
with open(digest_file_path, 'r') as digest_file:
377+
digest_value = digest_file.read().strip()
378+
local_image_digest = digest_value
379+
remote_image_digest = oci_downloader.digest(package)
380+
if remote_image_digest == local_image_digest:
381+
print('\n======= Skipping download of already installed dynamic plugin', package, flush=True)
382+
continue
383+
else:
384+
print('\n======= Installing dynamic plugin', package, flush=True)
385+
386+
else:
387+
print('\n======= Installing dynamic plugin', package, flush=True)
388+
359389
plugin_path = oci_downloader.download(package)
390+
digest_file_path = os.path.join(dynamicPluginsRoot, plugin_path, 'dynamic-plugin-image.hash')
391+
with open(digest_file_path, 'w') as digest_file:
392+
digest_file.write(oci_downloader.digest(package))
393+
# remove any duplicate hashes which can occur when only the version is updated
394+
for key in [k for k, v in plugin_path_by_hash.items() if v == plugin_path]:
395+
plugin_path_by_hash.pop(key)
360396
except Exception as e:
361397
raise InstallException(f"Error while adding OCI plugin {package} to downloader: {e}")
362-
elif not plugin_already_installed:
398+
else:
399+
# The NPM downloader
400+
plugin_already_installed = False
401+
pull_policy = plugin.get('pullPolicy', PullPolicy.IF_NOT_PRESENT)
402+
403+
if plugin['hash'] in plugin_path_by_hash:
404+
force_download = plugin.get('forceDownload', False)
405+
if pull_policy == PullPolicy.ALWAYS or force_download:
406+
print('\n======= Forcing download of already installed dynamic plugin', package, flush=True)
407+
else:
408+
print('\n======= Skipping download of already installed dynamic plugin', package, flush=True)
409+
plugin_already_installed = True
410+
# remove the hash from plugin_path_by_hash so that we can detect plugins that have been removed
411+
plugin_path_by_hash.pop(plugin['hash'])
412+
else:
413+
print('\n======= Installing dynamic plugin', package, flush=True)
414+
415+
if plugin_already_installed:
416+
continue
417+
363418
package_is_local = package.startswith('./')
364419

365420
# If package is not local, then integrity check is mandatory
@@ -434,18 +489,16 @@ def main():
434489
os.remove(archive)
435490

436491
# create a hash file in the plugin directory
437-
if not plugin_already_installed:
438-
hash = plugin['hash']
439-
hash_file_path = os.path.join(dynamicPluginsRoot, plugin_path, 'dynamic-plugin-config.hash')
440-
with open(hash_file_path, 'w') as hash_file:
441-
hash_file.write(hash)
492+
hash = plugin['hash']
493+
hash_file_path = os.path.join(dynamicPluginsRoot, plugin_path, 'dynamic-plugin-config.hash')
494+
with open(hash_file_path, 'w') as digest_file:
495+
digest_file.write(hash)
442496

443497
if 'pluginConfig' not in plugin:
444498
print('\t==> Successfully installed dynamic plugin', package, flush=True)
445499
continue
446500

447501
# if some plugin configuration is defined, merge it with the global configuration
448-
449502
print('\t==> Merging plugin-specific configuration', flush=True)
450503
config = plugin['pluginConfig']
451504
if config is not None and isinstance(config, dict):
@@ -456,9 +509,9 @@ def main():
456509
yaml.safe_dump(globalConfig, open(dynamicPluginsGlobalConfigFile, 'w'))
457510

458511
# remove plugins that have been removed from the configuration
459-
for hash_value in installed_plugins:
460-
plugin_directory = os.path.join(dynamicPluginsRoot, installed_plugins[hash_value])
461-
print('\n======= Removing previously installed dynamic plugin', installed_plugins[hash_value], flush=True)
512+
for hash_value in plugin_path_by_hash:
513+
plugin_directory = os.path.join(dynamicPluginsRoot, plugin_path_by_hash[hash_value])
514+
print('\n======= Removing previously installed dynamic plugin', plugin_path_by_hash[hash_value], flush=True)
462515
shutil.rmtree(plugin_directory, ignore_errors=True, onerror=None)
463516

464517
main()

0 commit comments

Comments
 (0)