14
14
#
15
15
16
16
import copy
17
+ from enum import StrEnum
17
18
import hashlib
18
19
import json
19
20
import os
62
63
# - merge the plugin-specific configuration fragment in a global configuration file named `app-config.dynamic-plugins.yaml`
63
64
#
64
65
66
+ class PullPolicy (StrEnum ):
67
+ IF_NOT_PRESENT = 'IfNotPresent'
68
+ ALWAYS = 'Always'
69
+ # NEVER = 'Never' not needed
70
+
65
71
class InstallException (Exception ):
66
72
"""Exception class from which every exception in this library will derive."""
67
73
pass
68
74
75
+ RECOGNIZED_ALGORITHMS = (
76
+ 'sha512' ,
77
+ 'sha384' ,
78
+ 'sha256' ,
79
+ )
80
+
69
81
def merge (source , destination , prefix = '' ):
70
82
for key , value in source .items ():
71
83
if isinstance (value , dict ):
@@ -81,11 +93,6 @@ def merge(source, destination, prefix = ''):
81
93
82
94
return destination
83
95
84
- RECOGNIZED_ALGORITHMS = (
85
- 'sha512' ,
86
- 'sha384' ,
87
- 'sha256' ,
88
- )
89
96
90
97
class OciDownloader :
91
98
def __init__ (self , destination : str ):
@@ -99,9 +106,10 @@ def __init__(self, destination: str):
99
106
self .destination = destination
100
107
101
108
def skopeo (self , command ):
102
- rv = subprocess .run ([self ._skopeo ] + command , check = True )
109
+ rv = subprocess .run ([self ._skopeo ] + command , check = True , capture_output = True )
103
110
if rv .returncode != 0 :
104
111
raise InstallException (f'Error while running skopeo command: { rv .stderr } ' )
112
+ return rv .stdout
105
113
106
114
def get_plugin_tar (self , image : str ) -> str :
107
115
if image not in self .image_to_tarball :
@@ -153,6 +161,15 @@ def download(self, package: str) -> str:
153
161
shutil .rmtree (plugin_directory , ignore_errors = True , onerror = None )
154
162
self .extract_plugin (tar_file = tar_file , plugin_path = plugin_path )
155
163
return plugin_path
164
+
165
+ def digest (self , package : str ) -> str :
166
+ (image , plugin_path ) = package .split ('!' )
167
+ image_url = image .replace ('oci://' , 'docker://' )
168
+ output = self .skopeo (['inspect' , image_url ])
169
+ data = json .loads (output )
170
+ # OCI artifact digest field is defined as "hash method" ":" "hash"
171
+ digest = data ['Digest' ].split (':' )[1 ]
172
+ return f"{ digest } "
156
173
157
174
def verify_package_integrity (plugin : dict , archive : str , working_directory : str ) -> None :
158
175
package = plugin ['package' ]
@@ -314,22 +331,24 @@ def main():
314
331
# add a hash for each plugin configuration to detect changes
315
332
for plugin in allPlugins .values ():
316
333
hash_dict = copy .deepcopy (plugin )
334
+ # remove elements that shouldn't be tracked for installation detection
317
335
hash_dict .pop ('pluginConfig' , None )
318
336
hash = hashlib .sha256 (json .dumps (hash_dict , sort_keys = True ).encode ('utf-8' )).hexdigest ()
319
337
plugin ['hash' ] = hash
320
338
321
- # create a dict installed_plugins of all installed plugins in dynamicPluginsRoot
322
- installed_plugins = {}
339
+ # create a dict of all currently installed plugins in dynamicPluginsRoot
340
+ plugin_path_by_hash = {}
323
341
for dir_name in os .listdir (dynamicPluginsRoot ):
324
342
dir_path = os .path .join (dynamicPluginsRoot , dir_name )
325
343
if os .path .isdir (dir_path ):
326
344
hash_file_path = os .path .join (dir_path , 'dynamic-plugin-config.hash' )
327
345
if os .path .isfile (hash_file_path ):
328
346
with open (hash_file_path , 'r' ) as hash_file :
329
347
hash_value = hash_file .read ().strip ()
330
- installed_plugins [hash_value ] = dir_name
331
-
348
+ plugin_path_by_hash [hash_value ] = dir_name
349
+
332
350
oci_downloader = OciDownloader (dynamicPluginsRoot )
351
+
333
352
# iterate through the list of plugins
334
353
for plugin in allPlugins .values ():
335
354
package = plugin ['package' ]
@@ -338,28 +357,64 @@ def main():
338
357
print ('\n ======= Skipping disabled dynamic plugin' , package , flush = True )
339
358
continue
340
359
341
- plugin_already_installed = False
342
- if plugin ['hash' ] in installed_plugins :
343
- force_download = plugin .get ('forceDownload' , False )
344
- if force_download :
345
- print ('\n ======= Forcing download of already installed dynamic plugin' , package , flush = True )
346
- else :
347
- print ('\n ======= Skipping download of already installed dynamic plugin' , package , flush = True )
348
- plugin_already_installed = True
349
- # remove the hash from installed_plugins so that we can detect plugins that have been removed
350
- installed_plugins .pop (plugin ['hash' ])
351
-
352
- if not plugin_already_installed :
353
- print ('\n ======= Installing dynamic plugin' , package , flush = True )
354
-
355
- package_is_oci = package .startswith ('oci://' )
360
+ # Stores the relative path of the plugin directory once downloaded
356
361
plugin_path = ''
357
- if package_is_oci and not plugin_already_installed :
362
+ if package .startswith ('oci://' ):
363
+ # The OCI downloader
358
364
try :
365
+ pull_policy = plugin .get ('pullPolicy' , PullPolicy .ALWAYS if ':latest!' in package else PullPolicy .IF_NOT_PRESENT )
366
+
367
+ if plugin ['hash' ] in plugin_path_by_hash and pull_policy == PullPolicy .IF_NOT_PRESENT :
368
+ print ('\n ======= Skipping download of already installed dynamic plugin' , package , flush = True )
369
+ plugin_path_by_hash .pop (plugin ['hash' ])
370
+ continue
371
+
372
+ if plugin ['hash' ] in plugin_path_by_hash and pull_policy == PullPolicy .ALWAYS :
373
+ digest_file_path = os .path .join (dynamicPluginsRoot , plugin_path_by_hash .pop (plugin ['hash' ]), 'dynamic-plugin-image.hash' )
374
+ local_image_digest = None
375
+ if os .path .isfile (digest_file_path ):
376
+ with open (digest_file_path , 'r' ) as digest_file :
377
+ digest_value = digest_file .read ().strip ()
378
+ local_image_digest = digest_value
379
+ remote_image_digest = oci_downloader .digest (package )
380
+ if remote_image_digest == local_image_digest :
381
+ print ('\n ======= Skipping download of already installed dynamic plugin' , package , flush = True )
382
+ continue
383
+ else :
384
+ print ('\n ======= Installing dynamic plugin' , package , flush = True )
385
+
386
+ else :
387
+ print ('\n ======= Installing dynamic plugin' , package , flush = True )
388
+
359
389
plugin_path = oci_downloader .download (package )
390
+ digest_file_path = os .path .join (dynamicPluginsRoot , plugin_path , 'dynamic-plugin-image.hash' )
391
+ with open (digest_file_path , 'w' ) as digest_file :
392
+ digest_file .write (oci_downloader .digest (package ))
393
+ # remove any duplicate hashes which can occur when only the version is updated
394
+ for key in [k for k , v in plugin_path_by_hash .items () if v == plugin_path ]:
395
+ plugin_path_by_hash .pop (key )
360
396
except Exception as e :
361
397
raise InstallException (f"Error while adding OCI plugin { package } to downloader: { e } " )
362
- elif not plugin_already_installed :
398
+ else :
399
+ # The NPM downloader
400
+ plugin_already_installed = False
401
+ pull_policy = plugin .get ('pullPolicy' , PullPolicy .IF_NOT_PRESENT )
402
+
403
+ if plugin ['hash' ] in plugin_path_by_hash :
404
+ force_download = plugin .get ('forceDownload' , False )
405
+ if pull_policy == PullPolicy .ALWAYS or force_download :
406
+ print ('\n ======= Forcing download of already installed dynamic plugin' , package , flush = True )
407
+ else :
408
+ print ('\n ======= Skipping download of already installed dynamic plugin' , package , flush = True )
409
+ plugin_already_installed = True
410
+ # remove the hash from plugin_path_by_hash so that we can detect plugins that have been removed
411
+ plugin_path_by_hash .pop (plugin ['hash' ])
412
+ else :
413
+ print ('\n ======= Installing dynamic plugin' , package , flush = True )
414
+
415
+ if plugin_already_installed :
416
+ continue
417
+
363
418
package_is_local = package .startswith ('./' )
364
419
365
420
# If package is not local, then integrity check is mandatory
@@ -434,18 +489,16 @@ def main():
434
489
os .remove (archive )
435
490
436
491
# create a hash file in the plugin directory
437
- if not plugin_already_installed :
438
- hash = plugin ['hash' ]
439
- hash_file_path = os .path .join (dynamicPluginsRoot , plugin_path , 'dynamic-plugin-config.hash' )
440
- with open (hash_file_path , 'w' ) as hash_file :
441
- hash_file .write (hash )
492
+ hash = plugin ['hash' ]
493
+ hash_file_path = os .path .join (dynamicPluginsRoot , plugin_path , 'dynamic-plugin-config.hash' )
494
+ with open (hash_file_path , 'w' ) as digest_file :
495
+ digest_file .write (hash )
442
496
443
497
if 'pluginConfig' not in plugin :
444
498
print ('\t ==> Successfully installed dynamic plugin' , package , flush = True )
445
499
continue
446
500
447
501
# if some plugin configuration is defined, merge it with the global configuration
448
-
449
502
print ('\t ==> Merging plugin-specific configuration' , flush = True )
450
503
config = plugin ['pluginConfig' ]
451
504
if config is not None and isinstance (config , dict ):
@@ -456,9 +509,9 @@ def main():
456
509
yaml .safe_dump (globalConfig , open (dynamicPluginsGlobalConfigFile , 'w' ))
457
510
458
511
# remove plugins that have been removed from the configuration
459
- for hash_value in installed_plugins :
460
- plugin_directory = os .path .join (dynamicPluginsRoot , installed_plugins [hash_value ])
461
- print ('\n ======= Removing previously installed dynamic plugin' , installed_plugins [hash_value ], flush = True )
512
+ for hash_value in plugin_path_by_hash :
513
+ plugin_directory = os .path .join (dynamicPluginsRoot , plugin_path_by_hash [hash_value ])
514
+ print ('\n ======= Removing previously installed dynamic plugin' , plugin_path_by_hash [hash_value ], flush = True )
462
515
shutil .rmtree (plugin_directory , ignore_errors = True , onerror = None )
463
516
464
517
main ()
0 commit comments