Skip to content
This repository was archived by the owner on Sep 12, 2024. It is now read-only.

Commit f0e0a53

Browse files
author
Jinesh Shah
authored
Added support for python 3 for DIY DSC (#789)
 Summary of changes included as a part of the PR: Porting changes (Fixes) to run all scripts on python 3 Updated DSC default resources offered under nx to run on python 3. Added additional logging to all python scripts. Fixes for bugs that were introduced as a part of DROP OMI Depenedency. Description of the changes: As a part of porting the code we had decided to keep the code for python3 in a seperate forlder.(Most of these changes were done as a part of OMSConfig Py3 support PR). The updated folder for python scripts is as follows: python2 : /opt/microsoft/dsc/Scripts python3: /opt/microsoft/dsc/Scripts/python3 The python version of the machine will be determined by checking the existance of python2 and python3 command. In this version python2 takes precidence if both python 2 and 3 are present. (This is done to maintain compabality for older distros.) Instead of just logging success and failure a lot of debug and detailed logs ave been added. Going forward DSC will have two log files: /var/opt/omi/log/dsc.log /var/opt/omi/log/dscdetailed.log A lot of issues were discovered for the master branch, as the last release was done years back. I have tried to fix most of them. The issue were as a part of the DROP OMI DEPENDECY pr, which was only intended for omsconfig and ended up breaking DIY DSC. Testing: Manually tested for sanity and happy path secnarios on both python 2 and 3. Distro testing is in progress using the test suite. Bug bas planned for this week. * Initial Changes to run locally * Update InstallModule.py * Updated helper scripts to account for new paths * Updated providers to get invoked with python3 instead of python * Fix FindFirstFile Failed bug that was introduced in 2019 * Fixed path to omicli for scripts * Fixed the bug causing test to fail, it was introduced by a 2019 change DROP OMI DEPENDENCY * Update CAEngine.c * Added logs to python scripts * Updated logging to capture python major version * Added logs around compare in nxFile * Removed logging from import module to get rid of circular dependency * Removed logging from RegenerateInit to remove circular dependency * Update RegenerateInitFiles.py * Fixed comparison in case of python 3 for nxFile * Updated imports for loggig to work * Using absolute path instead of relative paths for DSCLogger * Update Makefile to respect version provided in Build Repo * Updated paths * Updated Register.py * Updated to remove warnings thrown for 'is' * Update if such that we check instance details before reginstance * Additioal logs to debug a failure in get configuration will revert the ones not required * Removed additional logging (These were just for debugging and not required to be released) * Removed additional logs * Updating content to string as OMI is not able to handle bytes * Resolving sekhar's comments
1 parent 346c0bd commit f0e0a53

36 files changed

+252
-81
lines changed

‎LCM/dsc/engine/ConsistencyInvoker/ConsistencyInvoker.c

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,15 @@ int main(int argc, char *argv[])
3131

3232
char* dscScriptPath = malloc(strlen(DSC_SCRIPT_PATH) + 1);
3333
dscScriptPath = strcpy(dscScriptPath, DSC_SCRIPT_PATH);
34-
35-
//if oms config check to be added
36-
if(strstr(dscScriptPath, "omsconfig")!= NULL)
37-
{
38-
pythonCommand = getPythonProvider();
39-
40-
if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
41-
{
42-
dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
43-
dscScriptPath = strcat(dscScriptPath, "/python3");
44-
}
45-
}
4634

35+
pythonCommand = getPythonProvider();
4736

37+
if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
38+
{
39+
dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
40+
dscScriptPath = strcat(dscScriptPath, "/python3");
41+
}
42+
4843
int fullCommandLength = strlen(pythonCommand) + 1 + strlen(dscScriptPath) + 1 + strlen(PYTHON_SCRIPT_NAME) + 1;
4944
char fullCommand[fullCommandLength];
5045

@@ -59,7 +54,6 @@ int main(int argc, char *argv[])
5954
}
6055

6156

62-
// I may need to move this method in some file which is accessible to all other files in the project.
6357
char* getPythonProvider()
6458
{
6559
int buffer_length = 128;

‎LCM/dsc/engine/ModuleLoader/ModuleLibrary/ModuleHandler.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -469,7 +469,6 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
469469
#if defined(BUILD_OMS)
470470
if (g_DscHost == MI_FALSE)
471471
{
472-
#endif
473472
// Get the Registration information from shared objects
474473
r = GetRegistrationInstanceFromSharedObjects(NULL, miApp, de, options, strictOptions, &miClassArray, &miInstanceArray, extendedError);
475474
if( r != MI_RESULT_OK)
@@ -482,6 +481,7 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
482481
CleanUpInstanceCache(&miInstanceArray);
483482
return r;
484483
}
484+
#endif
485485

486486
// No need to do this after removing OMI since we will discover resources directly from shared objects (except for DIY DSC)
487487
/*Perform registration against schema validation*/

‎LCM/dsc/engine/ca/CAInfrastructure/CAEngine.c

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,10 +1286,11 @@ MI_Result MoveToDesiredState(_In_ ProviderCallbackContext *provContext,
12861286
if (
12871287
#if defined(BUILD_OMS)
12881288
g_DscHost == MI_FALSE ||
1289+
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
12891290
#else
1291+
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
12901292
Tcscasecmp(regInstance->classDecl->name, BASE_REGISTRATION_WMIV2PROVIDER) == 0 ||
12911293
#endif
1292-
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
12931294
Tcscasecmp(MSFT_LOGRESOURCENAME, instance->classDecl->name) == 0
12941295
)
12951296
{
@@ -1910,15 +1911,16 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
19101911
*extendedError = NULL; // Explicitly set *extendedError to NULL as _Outptr_ requires setting this at least once.
19111912

19121913
*outputInstance = NULL;
1913-
19141914
/*Get the operation result*/
19151915
r = MI_Operation_GetInstance(operation, &outInstance, &moreResults, &result, &errorMessage, &completionDetails);
19161916
if( result != MI_RESULT_OK)
19171917
{
1918+
DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed result not ok");
19181919
r = result;
19191920
}
19201921
if( r != MI_RESULT_OK)
19211922
{
1923+
DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed r not ok");
19221924
if( completionDetails != NULL)
19231925
{
19241926
innerR = DSC_MI_Instance_Clone( completionDetails, extendedError);
@@ -1927,6 +1929,7 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
19271929
{
19281930
r = GetCimMIError(r, extendedError,ID_CAINFRA_GETINSTANCE_FAILED);
19291931
}
1932+
19301933
return r;
19311934
}
19321935

@@ -2021,18 +2024,21 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
20212024
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWAPPLICATIONINSTANCE_FAILED);
20222025
}
20232026
value.instance = instance;
2027+
20242028
r = DSC_MI_Instance_AddElement(params, OMI_BaseResource_Method_InputResource, &value, MI_INSTANCE, 0 );
20252029
if( r != MI_RESULT_OK)
20262030
{
20272031
MI_Instance_Delete(params);
20282032
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_ADDELEM_FAILED);
20292033
}
2034+
20302035
r = MI_Application_NewOperationOptions(miApp, MI_FALSE, &sessionOptions);
20312036
if( r != MI_RESULT_OK )
20322037
{
20332038
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWOPERATIONOPTIONS_FAILED);
20342039
}
20352040
valueOperationOptions.string=g_ConfigurationDetails.jobGuidString;
2041+
20362042
r =MI_OperationOptions_SetCustomOption(&sessionOptions,DSC_JOBIDSTRING,MI_STRING,&valueOperationOptions,MI_FALSE);
20372043
if( r != MI_RESULT_OK)
20382044
{
@@ -2041,17 +2047,16 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
20412047
}
20422048

20432049
/* Perform Get*/
2044-
20452050
MI_Session_Invoke(miSession, 0, &sessionOptions, provNamespace,
20462051
instance->classDecl->name, OMI_BaseResource_GetMethodName,
20472052
NULL, params, &callbacks,&operation);
2048-
20492053
r = GetGetMethodResult(&operation, outputInstance , extendedError);
20502054
MI_Instance_Delete(params);
20512055
MI_OperationOptions_Delete(&sessionOptions);
20522056
MI_Operation_Close(&operation);
20532057
if( r != MI_RESULT_OK)
20542058
{
2059+
DSC_LOG_INFO("GetGetMethodResult r is not ok");
20552060
return r;
20562061
}
20572062
}

‎LCM/scripts/GetDscConfiguration.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,31 @@
88
import os
99
import os.path
1010
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
11-
from imp import load_source
12-
from os.path import dirname, isfile, join, realpath
13-
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
11+
from imp import load_source
12+
from os.path import dirname, isfile, join, realpath
13+
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
14+
from sys import argv
15+
1416

1517
pathToCurrentScript = realpath(__file__)
1618
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
1719

20+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
21+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
22+
LG = nxDSCLog.DSCLog
23+
1824
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1925
helperlib = load_source('helperlib', helperLibPath)
2026

21-
omi_bindir = "<CONFIG_BINDIR>"
22-
omicli_path = omi_bindir + "/omicli"
27+
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
2328
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
2429
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
2530
dsc_host_output_path = join(dsc_host_base_path, 'output')
2631
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
2732
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
2833

34+
LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
35+
2936
if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
3037
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
3138

@@ -93,4 +100,5 @@
93100
print(stdout)
94101
print(stderr)
95102

103+
LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))
96104

‎LCM/scripts/GetDscLocalConfigurationManager.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,30 @@
88
import os
99
import os.path
1010
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
11-
from imp import load_source
12-
from os.path import dirname, isfile, join, realpath
13-
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
11+
from imp import load_source
12+
from os.path import dirname, isfile, join, realpath
13+
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
14+
from sys import argv
1415

1516
pathToCurrentScript = realpath(__file__)
1617
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
1718

19+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
20+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
21+
LG = nxDSCLog.DSCLog
22+
1823
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1924
helperlib = load_source('helperlib', helperLibPath)
2025

21-
omi_bindir = "<CONFIG_BINDIR>"
22-
omicli_path = omi_bindir + "/omicli"
26+
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
2327
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
2428
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
2529
dsc_host_output_path = join(dsc_host_base_path, 'output')
2630
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
2731
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
2832

33+
LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
34+
2935
if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
3036
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
3137

@@ -93,4 +99,5 @@
9399
print(stdout)
94100
print(stderr)
95101

102+
LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))
96103

‎LCM/scripts/InstallModule.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import subprocess
77
import sys
88
import platform
9-
from os.path import basename, dirname, join, realpath, split
9+
from os.path import basename, dirname, join, realpath, split
1010

1111
pathToCurrentScript = realpath(__file__)
1212
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
@@ -357,4 +357,4 @@ def main(args):
357357
regenerateDscPythonScriptInitFiles()
358358

359359
if __name__ == "__main__":
360-
main(sys.argv[1:])
360+
main(sys.argv[1:])

‎LCM/scripts/PerformRequiredConfigurationChecks.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/usr/bin/python
2+
import sys
23
from imp import load_source
34
from os.path import dirname, join, realpath, isfile
45
from subprocess import PIPE, Popen
5-
from sys import exc_info, exit, version_info
6+
from sys import exc_info, exit, version_info, argv
67
from traceback import format_exc
78
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
89
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
@@ -11,6 +12,10 @@
1112
pathToCurrentScript = realpath(__file__)
1213
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
1314

15+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
16+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
17+
LG = nxDSCLog.DSCLog
18+
1419
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1520
helperlib = load_source('helperlib', helperLibPath)
1621

@@ -111,4 +116,6 @@ def run_perform_required_configuration_checks():
111116
print(stdout)
112117

113118
if __name__ == "__main__":
119+
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
114120
main()
121+
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))

‎LCM/scripts/RegenerateInitFiles.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
omi_libdir + "/2.6x-2.7x/Scripts",
99
omi_libdir + "/3.x/Scripts"]
1010

11+
1112
for current_dir in script_dirs:
1213
out_init = "__all__="
1314
py_files = glob.glob(current_dir + "/*.py")
@@ -24,5 +25,4 @@
2425
py_files_basename.append(current_basename[:-3])
2526

2627
out_init = out_init + str(py_files_basename)
27-
open(current_dir + "/__init__.py", "w").write(out_init)
28-
28+
open(current_dir + "/__init__.py", "w").write(out_init)

‎LCM/scripts/Register.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
import os.path
55
import tempfile
66
import shutil
7+
from sys import argv
8+
from imp import load_source
9+
from os.path import dirname, join, realpath
710

811
def usage():
912
print("""Usage: Register.py [OPTIONS]
@@ -18,6 +21,16 @@ def usage():
1821
--Help
1922
""")
2023

24+
25+
pathToCurrentScript = realpath(__file__)
26+
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
27+
28+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
29+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
30+
LG = nxDSCLog.DSCLog
31+
32+
LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
33+
2134
# Apply a DSC meta configuration based on a template
2235
Variables = dict()
2336

@@ -217,3 +230,5 @@ def usage():
217230
os.system("<DSC_SCRIPT_PATH>/SetDscLocalConfigurationManager.py -configurationmof " + meta_path)
218231

219232
shutil.rmtree(tempdir)
233+
234+
LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))

‎LCM/scripts/RemoveModule.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,16 @@
66
import platform
77
import imp
88
from os.path import dirname, join, realpath
9+
from sys import argv
10+
from imp import load_source
911

1012
pathToCurrentScript = realpath(__file__)
1113
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
14+
15+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
16+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
17+
LG = nxDSCLog.DSCLog
18+
1219
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1320
helperlib = imp.load_source('helperlib', helperLibPath)
1421
fullPathDSCLogger = os.path.join(pathToCommonScriptsFolder, 'nxDSCLog.py')
@@ -196,4 +203,6 @@ def main(args):
196203
shutil.rmtree(modulePath)
197204

198205
if __name__ == "__main__":
206+
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
199207
main(sys.argv[1:])
208+
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))

‎LCM/scripts/RestoreConfiguration.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,28 @@
77
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
88
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
99
from time import sleep
10+
from sys import argv
11+
1012

1113
pathToCurrentScript = realpath(__file__)
1214
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
1315

16+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
17+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
18+
LG = nxDSCLog.DSCLog
19+
1420
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1521
helperlib = load_source('helperlib', helperLibPath)
1622

17-
omi_bindir = "<CONFIG_BINDIR>"
18-
omicli_path = omi_bindir + "/omicli"
23+
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
1924
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
2025
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
2126
dsc_host_output_path = join(dsc_host_base_path, 'output')
2227
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
2328
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
2429

30+
LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
31+
2532
if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
2633
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
2734

@@ -86,3 +93,5 @@
8693

8794
print(stdout)
8895
print(stderr)
96+
97+
LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))

‎LCM/scripts/SetDscLocalConfigurationManager.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,16 @@
88
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
99
from time import sleep
1010
import signal
11+
import sys
12+
1113

1214
pathToCurrentScript = realpath(__file__)
1315
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
1416

17+
DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
18+
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
19+
LG = nxDSCLog.DSCLog
20+
1521
helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
1622
helperlib = load_source('helperlib', helperLibPath)
1723

@@ -170,4 +176,6 @@ def signal_handler(signalNumber, frame):
170176
if __name__ == "__main__":
171177
# register the SIGTERM handler
172178
signal.signal(signal.SIGTERM, signal_handler)
179+
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
173180
main(argv)
181+
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))

0 commit comments

Comments
 (0)