diff --git a/.editorconfig b/.editorconfig index 63a79b9250..a50a0c0691 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,6 +17,46 @@ indent_size = 2 [*.{cs,vb}] trim_trailing_whitespace = true +# Naming styles + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case + +dotnet_naming_style.begins_with_i.required_prefix = I +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.capitalization = pascal_case + +# Naming rules + +dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface +dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i + +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case + +# Symbol specifications + +dotnet_naming_symbols.interface.applicable_kinds = interface +dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.interface.required_modifiers = + +dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum +dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.types.required_modifiers = + +dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.non_field_members.required_modifiers = + # --- # naming conventions https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-naming-conventions # currently not supported in Rider/Resharper so not using these for now @@ -103,80 +143,183 @@ dotnet_naming_symbols.public_members.applicable_accessibilities = public dotnet_naming_style.public_members.capitalization = pascal_case +# this. and Me. preferences +dotnet_style_qualification_for_event = false:error dotnet_style_qualification_for_field = false:error -dotnet_style_qualification_for_property = false:error dotnet_style_qualification_for_method = false:error -dotnet_style_qualification_for_event = false:error +dotnet_style_qualification_for_property = false:error # Use language keywords instead of framework type names for type references dotnet_style_predefined_type_for_locals_parameters_members = true:error dotnet_style_predefined_type_for_member_access = true:error +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_operators = never_if_unnecessary +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members:error +dotnet_style_readonly_field = true:error + # Suggest more modern language features when available -dotnet_style_object_initializer = true:error +# Expression-level preferences +dotnet_style_coalesce_expression = true:error dotnet_style_collection_initializer = true:error dotnet_style_explicit_tuple_names = true:error +dotnet_style_namespace_match_folder = true +dotnet_style_null_propagation = true:error +dotnet_style_object_initializer = true:error # our setting:true:error. Set to false if performing a dotnet format +dotnet_style_operator_placement_when_wrapping = beginning_of_line +dotnet_style_prefer_auto_properties = true +dotnet_style_prefer_collection_expression = true +dotnet_style_prefer_compound_assignment = true +dotnet_style_prefer_conditional_expression_over_assignment = true +dotnet_style_prefer_conditional_expression_over_return = true +dotnet_style_prefer_foreach_explicit_cast_in_source = when_strongly_typed dotnet_style_prefer_inferred_anonymous_type_member_names = true:error dotnet_style_prefer_inferred_tuple_names = true:error -dotnet_style_coalesce_expression = true:error -dotnet_style_null_propagation = true:error +dotnet_style_prefer_is_null_check_over_reference_equality_method = true +dotnet_style_prefer_simplified_boolean_expressions = true +dotnet_style_prefer_simplified_interpolation = true -dotnet_style_require_accessibility_modifiers = for_non_interface_members:error -dotnet_style_readonly_field = true:error +# Field preferences +dotnet_style_readonly_field = true + +# Parameter preferences +dotnet_code_quality_unused_parameters = all + +# Suppression preferences +dotnet_remove_unnecessary_suppression_exclusions = none + +# New line preferences +dotnet_style_allow_multiple_blank_lines_experimental = true +dotnet_style_allow_statement_immediately_after_block_experimental = true + +#### C# Coding Conventions #### -# CSharp code style settings: [*.cs] # Prefer "var" everywhere +csharp_style_var_elsewhere = true:error csharp_style_var_for_built_in_types = true:error csharp_style_var_when_type_is_apparent = true:error -csharp_style_var_elsewhere = true:error -csharp_style_expression_bodied_methods = true:error +csharp_style_expression_bodied_accessors = true:error csharp_style_expression_bodied_constructors = true:error +csharp_style_expression_bodied_indexers = true:error +csharp_style_expression_bodied_lambdas = true +csharp_style_expression_bodied_local_functions = false +csharp_style_expression_bodied_methods = true:error csharp_style_expression_bodied_operators = true:error csharp_style_expression_bodied_properties = true:error -csharp_style_expression_bodied_indexers = true:error -csharp_style_expression_bodied_accessors = true:error # Suggest more modern language features when available -csharp_style_pattern_matching_over_is_with_cast_check = true:error -csharp_style_pattern_matching_over_as_with_null_check = true:error -csharp_style_inlined_variable_declaration = true:error -csharp_style_deconstructed_variable_declaration = true:error + csharp_style_pattern_local_over_anonymous_function = true:error -csharp_style_throw_expression = true:error +csharp_style_pattern_matching_over_as_with_null_check = true:error +csharp_style_pattern_matching_over_is_with_cast_check = true:error +csharp_style_prefer_extended_property_pattern = true +csharp_style_prefer_not_pattern = true +csharp_style_prefer_pattern_matching = true +csharp_style_prefer_switch_expression = true + +# Null-checking preferences csharp_style_conditional_delegate_call = true:error -csharp_prefer_braces = false:warning +# Modifier preferences +csharp_prefer_static_local_function = true csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:error +csharp_style_prefer_readonly_struct = true +csharp_style_prefer_readonly_struct_member = true -# --- -# formatting conventions https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-code-style-settings-reference#formatting-conventions +# Code-block preferences +csharp_prefer_braces = false:warning +csharp_prefer_simple_using_statement = true +csharp_style_namespace_declarations = file_scoped:error #This allows for no { } for namespaces +csharp_style_prefer_method_group_conversion = true +csharp_style_prefer_primary_constructors = false:warning +csharp_style_prefer_top_level_statements = true + +# Expression-level preferences +csharp_prefer_simple_default_expression = true +csharp_style_deconstructed_variable_declaration = true:error +csharp_style_implicit_object_creation_when_type_is_apparent = true +csharp_style_inlined_variable_declaration = true:error # our setting:true:error. Set to false if performing a dotnet format +csharp_style_prefer_index_operator = true +csharp_style_prefer_local_over_anonymous_function = true +csharp_style_prefer_null_check_over_type_check = true +csharp_style_prefer_range_operator = true +csharp_style_prefer_tuple_swap = true +csharp_style_prefer_utf8_string_literals = true +csharp_style_throw_expression = true:error +csharp_style_unused_value_assignment_preference = discard_variable +csharp_style_unused_value_expression_statement_preference = discard_variable + +# 'using' directive preferences +# csharp_using_directive_placement = outside_namespace + +# New line preferences +csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true +csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true +csharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = true +csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true +csharp_style_allow_embedded_statements_on_same_line_experimental = true # Newline settings (Allman yo!) -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true + csharp_new_line_before_catch = true +csharp_new_line_before_else = true csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true # just a suggestion do to our JSON tests that use anonymous types to # represent json quite a bit (makes copy paste easier). csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_open_brace = all csharp_new_line_between_query_expression_clauses = true -# Indent +# Indentation preferences https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/csharp-formatting-options +csharp_indent_block_contents = true +csharp_indent_braces = false csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = one_less_than_current # our setting:one_less_than_current. Change to 'no_change' if running dotnet format to keep the current formatting https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/csharp-formatting-options#csharp_indent_labels csharp_indent_switch_labels = true + +# Space preferences csharp_space_after_cast = false +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_around_binary_operators = before_and_after +csharp_space_around_declaration_statements = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_between_square_brackets = false -#Wrap -csharp_preserve_single_line_statements = false +# Wrapping preferences csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = true #our setting:false. Set to true if performing a dotnet format https://stackoverflow.com/questions/49760158/visual-studio-2017-auto-formatting-single-line-if-statements-without-braces # Resharper + +resharper_check_namespace_highlighting=do_not_show + +resharper_csharp_accessor_owner_body=expression_body + resharper_csharp_braces_for_lock=required_for_multiline resharper_csharp_braces_for_using=required_for_multiline resharper_csharp_braces_for_while=required_for_multiline @@ -185,11 +328,18 @@ resharper_csharp_braces_for_for=required_for_multiline resharper_csharp_braces_for_fixed=required_for_multiline resharper_csharp_braces_for_ifelse=required_for_multiline -resharper_csharp_accessor_owner_body=expression_body +resharper_inheritdoc_invalid_usage_highlighting=do_not_show resharper_redundant_case_label_highlighting=do_not_show +resharper_redundant_anonymous_type_property_name_highlighting=do_not_show resharper_redundant_argument_default_value_highlighting=do_not_show +resharper_unused_member_local_highlighting=do_not_show +resharper_unused_parameter_local_highlighting=do_not_show +resharper_unused_variable_highlighting=do_not_show + +resharper_xUnit_1013_highlighting=do_not_show + [*.{sh,bat,ps1}] trim_trailing_whitespace=true insert_final_newline=true diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs index 9f69796b96..a30d25e07f 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs @@ -26,36 +26,35 @@ * under the License. */ -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +/// +/// Authentication credentials for the cluster +/// +public class ClusterAuthentication { - /// - /// Authentication credentials for the cluster - /// - public class ClusterAuthentication - { - /// - /// Administrator credentials - /// - public static Credentials Admin => new Credentials {Username = "admin", Role = "admin"}; + /// + /// Administrator credentials + /// + public static Credentials Admin => new Credentials { Username = "admin", Role = "admin" }; - /// - /// User credentials - /// - public static Credentials User => new Credentials {Username = "admin", Role = "admin"}; + /// + /// User credentials + /// + public static Credentials User => new Credentials { Username = "admin", Role = "admin" }; - /// - /// Credentials for all users - /// - public static Credentials[] AllUsers { get; } = {Admin, User}; + /// + /// Credentials for all users + /// + public static Credentials[] AllUsers { get; } = { Admin, User }; - /// - /// Authentication credentials - /// - public class Credentials - { - public string Username { get; set; } - public string Role { get; set; } - public string Password => Username; - } - } + /// + /// Authentication credentials + /// + public class Credentials + { + public string Username { get; set; } + public string Role { get; set; } + public string Password => Username; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs index ce36b0d436..89a433f4c2 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs @@ -28,23 +28,22 @@ using System; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +/// +/// Hints to what features the cluster to be started should have. +/// It's up to the to actually bootstrap these features. +/// +[Flags] +public enum ClusterFeatures { - /// - /// Hints to what features the cluster to be started should have. - /// It's up to the to actually bootstrap these features. - /// - [Flags] - public enum ClusterFeatures - { - /// - /// No features - /// - None = 1 << 0, + /// + /// No features + /// + None = 1 << 0, - /// - /// SSL/TLS for HTTP and Transport layers - /// - SSL = 1 << 3, - } + /// + /// SSL/TLS for HTTP and Transport layers + /// + SSL = 1 << 3, } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index f0e3769c96..357194b0b1 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -37,105 +37,104 @@ using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public class EphemeralCluster : EphemeralCluster +{ + public EphemeralCluster(OpenSearchVersion version, int numberOfNodes = 1) + : base(new EphemeralClusterConfiguration(version, ClusterFeatures.None, numberOfNodes: numberOfNodes)) + { + } + + public EphemeralCluster(EphemeralClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } +} + +public abstract class EphemeralCluster : ClusterBase, + IEphemeralCluster + where TConfiguration : EphemeralClusterConfiguration { - public class EphemeralCluster : EphemeralCluster - { - public EphemeralCluster(OpenSearchVersion version, int numberOfNodes = 1) - : base(new EphemeralClusterConfiguration(version, ClusterFeatures.None, numberOfNodes: numberOfNodes)) - { - } - - public EphemeralCluster(EphemeralClusterConfiguration clusterConfiguration) : base(clusterConfiguration) - { - } - } - - public abstract class EphemeralCluster : ClusterBase, - IEphemeralCluster - where TConfiguration : EphemeralClusterConfiguration - { - protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterConfiguration) => - Composer = new EphemeralClusterComposer(this); - - protected EphemeralClusterComposer Composer { get; } - - protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) - { - base.ModifyNodeConfiguration(nodeConfiguration, port); - - if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); - } - - public virtual ICollection NodesUris(string hostName = null) - { - hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() - ? "ipv4.fiddler" - : "localhost"); - var ssl = ClusterConfiguration.EnableSsl ? "s" : ""; - return Nodes - .Select(n => $"http{ssl}://{hostName}:{n.Port ?? 9200}") - .Distinct() - .Select(n => new Uri(n)) - .ToList(); - } - - public bool CachingAndCachedHomeExists() - { - if (!ClusterConfiguration.CacheOpenSearchHomeInstallation) return false; - var cachedOpenSearchHomeFolder = Path.Combine(FileSystem.LocalFolder, GetCacheFolderName()); - return Directory.Exists(cachedOpenSearchHomeFolder); - } - - public virtual string GetCacheFolderName() - { - var config = ClusterConfiguration; - - var sb = new StringBuilder(); - sb.Append(EphemeralClusterComposerBase.InstallationTasks.Count()); - sb.Append("-"); - if (config.EnableSsl) sb.Append("ssl"); - if (config.Plugins != null && config.Plugins.Count > 0) - { - sb.Append("-"); - foreach (var p in config.Plugins.OrderBy(p => p.SubProductName)) - sb.Append(p.SubProductName.ToLowerInvariant()); - } - - var name = sb.ToString(); - - return CalculateSha1(name, Encoding.UTF8); - } - - protected override void OnBeforeStart() - { - Composer.Install(); - Composer.OnBeforeStart(); - } - - protected override void OnDispose() => Composer.OnStop(); - - protected override void OnAfterStarted() => Composer.OnAfterStart(); - - protected override string SeeLogsMessage(string message) - { - var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); - if (!File.Exists(log) || ClusterConfiguration.ShowOpenSearchOutputAfterStarted) return message; - if (!Started) return message; - using (var fileStream = new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (var textReader = new StreamReader(fileStream)) - { - var logContents = textReader.ReadToEnd(); - return message + $" contents of {log}:{Environment.NewLine}" + logContents; - } - } - - public static string CalculateSha1(string text, Encoding enc) - { - var buffer = enc.GetBytes(text); - var cryptoTransformSha1 = new SHA1CryptoServiceProvider(); - return BitConverter.ToString(cryptoTransformSha1.ComputeHash(buffer)) - .Replace("-", "").ToLowerInvariant().Substring(0, 12); - } - } + protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterConfiguration) => + Composer = new EphemeralClusterComposer(this); + + protected EphemeralClusterComposer Composer { get; } + + protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + base.ModifyNodeConfiguration(nodeConfiguration, port); + + if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); + } + + public virtual ICollection NodesUris(string hostName = null) + { + hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() + ? "ipv4.fiddler" + : "localhost"); + var ssl = ClusterConfiguration.EnableSsl ? "s" : ""; + return Nodes + .Select(n => $"http{ssl}://{hostName}:{n.Port ?? 9200}") + .Distinct() + .Select(n => new Uri(n)) + .ToList(); + } + + public bool CachingAndCachedHomeExists() + { + if (!ClusterConfiguration.CacheOpenSearchHomeInstallation) return false; + var cachedOpenSearchHomeFolder = Path.Combine(FileSystem.LocalFolder, GetCacheFolderName()); + return Directory.Exists(cachedOpenSearchHomeFolder); + } + + public virtual string GetCacheFolderName() + { + var config = ClusterConfiguration; + + var sb = new StringBuilder(); + sb.Append(EphemeralClusterComposerBase.InstallationTasks.Count()); + sb.Append("-"); + if (config.EnableSsl) sb.Append("ssl"); + if (config.Plugins != null && config.Plugins.Count > 0) + { + sb.Append("-"); + foreach (var p in config.Plugins.OrderBy(p => p.SubProductName)) + sb.Append(p.SubProductName.ToLowerInvariant()); + } + + var name = sb.ToString(); + + return CalculateSha1(name, Encoding.UTF8); + } + + protected override void OnBeforeStart() + { + Composer.Install(); + Composer.OnBeforeStart(); + } + + protected override void OnDispose() => Composer.OnStop(); + + protected override void OnAfterStarted() => Composer.OnAfterStart(); + + protected override string SeeLogsMessage(string message) + { + var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); + if (!File.Exists(log) || ClusterConfiguration.ShowOpenSearchOutputAfterStarted) return message; + if (!Started) return message; + using (var fileStream = new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (var textReader = new StreamReader(fileStream)) + { + var logContents = textReader.ReadToEnd(); + return message + $" contents of {log}:{Environment.NewLine}" + logContents; + } + } + + public static string CalculateSha1(string text, Encoding enc) + { + var buffer = enc.GetBytes(text); + var cryptoTransformSha1 = new SHA1CryptoServiceProvider(); + return BitConverter.ToString(cryptoTransformSha1.ComputeHash(buffer)) + .Replace("-", "").ToLowerInvariant().Substring(0, 12); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs index bd055f4874..580357ab62 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs @@ -35,101 +35,100 @@ using OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; using OpenSearch.OpenSearch.Managed.FileSystem; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public class EphemeralClusterComposerBase +{ + protected EphemeralClusterComposerBase() + { + } + + internal static IEnumerable InstallationTasks { get; } = new List + { + new PrintConfiguration(), + new CreateLocalApplicationDirectory(), + new CopyCachedOpenSearchInstallation(), + new EnsureJavaHomeEnvironmentVariableIsSet(), + new DownloadOpenSearchVersion(), + new UnzipOpenSearch(), + new SetOpenSearchBundledJdkJavaHome(), + new InstallPlugins(), + new InitialConfiguration() + }; + + protected static IEnumerable BeforeStart { get; } = new List + { + new CreateEphemeralDirectory(), + new CacheOpenSearchInstallation() + }; + + protected static IEnumerable NodeStoppedTasks { get; } = new List + { + new CleanUpDirectoriesAfterNodeStopped() + }; + + protected static IEnumerable AfterStartedTasks { get; } = new List + { + new ValidateRunningVersion(), + new ValidateClusterStateTask(), + new ValidatePluginsTask(), + }; +} + + +public class EphemeralClusterComposer : EphemeralClusterComposerBase + where TConfiguration : EphemeralClusterConfiguration { - public class EphemeralClusterComposerBase - { - protected EphemeralClusterComposerBase() - { - } - - internal static IEnumerable InstallationTasks { get; } = new List - { - new PrintConfiguration(), - new CreateLocalApplicationDirectory(), - new CopyCachedOpenSearchInstallation(), - new EnsureJavaHomeEnvironmentVariableIsSet(), - new DownloadOpenSearchVersion(), - new UnzipOpenSearch(), - new SetOpenSearchBundledJdkJavaHome(), - new InstallPlugins(), - new InitialConfiguration() - }; - - protected static IEnumerable BeforeStart { get; } = new List - { - new CreateEphemeralDirectory(), - new CacheOpenSearchInstallation() - }; - - protected static IEnumerable NodeStoppedTasks { get; } = new List - { - new CleanUpDirectoriesAfterNodeStopped() - }; - - protected static IEnumerable AfterStartedTasks { get; } = new List - { - new ValidateRunningVersion(), - new ValidateClusterStateTask(), - new ValidatePluginsTask(), - }; - } - - - public class EphemeralClusterComposer : EphemeralClusterComposerBase - where TConfiguration : EphemeralClusterConfiguration - { - private readonly object _lock = new object(); - public EphemeralClusterComposer(IEphemeralCluster cluster) => Cluster = cluster; - - private IEphemeralCluster Cluster { get; } - - private bool NodeStarted { get; set; } - - public void OnStop() => Itterate(NodeStoppedTasks, (t, c, fs) => t.Run(c, NodeStarted), false); - - public void Install() => Itterate(InstallationTasks, (t, c, fs) => t.Run(c)); - - public void OnBeforeStart() - { - var tasks = new List(BeforeStart); - if (Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks != null) - tasks.AddRange(Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks); - - if (Cluster.ClusterConfiguration.PrintYamlFilesInConfigFolder) - tasks.Add(new PrintYamlContents()); - - Itterate(tasks, (t, c, fs) => t.Run(c)); - - NodeStarted = true; - } - - public void OnAfterStart() - { - if (Cluster.ClusterConfiguration.SkipBuiltInAfterStartTasks) return; - var tasks = new List(AfterStartedTasks); - if (Cluster.ClusterConfiguration.AdditionalAfterStartedTasks != null) - tasks.AddRange(Cluster.ClusterConfiguration.AdditionalAfterStartedTasks); - Itterate(tasks, (t, c, fs) => t.Run(c), false); - } - - private void Itterate(IEnumerable collection, - Action, INodeFileSystem> act, bool callOnStop = true) - { - lock (_lock) - { - var cluster = Cluster; - foreach (var task in collection) - try - { - act(task, cluster, cluster.FileSystem); - } - catch (Exception) - { - if (callOnStop) OnStop(); - throw; - } - } - } - } + private readonly object _lock = new object(); + public EphemeralClusterComposer(IEphemeralCluster cluster) => Cluster = cluster; + + private IEphemeralCluster Cluster { get; } + + private bool NodeStarted { get; set; } + + public void OnStop() => Itterate(NodeStoppedTasks, (t, c, fs) => t.Run(c, NodeStarted), false); + + public void Install() => Itterate(InstallationTasks, (t, c, fs) => t.Run(c)); + + public void OnBeforeStart() + { + var tasks = new List(BeforeStart); + if (Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks != null) + tasks.AddRange(Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks); + + if (Cluster.ClusterConfiguration.PrintYamlFilesInConfigFolder) + tasks.Add(new PrintYamlContents()); + + Itterate(tasks, (t, c, fs) => t.Run(c)); + + NodeStarted = true; + } + + public void OnAfterStart() + { + if (Cluster.ClusterConfiguration.SkipBuiltInAfterStartTasks) return; + var tasks = new List(AfterStartedTasks); + if (Cluster.ClusterConfiguration.AdditionalAfterStartedTasks != null) + tasks.AddRange(Cluster.ClusterConfiguration.AdditionalAfterStartedTasks); + Itterate(tasks, (t, c, fs) => t.Run(c), false); + } + + private void Itterate(IEnumerable collection, + Action, INodeFileSystem> act, bool callOnStop = true) + { + lock (_lock) + { + var cluster = Cluster; + foreach (var task in collection) + try + { + act(task, cluster, cluster.FileSystem); + } + catch (Exception) + { + if (callOnStop) OnStop(); + throw; + } + } + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs index 77aaef02c4..9bd691a685 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs @@ -36,67 +36,66 @@ using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public class EphemeralClusterConfiguration : ClusterConfiguration { - public class EphemeralClusterConfiguration : ClusterConfiguration - { - public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null, - int numberOfNodes = 1) - : this(version, ClusterFeatures.None, plugins, numberOfNodes) - { - } - - public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features, - OpenSearchPlugins plugins = null, int numberOfNodes = 1) - : base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName) - { - Features = features; - - var pluginsList = plugins?.ToList() ?? new List(); - Plugins = new OpenSearchPlugins(pluginsList); - } - - private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6); - private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}"; - - /// - /// The features supported by the cluster - /// - public ClusterFeatures Features { get; } - - /// - /// The collection of plugins to install - /// - public OpenSearchPlugins Plugins { get; } - - /// - /// Validates that the plugins to install can be installed on the target OpenSearch version. - /// This can be useful to fail early when subsequent operations are relying on installation - /// succeeding. - /// - public bool ValidatePluginsToInstall { get; } = true; - - public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL); - - public IList AdditionalBeforeNodeStartedTasks { get; } = new List(); - - public IList AdditionalAfterStartedTasks { get; } = new List(); - - /// - /// Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the - /// first call into the cluster - /// - public bool SkipBuiltInAfterStartTasks { get; set; } - - /// Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running - public bool HttpFiddlerAware { get; set; } - - protected virtual string NodePrefix => "ephemeral"; - - public override string CreateNodeName(int? node) - { - var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); - return $"{NodePrefix}-node-{suffix}{node}"; - } - } + public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null, + int numberOfNodes = 1) + : this(version, ClusterFeatures.None, plugins, numberOfNodes) + { + } + + public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features, + OpenSearchPlugins plugins = null, int numberOfNodes = 1) + : base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName) + { + Features = features; + + var pluginsList = plugins?.ToList() ?? new List(); + Plugins = new OpenSearchPlugins(pluginsList); + } + + private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6); + private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}"; + + /// + /// The features supported by the cluster + /// + public ClusterFeatures Features { get; } + + /// + /// The collection of plugins to install + /// + public OpenSearchPlugins Plugins { get; } + + /// + /// Validates that the plugins to install can be installed on the target OpenSearch version. + /// This can be useful to fail early when subsequent operations are relying on installation + /// succeeding. + /// + public bool ValidatePluginsToInstall { get; } = true; + + public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL); + + public IList AdditionalBeforeNodeStartedTasks { get; } = new List(); + + public IList AdditionalAfterStartedTasks { get; } = new List(); + + /// + /// Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the + /// first call into the cluster + /// + public bool SkipBuiltInAfterStartTasks { get; set; } + + /// Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running + public bool HttpFiddlerAware { get; set; } + + protected virtual string NodePrefix => "ephemeral"; + + public override string CreateNodeName(int? node) + { + var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); + return $"{NodePrefix}-node-{suffix}{node}"; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs index 9eebfa981c..657bcdd3cb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs @@ -31,57 +31,56 @@ using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public class EphemeralFileSystem : NodeFileSystem { - public class EphemeralFileSystem : NodeFileSystem - { - public EphemeralFileSystem(OpenSearchVersion version, string clusterName) : base(version, - EphemeralHome(version, clusterName)) => ClusterName = clusterName; + public EphemeralFileSystem(OpenSearchVersion version, string clusterName) : base(version, + EphemeralHome(version, clusterName)) => ClusterName = clusterName; - private string ClusterName { get; } + private string ClusterName { get; } - public string TempFolder => Path.Combine(Path.GetTempPath(), SubFolder, Artifact.LocalFolderName, ClusterName); + public string TempFolder => Path.Combine(Path.GetTempPath(), SubFolder, Artifact.LocalFolderName, ClusterName); - public override string ConfigPath => Path.Combine(TempFolder, "config"); - public override string LogsPath => Path.Combine(TempFolder, "logs"); - public override string RepositoryPath => Path.Combine(TempFolder, "repositories"); - public override string DataPath => Path.Combine(TempFolder, "data"); + public override string ConfigPath => Path.Combine(TempFolder, "config"); + public override string LogsPath => Path.Combine(TempFolder, "logs"); + public override string RepositoryPath => Path.Combine(TempFolder, "repositories"); + public override string DataPath => Path.Combine(TempFolder, "data"); - //certificates - public string CertificateFolderName => "node-certificates"; - public string CertificateNodeName => "node01"; - public string ClientCertificateName => "cn=John Doe,ou=example,o=com"; - public string ClientCertificateFilename => "john_doe"; + //certificates + public string CertificateFolderName => "node-certificates"; + public string CertificateNodeName => "node01"; + public string ClientCertificateName => "cn=John Doe,ou=example,o=com"; + public string ClientCertificateFilename => "john_doe"; - public string CertificatesPath => Path.Combine(ConfigPath, CertificateFolderName); + public string CertificatesPath => Path.Combine(ConfigPath, CertificateFolderName); - public string CaCertificate => Path.Combine(CertificatesPath, "ca", "ca") + ".crt"; + public string CaCertificate => Path.Combine(CertificatesPath, "ca", "ca") + ".crt"; - public string NodePrivateKey => - Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".key"; + public string NodePrivateKey => + Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".key"; - public string NodeCertificate => - Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".crt"; + public string NodeCertificate => + Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".crt"; - public string ClientCertificate => - Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; + public string ClientCertificate => + Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; - public string ClientPrivateKey => - Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".key"; + public string ClientPrivateKey => + Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".key"; - public string UnusedCertificateFolderName => $"unused-{CertificateFolderName}"; - public string UnusedCertificatesPath => Path.Combine(ConfigPath, UnusedCertificateFolderName); - public string UnusedCaCertificate => Path.Combine(UnusedCertificatesPath, "ca", "ca") + ".crt"; + public string UnusedCertificateFolderName => $"unused-{CertificateFolderName}"; + public string UnusedCertificatesPath => Path.Combine(ConfigPath, UnusedCertificateFolderName); + public string UnusedCaCertificate => Path.Combine(UnusedCertificatesPath, "ca", "ca") + ".crt"; - public string UnusedClientCertificate => - Path.Combine(UnusedCertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; + public string UnusedClientCertificate => + Path.Combine(UnusedCertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; - protected static string EphemeralHome(OpenSearchVersion version, string clusterName) - { - var temp = Path.Combine(Path.GetTempPath(), SubFolder, - version.Artifact(Product.OpenSearch).LocalFolderName, clusterName); - return Path.Combine(temp, "home"); - } - } + protected static string EphemeralHome(OpenSearchVersion version, string clusterName) + { + var temp = Path.Combine(Path.GetTempPath(), SubFolder, + version.Artifact(Product.OpenSearch).LocalFolderName, clusterName); + return Path.Combine(temp, "home"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs index d97995713a..4f64e59c4d 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs @@ -30,17 +30,16 @@ using System.Collections.Generic; using OpenSearch.OpenSearch.Managed; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public interface IEphemeralCluster { - public interface IEphemeralCluster - { - ICollection NodesUris(string hostName = null); - string GetCacheFolderName(); - bool CachingAndCachedHomeExists(); - } + ICollection NodesUris(string hostName = null); + string GetCacheFolderName(); + bool CachingAndCachedHomeExists(); +} - public interface IEphemeralCluster : IEphemeralCluster, ICluster - where TConfiguration : EphemeralClusterConfiguration - { - } +public interface IEphemeralCluster : IEphemeralCluster, ICluster + where TConfiguration : EphemeralClusterConfiguration +{ } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs index 7c15f6a5a1..4f25f811b2 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs @@ -31,47 +31,46 @@ using System.Linq; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral.Plugins +namespace OpenSearch.OpenSearch.Ephemeral.Plugins; + +public class OpenSearchPlugins : ReadOnlyCollection { - public class OpenSearchPlugins : ReadOnlyCollection - { - public OpenSearchPlugins(IList list) : base(list) - { - } + public OpenSearchPlugins(IList list) : base(list) + { + } - public OpenSearchPlugins(params OpenSearchPlugin[] list) : base(list) - { - } + public OpenSearchPlugins(params OpenSearchPlugin[] list) : base(list) + { + } - public static OpenSearchPlugins Supported { get; } = - new OpenSearchPlugins(new List - { - OpenSearchPlugin.AnalysisIcu, - OpenSearchPlugin.AnalysisKuromoji, - OpenSearchPlugin.AnalysisNori, - OpenSearchPlugin.AnalysisPhonetic, - OpenSearchPlugin.AnalysisSmartCn, - OpenSearchPlugin.AnalysisStempel, - OpenSearchPlugin.AnalysisUkrainian, - OpenSearchPlugin.DiscoveryAzureClassic, - OpenSearchPlugin.DiscoveryEC2, - OpenSearchPlugin.DiscoveryFile, - OpenSearchPlugin.DiscoveryGCE, - OpenSearchPlugin.IngestAttachment, - OpenSearchPlugin.IngestGeoIp, - OpenSearchPlugin.IngestUserAgent, - OpenSearchPlugin.MapperAttachment, - OpenSearchPlugin.MapperMurmur3, - OpenSearchPlugin.MapperSize, - OpenSearchPlugin.RepositoryAzure, - OpenSearchPlugin.RepositoryGCS, - OpenSearchPlugin.RepositoryHDFS, - OpenSearchPlugin.RepositoryS3, - OpenSearchPlugin.Security, - OpenSearchPlugin.StoreSMB, - OpenSearchPlugin.DeleteByQuery, - }); + public static OpenSearchPlugins Supported { get; } = + new OpenSearchPlugins(new List + { + OpenSearchPlugin.AnalysisIcu, + OpenSearchPlugin.AnalysisKuromoji, + OpenSearchPlugin.AnalysisNori, + OpenSearchPlugin.AnalysisPhonetic, + OpenSearchPlugin.AnalysisSmartCn, + OpenSearchPlugin.AnalysisStempel, + OpenSearchPlugin.AnalysisUkrainian, + OpenSearchPlugin.DiscoveryAzureClassic, + OpenSearchPlugin.DiscoveryEC2, + OpenSearchPlugin.DiscoveryFile, + OpenSearchPlugin.DiscoveryGCE, + OpenSearchPlugin.IngestAttachment, + OpenSearchPlugin.IngestGeoIp, + OpenSearchPlugin.IngestUserAgent, + OpenSearchPlugin.MapperAttachment, + OpenSearchPlugin.MapperMurmur3, + OpenSearchPlugin.MapperSize, + OpenSearchPlugin.RepositoryAzure, + OpenSearchPlugin.RepositoryGCS, + OpenSearchPlugin.RepositoryHDFS, + OpenSearchPlugin.RepositoryS3, + OpenSearchPlugin.Security, + OpenSearchPlugin.StoreSMB, + OpenSearchPlugin.DeleteByQuery, + }); - public override string ToString() => string.Join(", ", Items.Select(s => s.SubProductName)); - } + public override string ToString() => string.Join(", ", Items.Select(s => s.SubProductName)); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs index 76bf8c751c..738013827f 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs @@ -26,12 +26,11 @@ * under the License. */ -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public static class SecurityRealms { - public static class SecurityRealms - { - public const string FileRealm = "file1"; + public const string FileRealm = "file1"; - public const string PkiRealm = "pki1"; - } + public const string PkiRealm = "pki1"; } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs index 8acaa36384..b390e4039e 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs @@ -30,55 +30,54 @@ using OpenSearch.OpenSearch.Managed.ConsoleWriters; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.AfterNodeStoppedTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.AfterNodeStoppedTasks; + +public class CleanUpDirectoriesAfterNodeStopped : IClusterTeardownTask { - public class CleanUpDirectoriesAfterNodeStopped : IClusterTeardownTask - { - public void Run(IEphemeralCluster cluster, bool nodeStarted) - { - var fs = cluster.FileSystem; - var w = cluster.Writer; - var a = cluster.ClusterConfiguration.Artifact; - if (cluster.ClusterConfiguration.NoCleanupAfterNodeStopped) - { - w.WriteDiagnostic( - $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} skipping cleanup as requested on cluster configuration"); - return; - } + public void Run(IEphemeralCluster cluster, bool nodeStarted) + { + var fs = cluster.FileSystem; + var w = cluster.Writer; + var a = cluster.ClusterConfiguration.Artifact; + if (cluster.ClusterConfiguration.NoCleanupAfterNodeStopped) + { + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} skipping cleanup as requested on cluster configuration"); + return; + } - DeleteDirectory(w, "cluster data", fs.DataPath); - DeleteDirectory(w, "cluster config", fs.ConfigPath); - DeleteDirectory(w, "cluster logs", fs.LogsPath); - DeleteDirectory(w, "repositories", fs.RepositoryPath); - var efs = fs as EphemeralFileSystem; - if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) - DeleteDirectory(w, "cluster temp folder", efs.TempFolder); + DeleteDirectory(w, "cluster data", fs.DataPath); + DeleteDirectory(w, "cluster config", fs.ConfigPath); + DeleteDirectory(w, "cluster logs", fs.LogsPath); + DeleteDirectory(w, "repositories", fs.RepositoryPath); + var efs = fs as EphemeralFileSystem; + if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) + DeleteDirectory(w, "cluster temp folder", efs.TempFolder); - if (efs != null) - { - var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); - if (extractedFolder != fs.OpenSearchHome) - DeleteDirectory(w, "ephemeral OPENSEARCH_HOME", fs.OpenSearchHome); - //if the node was not started delete the cached extractedFolder - if (!nodeStarted) - DeleteDirectory(w, "cached extracted folder - node failed to start", extractedFolder); - } + if (efs != null) + { + var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); + if (extractedFolder != fs.OpenSearchHome) + DeleteDirectory(w, "ephemeral OPENSEARCH_HOME", fs.OpenSearchHome); + //if the node was not started delete the cached extractedFolder + if (!nodeStarted) + DeleteDirectory(w, "cached extracted folder - node failed to start", extractedFolder); + } - //if the node did not start make sure we delete the cached folder as we can not assume its in a good state - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - if (cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && !nodeStarted) - DeleteDirectory(w, "cached installation - node failed to start", cachedOpenSearchHomeFolder); - else - w.WriteDiagnostic( - $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} Leaving [cached folder] on disk: {{{cachedOpenSearchHomeFolder}}}"); - } + //if the node did not start make sure we delete the cached folder as we can not assume its in a good state + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + if (cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && !nodeStarted) + DeleteDirectory(w, "cached installation - node failed to start", cachedOpenSearchHomeFolder); + else + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} Leaving [cached folder] on disk: {{{cachedOpenSearchHomeFolder}}}"); + } - private static void DeleteDirectory(IConsoleLineHandler w, string description, string path) - { - if (!Directory.Exists(path)) return; - w.WriteDiagnostic( - $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} attempting to delete [{description}]: {{{path}}}"); - Directory.Delete(path, true); - } - } + private static void DeleteDirectory(IConsoleLineHandler w, string description, string path) + { + if (!Directory.Exists(path)) return; + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} attempting to delete [{description}]: {{{path}}}"); + Directory.Delete(path, true); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs index 85a0a1fc89..d01a0964ac 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs @@ -29,29 +29,28 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks; + +public class CacheOpenSearchInstallation : ClusterComposeTask { - public class CacheOpenSearchInstallation : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + public override void Run(IEphemeralCluster cluster) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; - var fs = cluster.FileSystem; - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - var cachedOpenSearchConfig = Path.Combine(cachedOpenSearchHomeFolder, "config"); - if (File.Exists(cachedOpenSearchConfig)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CacheOpenSearchInstallation)}}} cached home already exists [{cachedOpenSearchHomeFolder}]"); - return; - } + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var cachedOpenSearchConfig = Path.Combine(cachedOpenSearchHomeFolder, "config"); + if (File.Exists(cachedOpenSearchConfig)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CacheOpenSearchInstallation)}}} cached home already exists [{cachedOpenSearchHomeFolder}]"); + return; + } - var source = fs.OpenSearchHome; - var target = cachedOpenSearchHomeFolder; - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CacheOpenSearchInstallation)}}} caching {{{source}}} to [{target}]"); - CopyFolder(source, target, false); - } - } + var source = fs.OpenSearchHome; + var target = cachedOpenSearchHomeFolder; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CacheOpenSearchInstallation)}}} caching {{{source}}} to [{target}]"); + CopyFolder(source, target, false); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs index 67e77cbc8b..b417342ed3 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs @@ -30,56 +30,55 @@ using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.OpenSearch.Managed.FileSystem; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks; + +public class CreateEphemeralDirectory : ClusterComposeTask { - public class CreateEphemeralDirectory : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var fs = cluster.FileSystem; - if (!(fs is EphemeralFileSystem f)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateEphemeralDirectory)}}} unexpected IFileSystem implementation {{{fs.GetType()}}}"); - return; - } + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + if (!(fs is EphemeralFileSystem f)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} unexpected IFileSystem implementation {{{fs.GetType()}}}"); + return; + } - cluster.Writer?.WriteDiagnostic($"{{{nameof(CreateEphemeralDirectory)}}} creating {{{f.TempFolder}}}"); + cluster.Writer?.WriteDiagnostic($"{{{nameof(CreateEphemeralDirectory)}}} creating {{{f.TempFolder}}}"); - Directory.CreateDirectory(f.TempFolder); + Directory.CreateDirectory(f.TempFolder); - if (!Directory.Exists(f.ConfigPath)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateEphemeralDirectory)}}} creating config folder {{{f.ConfigPath}}}"); - Directory.CreateDirectory(f.ConfigPath); - } + if (!Directory.Exists(f.ConfigPath)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} creating config folder {{{f.ConfigPath}}}"); + Directory.CreateDirectory(f.ConfigPath); + } - CopyHomeConfigToEphemeralConfig(cluster, f, fs); - } + CopyHomeConfigToEphemeralConfig(cluster, f, fs); + } - private static void CopyHomeConfigToEphemeralConfig(IEphemeralCluster cluster, - EphemeralFileSystem ephemeralFileSystem, INodeFileSystem fs) - { - var target = ephemeralFileSystem.ConfigPath; - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - var cachedOpenSearchYaml = Path.Combine(cachedOpenSearchHomeFolder, "config", "opensearch.yaml"); + private static void CopyHomeConfigToEphemeralConfig(IEphemeralCluster cluster, + EphemeralFileSystem ephemeralFileSystem, INodeFileSystem fs) + { + var target = ephemeralFileSystem.ConfigPath; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var cachedOpenSearchYaml = Path.Combine(cachedOpenSearchHomeFolder, "config", "opensearch.yaml"); - var homeSource = - cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && File.Exists(cachedOpenSearchYaml) - ? cachedOpenSearchHomeFolder - : fs.OpenSearchHome; - var source = Path.Combine(homeSource, "config"); - if (!Directory.Exists(source)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateEphemeralDirectory)}}} source config {{{source}}} does not exist nothing to copy"); - return; - } + var homeSource = + cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && File.Exists(cachedOpenSearchYaml) + ? cachedOpenSearchHomeFolder + : fs.OpenSearchHome; + var source = Path.Combine(homeSource, "config"); + if (!Directory.Exists(source)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} source config {{{source}}} does not exist nothing to copy"); + return; + } - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateEphemeralDirectory)}}} copying cached {{{source}}} as to [{target}]"); - CopyFolder(source, target); - } - } + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} copying cached {{{source}}} as to [{target}]"); + CopyFolder(source, target); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs index d525d8eb65..445f63fff0 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs @@ -30,34 +30,33 @@ using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks; + +public class PrintYamlContents : ClusterComposeTask { - public class PrintYamlContents : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var c = cluster.ClusterConfiguration; - var v = c.Version; - var fs = cluster.FileSystem; + public override void Run(IEphemeralCluster cluster) + { + var c = cluster.ClusterConfiguration; + var v = c.Version; + var fs = cluster.FileSystem; - var files = Directory.GetFiles(fs.ConfigPath, "*.yml", SearchOption.AllDirectories); - foreach (var file in files) DumpFile(cluster, file); - } + var files = Directory.GetFiles(fs.ConfigPath, "*.yml", SearchOption.AllDirectories); + foreach (var file in files) DumpFile(cluster, file); + } - private static void DumpFile(IEphemeralCluster cluster, string configFile) - { - if (!File.Exists(configFile)) - { - cluster.Writer.WriteDiagnostic( - $"{{{nameof(PrintYamlContents)}}} skipped printing [{configFile}] as it does not exists"); - return; - } + private static void DumpFile(IEphemeralCluster cluster, string configFile) + { + if (!File.Exists(configFile)) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(PrintYamlContents)}}} skipped printing [{configFile}] as it does not exists"); + return; + } - var fileName = Path.GetFileName(configFile); - cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} printing [{configFile}]"); - var lines = File.ReadAllLines(configFile).ToList(); - foreach (var l in lines.Where(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) - cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} [{fileName}] {l}"); - } - } + var fileName = Path.GetFileName(configFile); + cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} printing [{configFile}]"); + var lines = File.ReadAllLines(configFile).ToList(); + foreach (var l in lines.Where(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) + cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} [{fileName}] {l}"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs index 74441951ef..3c5b0bc5f8 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -40,234 +40,233 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using OpenSearch.OpenSearch.Managed.ConsoleWriters; using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; using ProcNet; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks; + +public interface IClusterComposeTask +{ + void Run(IEphemeralCluster cluster); +} + +public interface IClusterTeardownTask +{ + /// + /// Called when the cluster disposes, used to clean up after itself. + /// + /// The cluster configuration of the node that was started + /// Whether the cluster composer was successful in starting the node + void Run(IEphemeralCluster cluster, bool nodeStarted); +} + +public abstract class ClusterComposeTask : IClusterComposeTask { - public interface IClusterComposeTask - { - void Run(IEphemeralCluster cluster); - } - - public interface IClusterTeardownTask - { - /// - /// Called when the cluster disposes, used to clean up after itself. - /// - /// The cluster configuration of the node that was started - /// Whether the cluster composer was successful in starting the node - void Run(IEphemeralCluster cluster, bool nodeStarted); - } - - public abstract class ClusterComposeTask : IClusterComposeTask - { - protected static bool IsWindows { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - protected static string BinarySuffix => IsWindows ? ".bat" : string.Empty; - public abstract void Run(IEphemeralCluster cluster); - - protected static void DownloadFile(string from, string to) - { - if (File.Exists(to)) return; - var http = new HttpClient(); - using (var stream = http.GetStreamAsync(new Uri(from)).GetAwaiter().GetResult()) - using (var fileStream = File.Create(to)) - { - stream.CopyTo(fileStream); - fileStream.Flush(); - } - } - - protected string GetResponseException(HttpResponseMessage m) => - $"Code: {m?.StatusCode} Reason: {m?.ReasonPhrase} Content: {GetResponseString(m)}"; - - protected string GetResponseString(HttpResponseMessage m) => - m?.Content?.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult() ?? string.Empty; - - protected HttpResponseMessage Get(IEphemeralCluster cluster, string path, - string query) => - Call(cluster, path, query, (c, u, t) => c.GetAsync(u, t)); - - protected HttpResponseMessage Post(IEphemeralCluster cluster, string path, - string query, string json) => - Call(cluster, path, query, - (c, u, t) => c.PostAsync(u, new StringContent(json, Encoding.UTF8, "application/json"), t)); - - private HttpResponseMessage Call( - IEphemeralCluster cluster, - string path, - string query, - Func> verb) - { - var q = string.IsNullOrEmpty(query) ? "pretty=true" : query + "&pretty=true"; - var statusUrl = new UriBuilder(cluster.NodesUris().First()) {Path = path, Query = q}.Uri; - - var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20)); - var handler = new HttpClientHandler - { - AutomaticDecompression = - DecompressionMethods.Deflate | DecompressionMethods.GZip | DecompressionMethods.None, - }; - cluster.Writer.WriteDiagnostic( - $"{{{nameof(Call)}}} [{statusUrl}] SSL: {cluster.ClusterConfiguration.EnableSsl}"); - if (cluster.ClusterConfiguration.EnableSsl) - { + protected static bool IsWindows { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + protected static string BinarySuffix => IsWindows ? ".bat" : string.Empty; + public abstract void Run(IEphemeralCluster cluster); + + protected static void DownloadFile(string from, string to) + { + if (File.Exists(to)) return; + var http = new HttpClient(); + using (var stream = http.GetStreamAsync(new Uri(from)).GetAwaiter().GetResult()) + using (var fileStream = File.Create(to)) + { + stream.CopyTo(fileStream); + fileStream.Flush(); + } + } + + protected string GetResponseException(HttpResponseMessage m) => + $"Code: {m?.StatusCode} Reason: {m?.ReasonPhrase} Content: {GetResponseString(m)}"; + + protected string GetResponseString(HttpResponseMessage m) => + m?.Content?.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult() ?? string.Empty; + + protected HttpResponseMessage Get(IEphemeralCluster cluster, string path, + string query) => + Call(cluster, path, query, (c, u, t) => c.GetAsync(u, t)); + + protected HttpResponseMessage Post(IEphemeralCluster cluster, string path, + string query, string json) => + Call(cluster, path, query, + (c, u, t) => c.PostAsync(u, new StringContent(json, Encoding.UTF8, "application/json"), t)); + + private HttpResponseMessage Call( + IEphemeralCluster cluster, + string path, + string query, + Func> verb) + { + var q = string.IsNullOrEmpty(query) ? "pretty=true" : query + "&pretty=true"; + var statusUrl = new UriBuilder(cluster.NodesUris().First()) { Path = path, Query = q }.Uri; + + var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + var handler = new HttpClientHandler + { + AutomaticDecompression = + DecompressionMethods.Deflate | DecompressionMethods.GZip | DecompressionMethods.None, + }; + cluster.Writer.WriteDiagnostic( + $"{{{nameof(Call)}}} [{statusUrl}] SSL: {cluster.ClusterConfiguration.EnableSsl}"); + if (cluster.ClusterConfiguration.EnableSsl) + { #if !NETSTANDARD ServicePointManager.ServerCertificateValidationCallback += ServerCertificateValidationCallback; #else - handler.ServerCertificateCustomValidationCallback += (m, c, cn, p) => true; + handler.ServerCertificateCustomValidationCallback += (m, c, cn, p) => true; #endif - } - - using (var client = new HttpClient(handler) {Timeout = TimeSpan.FromSeconds(20)}) - { - if (cluster.ClusterConfiguration.EnableSsl) - { - var byteArray = - Encoding.ASCII.GetBytes( - $"{ClusterAuthentication.Admin.Username}:{ClusterAuthentication.Admin.Password}"); - client.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); - } - - try - { - var response = verb(client, statusUrl, tokenSource.Token).ConfigureAwait(false).GetAwaiter() - .GetResult(); - if (!response.IsSuccessStatusCode) - { - cluster.Writer.WriteDiagnostic( - $"{{{nameof(Call)}}} [{statusUrl}] Unsuccessful status code: [{(int) response.StatusCode}]"); - var body = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - foreach (var l in (body ?? string.Empty).Split('\n', '\r')) - cluster.Writer.WriteDiagnostic($"{{{nameof(Call)}}} [{statusUrl}] returned [{l}]"); - } - - return response; - } - catch (Exception e) - { - cluster.Writer.WriteError($"{{{nameof(Call)}}} [{statusUrl}] exception: {e}"); - throw; - } - finally - { + } + + using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(20) }) + { + if (cluster.ClusterConfiguration.EnableSsl) + { + var byteArray = + Encoding.ASCII.GetBytes( + $"{ClusterAuthentication.Admin.Username}:{ClusterAuthentication.Admin.Password}"); + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); + } + + try + { + var response = verb(client, statusUrl, tokenSource.Token).ConfigureAwait(false).GetAwaiter() + .GetResult(); + if (!response.IsSuccessStatusCode) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(Call)}}} [{statusUrl}] Unsuccessful status code: [{(int)response.StatusCode}]"); + var body = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + foreach (var l in (body ?? string.Empty).Split('\n', '\r')) + cluster.Writer.WriteDiagnostic($"{{{nameof(Call)}}} [{statusUrl}] returned [{l}]"); + } + + return response; + } + catch (Exception e) + { + cluster.Writer.WriteError($"{{{nameof(Call)}}} [{statusUrl}] exception: {e}"); + throw; + } + finally + { #if !NETSTANDARD ServicePointManager.ServerCertificateValidationCallback -= ServerCertificateValidationCallback; #endif - } - } - } - - private static bool ServerCertificateValidationCallback(object sender, X509Certificate certificate, - X509Chain chain, SslPolicyErrors sslpolicyerrors) => true; - - protected static void WriteFileIfNotExist(string fileLocation, string contents) - { - if (!File.Exists(fileLocation)) File.WriteAllText(fileLocation, contents); - } - - protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, null, arguments); - - protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, IDictionary environmentVariables, - params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); - - private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, IDictionary environmentVariables, params string[] arguments) - { - var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; - writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); - - var environment = new Dictionary - { - {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, - {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} - }; - - if (environmentVariables != null) - { - foreach (var kvp in environmentVariables) - environment[kvp.Key] = kvp.Value; - } - - var timeout = TimeSpan.FromSeconds(420); - var processStartArguments = new StartArguments(binary, arguments) - { - Environment = environment - }; - - var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); - - if (!result.Completed) - throw new Exception($"Timeout while executing {description} exceeded {timeout}"); - - if (result.ExitCode != 0) - throw new Exception( - $"Expected exit code 0 but received ({result.ExitCode}) while executing {description}: {command}"); - - var errorOut = result.ConsoleOut.Where(c => c.Error).ToList(); - - if (errorOut.Any(e => - !string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated")) && - !binary.Contains("plugin") && !binary.Contains("cert")) - throw new Exception( - $"Received error out with exitCode ({result.ExitCode}) while executing {description}: {command}"); - - writer?.WriteDiagnostic( - $"{{{nameof(ExecuteBinary)}}} finished process [{description}] {{{result.ExitCode}}}"); - } - - protected static void CopyFolder(string source, string target, bool overwrite = true) - { - foreach (var sourceDir in Directory.GetDirectories(source, "*", SearchOption.AllDirectories)) - { - var targetDir = sourceDir.Replace(source, target); - Directory.CreateDirectory(targetDir); - } - - foreach (var sourcePath in Directory.GetFiles(source, "*.*", SearchOption.AllDirectories)) - { - var targetPath = sourcePath.Replace(source, target); - if (!overwrite && File.Exists(targetPath)) continue; - File.Copy(sourcePath, targetPath, overwrite); - } - } - - protected static void Extract(string file, string toFolder) - { - if (file.EndsWith(".zip")) ExtractZip(file, toFolder); - else if (file.EndsWith(".tar.gz")) ExtractTarGz(file, toFolder); - else if (file.EndsWith(".tar")) ExtractTar(file, toFolder); - else throw new Exception("Can not extract:" + file); - } - - private static void ExtractTar(string file, string toFolder) - { - using (var inStream = File.OpenRead(file)) - using (var tarArchive = TarArchive.CreateInputTarArchive(inStream, Encoding.UTF8)) - tarArchive.ExtractContents(toFolder); - } - - private static void ExtractTarGz(string file, string toFolder) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - using (var inStream = File.OpenRead(file)) - using (var gzipStream = new GZipInputStream(inStream)) - using (var tarArchive = TarArchive.CreateInputTarArchive(gzipStream, Encoding.UTF8)) - tarArchive.ExtractContents(toFolder); - else - //SharpZipLib loses permissions when untarring - Proc.Exec("tar", "-xvf", file, "-C", toFolder); - } - - private static void ExtractZip(string file, string toFolder) => - ZipFile.ExtractToDirectory(file, toFolder); - } + } + } + } + + private static bool ServerCertificateValidationCallback(object sender, X509Certificate certificate, + X509Chain chain, SslPolicyErrors sslpolicyerrors) => true; + + protected static void WriteFileIfNotExist(string fileLocation, string contents) + { + if (!File.Exists(fileLocation)) File.WriteAllText(fileLocation, contents); + } + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, + params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); + + private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, params string[] arguments) + { + var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; + writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + + var environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} + }; + + if (environmentVariables != null) + { + foreach (var kvp in environmentVariables) + environment[kvp.Key] = kvp.Value; + } + + var timeout = TimeSpan.FromSeconds(420); + var processStartArguments = new StartArguments(binary, arguments) + { + Environment = environment + }; + + var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); + + if (!result.Completed) + throw new Exception($"Timeout while executing {description} exceeded {timeout}"); + + if (result.ExitCode != 0) + throw new Exception( + $"Expected exit code 0 but received ({result.ExitCode}) while executing {description}: {command}"); + + var errorOut = result.ConsoleOut.Where(c => c.Error).ToList(); + + if (errorOut.Any(e => + !string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated")) && + !binary.Contains("plugin") && !binary.Contains("cert")) + throw new Exception( + $"Received error out with exitCode ({result.ExitCode}) while executing {description}: {command}"); + + writer?.WriteDiagnostic( + $"{{{nameof(ExecuteBinary)}}} finished process [{description}] {{{result.ExitCode}}}"); + } + + protected static void CopyFolder(string source, string target, bool overwrite = true) + { + foreach (var sourceDir in Directory.GetDirectories(source, "*", SearchOption.AllDirectories)) + { + var targetDir = sourceDir.Replace(source, target); + Directory.CreateDirectory(targetDir); + } + + foreach (var sourcePath in Directory.GetFiles(source, "*.*", SearchOption.AllDirectories)) + { + var targetPath = sourcePath.Replace(source, target); + if (!overwrite && File.Exists(targetPath)) continue; + File.Copy(sourcePath, targetPath, overwrite); + } + } + + protected static void Extract(string file, string toFolder) + { + if (file.EndsWith(".zip")) ExtractZip(file, toFolder); + else if (file.EndsWith(".tar.gz")) ExtractTarGz(file, toFolder); + else if (file.EndsWith(".tar")) ExtractTar(file, toFolder); + else throw new Exception("Can not extract:" + file); + } + + private static void ExtractTar(string file, string toFolder) + { + using (var inStream = File.OpenRead(file)) + using (var tarArchive = TarArchive.CreateInputTarArchive(inStream, Encoding.UTF8)) + tarArchive.ExtractContents(toFolder); + } + + private static void ExtractTarGz(string file, string toFolder) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + using (var inStream = File.OpenRead(file)) + using (var gzipStream = new GZipInputStream(inStream)) + using (var tarArchive = TarArchive.CreateInputTarArchive(gzipStream, Encoding.UTF8)) + tarArchive.ExtractContents(toFolder); + else + //SharpZipLib loses permissions when untarring + Proc.Exec("tar", "-xvf", file, "-C", toFolder); + } + + private static void ExtractZip(string file, string toFolder) => + ZipFile.ExtractToDirectory(file, toFolder); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs index 5c7862de44..e52f814489 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs @@ -29,23 +29,22 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class CopyCachedOpenSearchInstallation : ClusterComposeTask { - public class CopyCachedOpenSearchInstallation : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + public override void Run(IEphemeralCluster cluster) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; - var fs = cluster.FileSystem; - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - if (!Directory.Exists(cachedOpenSearchHomeFolder)) return; + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + if (!Directory.Exists(cachedOpenSearchHomeFolder)) return; - var source = cachedOpenSearchHomeFolder; - var target = fs.OpenSearchHome; - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CopyCachedOpenSearchInstallation)}}} using cached OPENSEARCH_HOME {{{source}}} and copying it to [{target}]"); - CopyFolder(source, target); - } - } + var source = cachedOpenSearchHomeFolder; + var target = fs.OpenSearchHome; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CopyCachedOpenSearchInstallation)}}} using cached OPENSEARCH_HOME {{{source}}} and copying it to [{target}]"); + CopyFolder(source, target); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs index 35c8e98ec0..f40b7a1965 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs @@ -29,24 +29,23 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class CreateLocalApplicationDirectory : ClusterComposeTask { - public class CreateLocalApplicationDirectory : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var fs = cluster.FileSystem; - if (Directory.Exists(fs.LocalFolder)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateLocalApplicationDirectory)}}} already exists: {{{fs.LocalFolder}}}"); - return; - } + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + if (Directory.Exists(fs.LocalFolder)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateLocalApplicationDirectory)}}} already exists: {{{fs.LocalFolder}}}"); + return; + } - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(CreateLocalApplicationDirectory)}}} creating {{{fs.LocalFolder}}}"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateLocalApplicationDirectory)}}} creating {{{fs.LocalFolder}}}"); - Directory.CreateDirectory(fs.LocalFolder); - } - } + Directory.CreateDirectory(fs.LocalFolder); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs index 6f2131a7a9..682010cce4 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs @@ -31,41 +31,40 @@ using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class DownloadOpenSearchVersion : ClusterComposeTask { - public class DownloadOpenSearchVersion : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - if (cluster.CachingAndCachedHomeExists()) return; + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; - var fs = cluster.FileSystem; - var v = cluster.ClusterConfiguration.Version; - var a = cluster.ClusterConfiguration.Artifact; - var from = v.Artifact(Product.OpenSearch).DownloadUrl; - var to = Path.Combine(fs.LocalFolder, a.Archive); - if (File.Exists(to)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(DownloadOpenSearchVersion)}}} {v} was already downloaded"); - return; - } + var fs = cluster.FileSystem; + var v = cluster.ClusterConfiguration.Version; + var a = cluster.ClusterConfiguration.Artifact; + var from = v.Artifact(Product.OpenSearch).DownloadUrl; + var to = Path.Combine(fs.LocalFolder, a.Archive); + if (File.Exists(to)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} {v} was already downloaded"); + return; + } - if (Environment.GetEnvironmentVariable("OPENSEARCH_DISTRIBUTION") is {} distributionPath) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(DownloadOpenSearchVersion)}}} copying OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); - File.Copy(distributionPath, to); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(DownloadOpenSearchVersion)}}} copied OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); - return; - } + if (Environment.GetEnvironmentVariable("OPENSEARCH_DISTRIBUTION") is { } distributionPath) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} copying OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); + File.Copy(distributionPath, to); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} copied OpenSearch [{v}] from {{{distributionPath}}} {{{to}}}"); + return; + } - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(DownloadOpenSearchVersion)}}} downloading OpenSearch [{v}] from {{{from}}} {{{to}}}"); - DownloadFile(from, to); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(DownloadOpenSearchVersion)}}} downloaded OpenSearch [{v}] from {{{from}}} {{{to}}}"); - } - } + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} downloading OpenSearch [{v}] from {{{from}}} {{{to}}}"); + DownloadFile(from, to); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} downloaded OpenSearch [{v}] from {{{from}}} {{{to}}}"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs index faa8a00641..fb3fecd045 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs @@ -30,34 +30,33 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class EnsureJavaHomeEnvironmentVariableIsSet : ClusterComposeTask { - public class EnsureJavaHomeEnvironmentVariableIsSet : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var fs = cluster.FileSystem; + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; - var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; - var javaHome = Environment.GetEnvironmentVariable(envVarName); - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - var jdkFolder = Path.Combine(cachedOpenSearchHomeFolder, "jdk"); - if (Directory.Exists(jdkFolder)) - { - //prefer bundled jdk - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); - Environment.SetEnvironmentVariable("JAVA_HOME", jdkFolder); - } - else if (!string.IsNullOrWhiteSpace(javaHome)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set; clearing value for process to prefer bundled jdk..."); - Environment.SetEnvironmentVariable(envVarName, null); - } - else - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} {envVarName} is not set proceeding or using default JDK"); - } - } + var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; + var javaHome = Environment.GetEnvironmentVariable(envVarName); + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var jdkFolder = Path.Combine(cachedOpenSearchHomeFolder, "jdk"); + if (Directory.Exists(jdkFolder)) + { + //prefer bundled jdk + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); + Environment.SetEnvironmentVariable("JAVA_HOME", jdkFolder); + } + else if (!string.IsNullOrWhiteSpace(javaHome)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set; clearing value for process to prefer bundled jdk..."); + Environment.SetEnvironmentVariable(envVarName, null); + } + else + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} {envVarName} is not set proceeding or using default JDK"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index 9540c07edb..ac14dd0e80 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -33,46 +33,45 @@ using OpenSearch.Stack.ArtifactsApi.Products; using SemanticVersioning; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class InitialConfiguration : ClusterComposeTask { - public class InitialConfiguration : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var fs = cluster.FileSystem; + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; - var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); - var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); - var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); + var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); + var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); + var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); - if (!Directory.Exists(pluginSecurity)) - return; + if (!Directory.Exists(pluginSecurity)) + return; - var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); + var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); - const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; - var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); + const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); - cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); + cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); - if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; + if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; - var env = new Dictionary(); - var args = new List { securityInstallDemoConfig, "-y", "-i" }; + var env = new Dictionary(); + var args = new List { securityInstallDemoConfig, "-y", "-i" }; - if (isNewDemoScript) - { - env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); - args.Add("-t"); - } + if (isNewDemoScript) + { + env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); + args.Add("-t"); + } - ExecuteBinary( - cluster.ClusterConfiguration, - cluster.Writer, - "/bin/bash", - "install security plugin demo configuration", - env, - args.ToArray()); - } - } + ExecuteBinary( + cluster.ClusterConfiguration, + cluster.Writer, + "/bin/bash", + "install security plugin demo configuration", + env, + args.ToArray()); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index 3225476353..fe64dc7a1a 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -40,180 +40,179 @@ using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class InstallPlugins : ClusterComposeTask { - public class InstallPlugins : ClusterComposeTask - { - private static readonly HttpClient HttpClient = new(); - - public override void Run(IEphemeralCluster cluster) - { - if (cluster.CachingAndCachedHomeExists()) return; - - var v = cluster.ClusterConfiguration.Version; - - var fs = cluster.FileSystem; - var requiredPlugins = cluster.ClusterConfiguration.Plugins; - - if (cluster.ClusterConfiguration.ValidatePluginsToInstall) - { - var invalidPlugins = requiredPlugins - .Where(p => !p.IsValid(v)) - .Select(p => p.SubProductName).ToList(); - if (invalidPlugins.Any()) - throw new OpenSearchCleanExitException( - $"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} "); - } - - foreach (var plugin in requiredPlugins) - { - var includedByDefault = plugin.IsIncludedOutOfTheBox(v); - if (includedByDefault) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}"); - continue; - } - - var validForCurrentVersion = plugin.IsValid(v); - if (!validForCurrentVersion) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}"); - continue; - } - - var alreadyInstalled = AlreadyInstalled(fs, plugin.SubProductName); - if (alreadyInstalled) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] already installed"); - continue; - } - - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - - var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); - - if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); - - var env = new Dictionary - { - { fs.ConfigEnvironmentVariableName, homeConfigPath } - }; - - ExecuteBinary( - cluster.ClusterConfiguration, - cluster.Writer, - fs.PluginBinary, - $"install opensearch plugin: {plugin.SubProductName}", - env, - "install", "--batch", GetPluginLocation(plugin, v)); - - CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); - } - - cluster.Writer?.WriteDiagnostic($"{{{nameof(InstallPlugins)}}} all plugins installed"); - } - - private static string GetPluginLocation(OpenSearchPlugin plugin, OpenSearchVersion v) - { - var pluginName = plugin.SubProductName; - var versionVariants = new[] - { - v.ToString(), - $"{v.BaseVersion()}.0{(v.IsPreRelease ? $"-{v.PreRelease}" : string.Empty)}", - }; - - if (Environment.GetEnvironmentVariable("OPENSEARCH_PLUGINS_DIRECTORY") is { } pluginsDirectory) - { - foreach (var versionVariant in versionVariants) - { - var pluginFile = Path.Combine(pluginsDirectory, $"{pluginName}-{versionVariant}.zip"); - if (File.Exists(pluginFile)) - { - return new UriBuilder("file",string.Empty) - { - Path = pluginFile - .Replace("%",$"%{(int)'%':X2}") - .Replace("[",$"%{(int)'[':X2}") - .Replace("]",$"%{(int)']':X2}"), - } - .Uri - .AbsoluteUri; - } - } - } - - if (v.IsSnapshot) - return DeterminePluginSnapshotUrl(pluginName, versionVariants); - - return pluginName; - } - - private static string DeterminePluginSnapshotUrl(string pluginName, string[] versionVariants) - { - try - { - var baseUrl = $"https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/{pluginName}"; - - var versionConditions = string.Join(" or ", versionVariants.Select(v => $".='{v}'")); - var version = SelectNodeWithinRemoteXml( - $"{baseUrl}/maven-metadata.xml", - $"metadata/versioning/versions/version[{versionConditions}]") - .InnerText; - - var versionUrl = $"{baseUrl}/{version}"; - - var snapshotVersion = SelectNodeWithinRemoteXml( - $"{versionUrl}/maven-metadata.xml", - "metadata/versioning/snapshotVersions/snapshotVersion[extension='zip']/value") - .InnerText; - - return $"{versionUrl}/{pluginName}-{snapshotVersion}.zip"; - } - catch (Exception e) - { - throw new Exception($"Could not determine snapshot url for plugin `{pluginName}` at versions `{string.Join(", ", versionVariants)}`", e); - } - } - - private static XmlNode SelectNodeWithinRemoteXml(string url, [LanguageInjection("XPath")] string xPath) - { - var task = Task.Run(async () => - { - var msg = await HttpClient.GetAsync(url); - msg.EnsureSuccessStatusCode(); - var xml = await msg.Content.ReadAsStringAsync(); - var doc = new XmlDocument(); - doc.LoadXml(xml); - return doc.SelectSingleNode(xPath) ?? throw new Exception($"Could not find node matching XPath: `{xPath}` within `{xml}`"); - }); - task.Wait(); - return task.Result; - } - - private static void CopyConfigDirectoryToHomeCacheConfigDirectory( - IEphemeralCluster cluster, OpenSearchPlugin plugin) - { - if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; - var fs = cluster.FileSystem; - var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); - var configTarget = Path.Combine(cachedOpenSearchHomeFolder, "config"); - - var configPluginPath = Path.Combine(fs.ConfigPath, plugin.SubProductName); - var configPluginPathCached = Path.Combine(configTarget, plugin.SubProductName); - if (!Directory.Exists(configPluginPath) || Directory.Exists(configPluginPathCached)) return; - - Directory.CreateDirectory(configPluginPathCached); - CopyFolder(configPluginPath, configPluginPathCached); - } - - private static bool AlreadyInstalled(INodeFileSystem fileSystem, string folderName) - { - var pluginFolder = Path.Combine(fileSystem.OpenSearchHome, "plugins", folderName); - return Directory.Exists(pluginFolder); - } - } + private static readonly HttpClient HttpClient = new(); + + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; + + var v = cluster.ClusterConfiguration.Version; + + var fs = cluster.FileSystem; + var requiredPlugins = cluster.ClusterConfiguration.Plugins; + + if (cluster.ClusterConfiguration.ValidatePluginsToInstall) + { + var invalidPlugins = requiredPlugins + .Where(p => !p.IsValid(v)) + .Select(p => p.SubProductName).ToList(); + if (invalidPlugins.Any()) + throw new OpenSearchCleanExitException( + $"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} "); + } + + foreach (var plugin in requiredPlugins) + { + var includedByDefault = plugin.IsIncludedOutOfTheBox(v); + if (includedByDefault) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}"); + continue; + } + + var validForCurrentVersion = plugin.IsValid(v); + if (!validForCurrentVersion) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}"); + continue; + } + + var alreadyInstalled = AlreadyInstalled(fs, plugin.SubProductName); + if (alreadyInstalled) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] already installed"); + continue; + } + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); + + var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); + + if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); + + var env = new Dictionary + { + { fs.ConfigEnvironmentVariableName, homeConfigPath } + }; + + ExecuteBinary( + cluster.ClusterConfiguration, + cluster.Writer, + fs.PluginBinary, + $"install opensearch plugin: {plugin.SubProductName}", + env, + "install", "--batch", GetPluginLocation(plugin, v)); + + CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); + } + + cluster.Writer?.WriteDiagnostic($"{{{nameof(InstallPlugins)}}} all plugins installed"); + } + + private static string GetPluginLocation(OpenSearchPlugin plugin, OpenSearchVersion v) + { + var pluginName = plugin.SubProductName; + var versionVariants = new[] + { + v.ToString(), + $"{v.BaseVersion()}.0{(v.IsPreRelease ? $"-{v.PreRelease}" : string.Empty)}", + }; + + if (Environment.GetEnvironmentVariable("OPENSEARCH_PLUGINS_DIRECTORY") is { } pluginsDirectory) + { + foreach (var versionVariant in versionVariants) + { + var pluginFile = Path.Combine(pluginsDirectory, $"{pluginName}-{versionVariant}.zip"); + if (File.Exists(pluginFile)) + { + return new UriBuilder("file", string.Empty) + { + Path = pluginFile + .Replace("%", $"%{(int)'%':X2}") + .Replace("[", $"%{(int)'[':X2}") + .Replace("]", $"%{(int)']':X2}"), + } + .Uri + .AbsoluteUri; + } + } + } + + if (v.IsSnapshot) + return DeterminePluginSnapshotUrl(pluginName, versionVariants); + + return pluginName; + } + + private static string DeterminePluginSnapshotUrl(string pluginName, string[] versionVariants) + { + try + { + var baseUrl = $"https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/{pluginName}"; + + var versionConditions = string.Join(" or ", versionVariants.Select(v => $".='{v}'")); + var version = SelectNodeWithinRemoteXml( + $"{baseUrl}/maven-metadata.xml", + $"metadata/versioning/versions/version[{versionConditions}]") + .InnerText; + + var versionUrl = $"{baseUrl}/{version}"; + + var snapshotVersion = SelectNodeWithinRemoteXml( + $"{versionUrl}/maven-metadata.xml", + "metadata/versioning/snapshotVersions/snapshotVersion[extension='zip']/value") + .InnerText; + + return $"{versionUrl}/{pluginName}-{snapshotVersion}.zip"; + } + catch (Exception e) + { + throw new Exception($"Could not determine snapshot url for plugin `{pluginName}` at versions `{string.Join(", ", versionVariants)}`", e); + } + } + + private static XmlNode SelectNodeWithinRemoteXml(string url, [LanguageInjection("XPath")] string xPath) + { + var task = Task.Run(async () => + { + var msg = await HttpClient.GetAsync(url); + msg.EnsureSuccessStatusCode(); + var xml = await msg.Content.ReadAsStringAsync(); + var doc = new XmlDocument(); + doc.LoadXml(xml); + return doc.SelectSingleNode(xPath) ?? throw new Exception($"Could not find node matching XPath: `{xPath}` within `{xml}`"); + }); + task.Wait(); + return task.Result; + } + + private static void CopyConfigDirectoryToHomeCacheConfigDirectory( + IEphemeralCluster cluster, OpenSearchPlugin plugin) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var configTarget = Path.Combine(cachedOpenSearchHomeFolder, "config"); + + var configPluginPath = Path.Combine(fs.ConfigPath, plugin.SubProductName); + var configPluginPathCached = Path.Combine(configTarget, plugin.SubProductName); + if (!Directory.Exists(configPluginPath) || Directory.Exists(configPluginPathCached)) return; + + Directory.CreateDirectory(configPluginPathCached); + CopyFolder(configPluginPath, configPluginPathCached); + } + + private static bool AlreadyInstalled(INodeFileSystem fileSystem, string folderName) + { + var pluginFolder = Path.Combine(fileSystem.OpenSearchHome, "plugins", folderName); + return Directory.Exists(pluginFolder); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs index 42248300c5..91fb616cfa 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs @@ -31,46 +31,45 @@ using OpenSearch.OpenSearch.Managed.ConsoleWriters; using static OpenSearch.OpenSearch.Ephemeral.ClusterFeatures; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class PrintConfiguration : ClusterComposeTask { - public class PrintConfiguration : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var c = cluster.ClusterConfiguration; - var version = c.Version; + public override void Run(IEphemeralCluster cluster) + { + var c = cluster.ClusterConfiguration; + var version = c.Version; - string F(ClusterFeatures feature) - { - return c.Features.HasFlag(feature) ? Enum.GetName(typeof(ClusterFeatures), feature) : string.Empty; - } + string F(ClusterFeatures feature) + { + return c.Features.HasFlag(feature) ? Enum.GetName(typeof(ClusterFeatures), feature) : string.Empty; + } - var features = string.Join("|", - new[] { F(SSL)}.Where(v => !string.IsNullOrWhiteSpace(v))); - features = string.IsNullOrWhiteSpace(features) ? "None" : features; - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} starting {{{version}}} with features [{features}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NumberOfNodes)}}} [{c.NumberOfNodes}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ClusterName)}}} [{c.ClusterName}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSsl)}}} [{c.EnableSsl}]"); - cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.Plugins)}}} [{c.Plugins}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.CacheOpenSearchHomeInstallation)}}} [{c.CacheOpenSearchHomeInstallation}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ShowOpenSearchOutputAfterStarted)}}} [{c.ShowOpenSearchOutputAfterStarted}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ValidatePluginsToInstall)}}} [{c.ValidatePluginsToInstall}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.PrintYamlFilesInConfigFolder)}}} [{c.PrintYamlFilesInConfigFolder}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.SkipBuiltInAfterStartTasks)}}} [{c.SkipBuiltInAfterStartTasks}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.HttpFiddlerAware)}}} [{c.HttpFiddlerAware}]"); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NoCleanupAfterNodeStopped)}}} [{c.NoCleanupAfterNodeStopped}]"); - } - } + var features = string.Join("|", + new[] { F(SSL) }.Where(v => !string.IsNullOrWhiteSpace(v))); + features = string.IsNullOrWhiteSpace(features) ? "None" : features; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} starting {{{version}}} with features [{features}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NumberOfNodes)}}} [{c.NumberOfNodes}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ClusterName)}}} [{c.ClusterName}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSsl)}}} [{c.EnableSsl}]"); + cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.Plugins)}}} [{c.Plugins}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.CacheOpenSearchHomeInstallation)}}} [{c.CacheOpenSearchHomeInstallation}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ShowOpenSearchOutputAfterStarted)}}} [{c.ShowOpenSearchOutputAfterStarted}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ValidatePluginsToInstall)}}} [{c.ValidatePluginsToInstall}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.PrintYamlFilesInConfigFolder)}}} [{c.PrintYamlFilesInConfigFolder}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.SkipBuiltInAfterStartTasks)}}} [{c.SkipBuiltInAfterStartTasks}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.HttpFiddlerAware)}}} [{c.HttpFiddlerAware}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NoCleanupAfterNodeStopped)}}} [{c.NoCleanupAfterNodeStopped}]"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs index 82a28dac64..7f9c207472 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs @@ -30,24 +30,23 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class SetOpenSearchBundledJdkJavaHome : ClusterComposeTask { - public class SetOpenSearchBundledJdkJavaHome : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var fs = cluster.FileSystem; - var jdkFolder = Path.Combine(fs.OpenSearchHome, "jdk"); - if (Directory.Exists(jdkFolder)) - { - var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); - Environment.SetEnvironmentVariable(envVarName, jdkFolder); - } - else - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [No bundled jdk found] looked in: {{{jdkFolder}}} "); - } - } + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + var jdkFolder = Path.Combine(fs.OpenSearchHome, "jdk"); + if (Directory.Exists(jdkFolder)) + { + var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); + Environment.SetEnvironmentVariable(envVarName, jdkFolder); + } + else + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [No bundled jdk found] looked in: {{{jdkFolder}}} "); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs index 81a80693f4..0bff53785a 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs @@ -29,40 +29,39 @@ using System.IO; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; + +public class UnzipOpenSearch : ClusterComposeTask { - public class UnzipOpenSearch : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - if (cluster.CachingAndCachedHomeExists()) return; + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; - var fs = cluster.FileSystem; - var v = cluster.ClusterConfiguration.Version; - var a = cluster.ClusterConfiguration.Artifact; - if (Directory.Exists(fs.OpenSearchHome)) - { - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(UnzipOpenSearch)}}} skipping [{fs.OpenSearchHome}] already exists"); - return; - } + var fs = cluster.FileSystem; + var v = cluster.ClusterConfiguration.Version; + var a = cluster.ClusterConfiguration.Artifact; + if (Directory.Exists(fs.OpenSearchHome)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} skipping [{fs.OpenSearchHome}] already exists"); + return; + } - var from = Path.Combine(fs.LocalFolder, a.Archive); - var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); - if (!Directory.Exists(extractedFolder)) - { - cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipOpenSearch)}}} unzipping version [{v}] {{{from}}}"); - Extract(from, fs.LocalFolder); + var from = Path.Combine(fs.LocalFolder, a.Archive); + var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); + if (!Directory.Exists(extractedFolder)) + { + cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipOpenSearch)}}} unzipping version [{v}] {{{from}}}"); + Extract(from, fs.LocalFolder); - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(UnzipOpenSearch)}}} extracted version [{v}] to {{{fs.LocalFolder}}}"); - } + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} extracted version [{v}] to {{{fs.LocalFolder}}}"); + } - if (extractedFolder == fs.OpenSearchHome) return; + if (extractedFolder == fs.OpenSearchHome) return; - cluster.Writer?.WriteDiagnostic( - $"{{{nameof(UnzipOpenSearch)}}} Copying extracted folder {{{extractedFolder}}} => {fs.OpenSearchHome}"); - CopyFolder(extractedFolder, fs.OpenSearchHome); - } - } + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} Copying extracted folder {{{extractedFolder}}} => {fs.OpenSearchHome}"); + CopyFolder(extractedFolder, fs.OpenSearchHome); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs index 1778867b89..d255378a1e 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs @@ -30,18 +30,17 @@ using System.Net; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; + +public class ValidateClusterStateTask : ClusterComposeTask { - public class ValidateClusterStateTask : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - cluster.Writer.WriteDiagnostic( - $"{{{nameof(ValidateClusterStateTask)}}} waiting cluster to go into yellow health state"); - var healthyResponse = Get(cluster, "_cluster/health", "wait_for_status=yellow&timeout=20s"); - if (healthyResponse == null || healthyResponse.StatusCode != HttpStatusCode.OK) - throw new Exception( - $"Cluster health waiting for status yellow failed after 20s {GetResponseException(healthyResponse)}"); - } - } + public override void Run(IEphemeralCluster cluster) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(ValidateClusterStateTask)}}} waiting cluster to go into yellow health state"); + var healthyResponse = Get(cluster, "_cluster/health", "wait_for_status=yellow&timeout=20s"); + if (healthyResponse == null || healthyResponse.StatusCode != HttpStatusCode.OK) + throw new Exception( + $"Cluster health waiting for status yellow failed after 20s {GetResponseException(healthyResponse)}"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs index 045a53b263..9b5e80129c 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs @@ -32,37 +32,36 @@ using OpenSearch.OpenSearch.Managed.ConsoleWriters; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; + +public class ValidatePluginsTask : ClusterComposeTask { - public class ValidatePluginsTask : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - var v = cluster.ClusterConfiguration.Version; - var requestPlugins = cluster.ClusterConfiguration.Plugins - .Where(p => p.IsValid(v)) - .Where(p => !p.IsIncludedOutOfTheBox(v)) - .Select(p => p.GetExistsMoniker(v)) - .ToList(); - if (!requestPlugins.Any()) return; + public override void Run(IEphemeralCluster cluster) + { + var v = cluster.ClusterConfiguration.Version; + var requestPlugins = cluster.ClusterConfiguration.Plugins + .Where(p => p.IsValid(v)) + .Where(p => !p.IsIncludedOutOfTheBox(v)) + .Select(p => p.GetExistsMoniker(v)) + .ToList(); + if (!requestPlugins.Any()) return; - cluster.Writer.WriteDiagnostic( - $"{{{nameof(ValidatePluginsTask)}}} validating the cluster is running the requested plugins"); - var catPlugins = Get(cluster, "_cat/plugins", "h=component"); - if (catPlugins == null || !catPlugins.IsSuccessStatusCode) - throw new Exception( - $"Calling _cat/plugins did not result in an OK response {GetResponseException(catPlugins)}"); + cluster.Writer.WriteDiagnostic( + $"{{{nameof(ValidatePluginsTask)}}} validating the cluster is running the requested plugins"); + var catPlugins = Get(cluster, "_cat/plugins", "h=component"); + if (catPlugins == null || !catPlugins.IsSuccessStatusCode) + throw new Exception( + $"Calling _cat/plugins did not result in an OK response {GetResponseException(catPlugins)}"); - var installedPlugins = GetResponseString(catPlugins) - .Split(new[] {'\n'}, StringSplitOptions.RemoveEmptyEntries).ToList(); + var installedPlugins = GetResponseString(catPlugins) + .Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).ToList(); - var missingPlugins = requestPlugins.Except(installedPlugins).ToList(); - if (!missingPlugins.Any()) return; + var missingPlugins = requestPlugins.Except(installedPlugins).ToList(); + if (!missingPlugins.Any()) return; - var missingString = string.Join(", ", missingPlugins); - var pluginsString = string.Join(", ", installedPlugins); - throw new Exception( - $"Already running opensearch missed the following plugin(s): {missingString} currently installed: {pluginsString}."); - } - } + var missingString = string.Join(", ", missingPlugins); + var pluginsString = string.Join(", ", installedPlugins); + throw new Exception( + $"Already running opensearch missed the following plugin(s): {missingString} currently installed: {pluginsString}."); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs index 3dba4bd000..377f5581b7 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs @@ -32,45 +32,44 @@ using System.Threading; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; + +public class ValidateRunningVersion : ClusterComposeTask { - public class ValidateRunningVersion : ClusterComposeTask - { - public override void Run(IEphemeralCluster cluster) - { - void WriteDiagnostic(string message) => - cluster.Writer?.WriteDiagnostic($"{{{nameof(ValidateRunningVersion)}}} {message}"); - var requestedVersion = cluster.ClusterConfiguration.Version; + public override void Run(IEphemeralCluster cluster) + { + void WriteDiagnostic(string message) => + cluster.Writer?.WriteDiagnostic($"{{{nameof(ValidateRunningVersion)}}} {message}"); + var requestedVersion = cluster.ClusterConfiguration.Version; - WriteDiagnostic($"validating the cluster is running the requested version: {requestedVersion}"); + WriteDiagnostic($"validating the cluster is running the requested version: {requestedVersion}"); - HttpResponseMessage catNodes = null; - var retryCount = 4; - var initialRetryWait = 5; - foreach (var retry in Enumerable.Range(1, retryCount)) - { - catNodes = Get(cluster, "_cat/nodes", "h=version"); - if (catNodes.IsSuccessStatusCode) break; - var retryWait = TimeSpan.FromSeconds(initialRetryWait * retry); - WriteDiagnostic($"{catNodes.StatusCode} response for GET _cat/nodes. Waiting to retry #{retry}"); - Thread.Sleep(retryWait); - } - if (catNodes is not {IsSuccessStatusCode: true}) - { - throw new Exception( - $"Calling _cat/nodes for version checking did not result in an OK response {GetResponseException(catNodes)}"); - } + HttpResponseMessage catNodes = null; + var retryCount = 4; + var initialRetryWait = 5; + foreach (var retry in Enumerable.Range(1, retryCount)) + { + catNodes = Get(cluster, "_cat/nodes", "h=version"); + if (catNodes.IsSuccessStatusCode) break; + var retryWait = TimeSpan.FromSeconds(initialRetryWait * retry); + WriteDiagnostic($"{catNodes.StatusCode} response for GET _cat/nodes. Waiting to retry #{retry}"); + Thread.Sleep(retryWait); + } + if (catNodes is not { IsSuccessStatusCode: true }) + { + throw new Exception( + $"Calling _cat/nodes for version checking did not result in an OK response {GetResponseException(catNodes)}"); + } - var nodeVersions = GetResponseString(catNodes).Split(new[] {'\n'}, StringSplitOptions.RemoveEmptyEntries) - .ToList(); + var nodeVersions = GetResponseString(catNodes).Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries) + .ToList(); - var anchorVersion = $"{requestedVersion.Major}.{requestedVersion.Minor}.{requestedVersion.Patch}"; - var allOnRequestedVersion = nodeVersions.All(v => v.Trim() == anchorVersion); - if (!allOnRequestedVersion) - { - throw new Exception( - $"Not all the running nodes in the cluster are on requested version: {anchorVersion} received: {string.Join(", ", nodeVersions)}"); - } - } - } + var anchorVersion = $"{requestedVersion.Major}.{requestedVersion.Minor}.{requestedVersion.Patch}"; + var allOnRequestedVersion = nodeVersions.All(v => v.Trim() == anchorVersion); + if (!allOnRequestedVersion) + { + throw new Exception( + $"Not all the running nodes in the cluster are on requested version: {anchorVersion} received: {string.Join(", ", nodeVersions)}"); + } + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs index 3cdb74e966..b23d06cfba 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs @@ -38,187 +38,186 @@ using OpenSearch.Stack.ArtifactsApi; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed +namespace OpenSearch.OpenSearch.Managed; + +public interface ICluster : IDisposable + where TConfiguration : IClusterConfiguration +{ + string ClusterMoniker { get; } + TConfiguration ClusterConfiguration { get; } + INodeFileSystem FileSystem { get; } + bool Started { get; } + ReadOnlyCollection Nodes { get; } + IConsoleLineHandler Writer { get; } + + IDisposable Start(); + + IDisposable Start(TimeSpan waitForStarted); + + IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted); +} + + +public abstract class ClusterBase : ClusterBase +{ + protected ClusterBase(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } +} + +public abstract class ClusterBase : ICluster + where TConfiguration : IClusterConfiguration { - public interface ICluster : IDisposable - where TConfiguration : IClusterConfiguration - { - string ClusterMoniker { get; } - TConfiguration ClusterConfiguration { get; } - INodeFileSystem FileSystem { get; } - bool Started { get; } - ReadOnlyCollection Nodes { get; } - IConsoleLineHandler Writer { get; } - - IDisposable Start(); - - IDisposable Start(TimeSpan waitForStarted); - - IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted); - } - - - public abstract class ClusterBase : ClusterBase - { - protected ClusterBase(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) - { - } - } - - public abstract class ClusterBase : ICluster - where TConfiguration : IClusterConfiguration - { - private Action _defaultConfigSelector = (n, i) => { }; - - protected ClusterBase(TConfiguration clusterConfiguration) - { - ClusterConfiguration = clusterConfiguration; - ClusterMoniker = GetType().Name.Replace("Cluster", ""); - - NodeConfiguration Modify(NodeConfiguration n, int p) - { - ModifyNodeConfiguration(n, p); - return n; - } - - var nodes = - (from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber, - ClusterConfiguration.NumberOfNodes) - let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker) - { - ShowOpenSearchOutputAfterStarted = - clusterConfiguration.ShowOpenSearchOutputAfterStarted, - } - let node = new OpenSearchNode(Modify(config, port)) - { - AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1, - } - select node).ToList(); - - var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName)); - foreach (var node in nodes) - node.NodeConfiguration.InitialMasterNodes(initialMasterNodes); - - Nodes = new ReadOnlyCollection(nodes); - } - - /// - /// A short name to identify the cluster defaults to the subclass name with Cluster - /// removed - /// - public virtual string ClusterMoniker { get; } - - public TConfiguration ClusterConfiguration { get; } - public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; - - public ReadOnlyCollection Nodes { get; } - public bool Started { get; private set; } - public IConsoleLineHandler Writer { get; private set; } = NoopConsoleLineWriter.Instance; - - public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); - - public IDisposable Start(TimeSpan waitForStarted) - { - var nodes = Nodes.Select(n => n.NodeConfiguration.DesiredNodeName).ToArray(); - var lineHighlightWriter = new LineHighlightWriter(nodes, LineOutParser.OpenSearch); - return Start(lineHighlightWriter, waitForStarted); - } - - public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) - { - Writer = writer ?? NoopConsoleLineWriter.Instance; - - OnBeforeStart(); - - var subscriptions = new Subscriptions(); - - foreach (var node in Nodes) - { - subscriptions.Add(node.SubscribeLines(writer)); - if (node.WaitForStarted(waitForStarted)) continue; - - var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); - writer?.WriteError( - $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start after {waitForStarted}"); - throw new AggregateException($"Not all nodes started after waiting {waitForStarted}", nodeExceptions); - } - - Started = Nodes.All(n => n.NodeStarted); - if (!Started) - { - var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); - var message = $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start successfully"; - var seeLogsMessage = SeeLogsMessage(message); - writer?.WriteError(seeLogsMessage); - throw new AggregateException(seeLogsMessage, nodeExceptions); - } - - try - { - OnAfterStarted(); - SeedCluster(); - } - catch (Exception e) - { - writer?.WriteError(e.ToString()); - throw; - } - - return subscriptions; - } - - public void Dispose() - { - Started = false; - foreach (var node in Nodes) - node?.Dispose(); - - OnDispose(); - } - - protected virtual void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) - { - } - - protected virtual void SeedCluster() - { - } - - - protected virtual string SeeLogsMessage(string message) - { - var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); - return $"{message} see {log} to diagnose the issue"; - } - - public void WaitForExit(TimeSpan waitForCompletion) - { - foreach (var node in Nodes) - node.WaitForCompletion(waitForCompletion); - } - - protected virtual void OnAfterStarted() - { - } - - protected virtual void OnBeforeStart() - { - } - - protected virtual void OnDispose() - { - } - - private class Subscriptions : IDisposable - { - private List Disposables { get; } = new List(); - - public void Dispose() - { - foreach (var d in Disposables) d.Dispose(); - } - - internal void Add(IDisposable disposable) => Disposables.Add(disposable); - } - } + private Action _defaultConfigSelector = (n, i) => { }; + + protected ClusterBase(TConfiguration clusterConfiguration) + { + ClusterConfiguration = clusterConfiguration; + ClusterMoniker = GetType().Name.Replace("Cluster", ""); + + NodeConfiguration Modify(NodeConfiguration n, int p) + { + ModifyNodeConfiguration(n, p); + return n; + } + + var nodes = + (from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber, + ClusterConfiguration.NumberOfNodes) + let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker) + { + ShowOpenSearchOutputAfterStarted = + clusterConfiguration.ShowOpenSearchOutputAfterStarted, + } + let node = new OpenSearchNode(Modify(config, port)) + { + AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1, + } + select node).ToList(); + + var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName)); + foreach (var node in nodes) + node.NodeConfiguration.InitialMasterNodes(initialMasterNodes); + + Nodes = new ReadOnlyCollection(nodes); + } + + /// + /// A short name to identify the cluster defaults to the subclass name with Cluster + /// removed + /// + public virtual string ClusterMoniker { get; } + + public TConfiguration ClusterConfiguration { get; } + public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; + + public ReadOnlyCollection Nodes { get; } + public bool Started { get; private set; } + public IConsoleLineHandler Writer { get; private set; } = NoopConsoleLineWriter.Instance; + + public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); + + public IDisposable Start(TimeSpan waitForStarted) + { + var nodes = Nodes.Select(n => n.NodeConfiguration.DesiredNodeName).ToArray(); + var lineHighlightWriter = new LineHighlightWriter(nodes, LineOutParser.OpenSearch); + return Start(lineHighlightWriter, waitForStarted); + } + + public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) + { + Writer = writer ?? NoopConsoleLineWriter.Instance; + + OnBeforeStart(); + + var subscriptions = new Subscriptions(); + + foreach (var node in Nodes) + { + subscriptions.Add(node.SubscribeLines(writer)); + if (node.WaitForStarted(waitForStarted)) continue; + + var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); + writer?.WriteError( + $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start after {waitForStarted}"); + throw new AggregateException($"Not all nodes started after waiting {waitForStarted}", nodeExceptions); + } + + Started = Nodes.All(n => n.NodeStarted); + if (!Started) + { + var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); + var message = $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start successfully"; + var seeLogsMessage = SeeLogsMessage(message); + writer?.WriteError(seeLogsMessage); + throw new AggregateException(seeLogsMessage, nodeExceptions); + } + + try + { + OnAfterStarted(); + SeedCluster(); + } + catch (Exception e) + { + writer?.WriteError(e.ToString()); + throw; + } + + return subscriptions; + } + + public void Dispose() + { + Started = false; + foreach (var node in Nodes) + node?.Dispose(); + + OnDispose(); + } + + protected virtual void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + } + + protected virtual void SeedCluster() + { + } + + + protected virtual string SeeLogsMessage(string message) + { + var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); + return $"{message} see {log} to diagnose the issue"; + } + + public void WaitForExit(TimeSpan waitForCompletion) + { + foreach (var node in Nodes) + node.WaitForCompletion(waitForCompletion); + } + + protected virtual void OnAfterStarted() + { + } + + protected virtual void OnBeforeStart() + { + } + + protected virtual void OnDispose() + { + } + + private class Subscriptions : IDisposable + { + private List Disposables { get; } = new List(); + + public void Dispose() + { + foreach (var d in Disposables) d.Dispose(); + } + + internal void Add(IDisposable disposable) => Disposables.Add(disposable); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs index e74d8bd116..0c719dab7c 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs @@ -33,137 +33,136 @@ using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Managed.Configuration +namespace OpenSearch.OpenSearch.Managed.Configuration; + +public interface IClusterConfiguration where TFileSystem : INodeFileSystem +{ + TFileSystem FileSystem { get; } + + string ClusterName { get; } + NodeSettings DefaultNodeSettings { get; } + OpenSearchVersion Version { get; } + int NumberOfNodes { get; } + int StartingPortNumber { get; set; } + bool NoCleanupAfterNodeStopped { get; set; } + + bool ShowOpenSearchOutputAfterStarted { get; set; } + bool CacheOpenSearchHomeInstallation { get; set; } + + string CreateNodeName(int? node); +} + +public class ClusterConfiguration : ClusterConfiguration +{ + public ClusterConfiguration(OpenSearchVersion version, string openSearchHome, int numberOfNodes = 1) + : base(version, (v, s) => new NodeFileSystem(v, openSearchHome), numberOfNodes, null) { } + + public ClusterConfiguration(OpenSearchVersion version, Func fileSystem = null, + int numberOfNodes = 1, + string clusterName = null + ) + : base(version, fileSystem ?? ((v, s) => new NodeFileSystem(v, s)), numberOfNodes, clusterName) { } +} + +public class ClusterConfiguration : IClusterConfiguration + where TFileSystem : INodeFileSystem { - public interface IClusterConfiguration where TFileSystem : INodeFileSystem - { - TFileSystem FileSystem { get; } - - string ClusterName { get; } - NodeSettings DefaultNodeSettings { get; } - OpenSearchVersion Version { get; } - int NumberOfNodes { get; } - int StartingPortNumber { get; set; } - bool NoCleanupAfterNodeStopped { get; set; } - - bool ShowOpenSearchOutputAfterStarted { get; set; } - bool CacheOpenSearchHomeInstallation { get; set; } - - string CreateNodeName(int? node); - } - - public class ClusterConfiguration : ClusterConfiguration - { - public ClusterConfiguration(OpenSearchVersion version, string openSearchHome, int numberOfNodes = 1) - : base(version, (v, s) => new NodeFileSystem(v, openSearchHome), numberOfNodes, null) { } - - public ClusterConfiguration(OpenSearchVersion version, Func fileSystem = null, - int numberOfNodes = 1, - string clusterName = null - ) - : base(version, fileSystem ?? ((v, s) => new NodeFileSystem(v, s)), numberOfNodes, clusterName) { } - } - - public class ClusterConfiguration : IClusterConfiguration - where TFileSystem : INodeFileSystem - { - /// - /// Creates a new instance of a configuration for an OpenSearch cluster. - /// - /// The version of OpenSearch - /// - /// A delegate to create the instance of . - /// Passed the OpenSearch version and the Cluster name - /// - /// The number of nodes in the cluster - /// The name of the cluster - public ClusterConfiguration(OpenSearchVersion version, Func fileSystem, - int numberOfNodes = 1, string clusterName = null - ) - { - if (fileSystem == null) throw new ArgumentException(nameof(fileSystem)); - - ClusterName = clusterName; - Version = version; - Artifact = version.Artifact(Product.From("opensearch")); - FileSystem = fileSystem(Version, ClusterName); - NumberOfNodes = numberOfNodes; - - var fs = FileSystem; - Add("node.max_local_storage_nodes", numberOfNodes.ToString(CultureInfo.InvariantCulture), "1.0.0"); - - Add("cluster.name", clusterName); - Add("path.repo", fs.RepositoryPath); - Add("path.data", fs.DataPath); - var logsPathDefault = Path.Combine(fs.OpenSearchHome, "logs"); - if (logsPathDefault != fs.LogsPath) Add("path.logs", fs.LogsPath); - } - - public Artifact Artifact { get; } - - public string JavaHomeEnvironmentVariable => "JAVA_HOME"; - - /// Will print the contents of all the yaml files when starting the cluster up, great for debugging purposes - public bool PrintYamlFilesInConfigFolder { get; set; } - - public string ClusterName { get; } - public OpenSearchVersion Version { get; } - public TFileSystem FileSystem { get; } - public int NumberOfNodes { get; } - public int StartingPortNumber { get; set; } = 9200; - public bool NoCleanupAfterNodeStopped { get; set; } - - /// - /// Whether should continue to write output to console after it has started. - /// Defaults to true - /// - public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; - - public bool CacheOpenSearchHomeInstallation { get; set; } - - /// The node settings to apply to each started node - public NodeSettings DefaultNodeSettings { get; } = new NodeSettings(); - - /// - /// Creates a node name - /// - public virtual string CreateNodeName(int? node) => - node.HasValue ? $"managed-opensearch-{node}" : " managed-opensearch"; - - /// - /// Calculates the quorum given the number of instances - /// - private static int Quorum(int instanceCount) => Math.Max(1, (int)Math.Floor((double)instanceCount / 2) + 1); - - /// - /// Creates a node attribute for the version of OpenSearch - /// - public string AttributeKey(string attribute) - { - var attr = "attr."; - return $"node.{attr}{attribute}"; - } - - /// - /// Adds a node setting to the default node settings - /// - protected void Add(string key, string value) - { - if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; - - DefaultNodeSettings.Add(key, value); - } - - /// - /// Adds a node setting to the default node settings only if the OpenSearch - /// version is in the range. - /// - protected void Add(string key, string value, string range) - { - if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; - - if (string.IsNullOrWhiteSpace(range) || Version.InRange(range)) - DefaultNodeSettings.Add(key, value, range); - } - } + /// + /// Creates a new instance of a configuration for an OpenSearch cluster. + /// + /// The version of OpenSearch + /// + /// A delegate to create the instance of . + /// Passed the OpenSearch version and the Cluster name + /// + /// The number of nodes in the cluster + /// The name of the cluster + public ClusterConfiguration(OpenSearchVersion version, Func fileSystem, + int numberOfNodes = 1, string clusterName = null + ) + { + if (fileSystem == null) throw new ArgumentException(nameof(fileSystem)); + + ClusterName = clusterName; + Version = version; + Artifact = version.Artifact(Product.From("opensearch")); + FileSystem = fileSystem(Version, ClusterName); + NumberOfNodes = numberOfNodes; + + var fs = FileSystem; + Add("node.max_local_storage_nodes", numberOfNodes.ToString(CultureInfo.InvariantCulture), "1.0.0"); + + Add("cluster.name", clusterName); + Add("path.repo", fs.RepositoryPath); + Add("path.data", fs.DataPath); + var logsPathDefault = Path.Combine(fs.OpenSearchHome, "logs"); + if (logsPathDefault != fs.LogsPath) Add("path.logs", fs.LogsPath); + } + + public Artifact Artifact { get; } + + public string JavaHomeEnvironmentVariable => "JAVA_HOME"; + + /// Will print the contents of all the yaml files when starting the cluster up, great for debugging purposes + public bool PrintYamlFilesInConfigFolder { get; set; } + + public string ClusterName { get; } + public OpenSearchVersion Version { get; } + public TFileSystem FileSystem { get; } + public int NumberOfNodes { get; } + public int StartingPortNumber { get; set; } = 9200; + public bool NoCleanupAfterNodeStopped { get; set; } + + /// + /// Whether should continue to write output to console after it has started. + /// Defaults to true + /// + public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; + + public bool CacheOpenSearchHomeInstallation { get; set; } + + /// The node settings to apply to each started node + public NodeSettings DefaultNodeSettings { get; } = new NodeSettings(); + + /// + /// Creates a node name + /// + public virtual string CreateNodeName(int? node) => + node.HasValue ? $"managed-opensearch-{node}" : " managed-opensearch"; + + /// + /// Calculates the quorum given the number of instances + /// + private static int Quorum(int instanceCount) => Math.Max(1, (int)Math.Floor((double)instanceCount / 2) + 1); + + /// + /// Creates a node attribute for the version of OpenSearch + /// + public string AttributeKey(string attribute) + { + var attr = "attr."; + return $"node.{attr}{attribute}"; + } + + /// + /// Adds a node setting to the default node settings + /// + protected void Add(string key, string value) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; + + DefaultNodeSettings.Add(key, value); + } + + /// + /// Adds a node setting to the default node settings only if the OpenSearch + /// version is in the range. + /// + protected void Add(string key, string value, string range) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; + + if (string.IsNullOrWhiteSpace(range) || Version.InRange(range)) + DefaultNodeSettings.Add(key, value, range); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs index 1e379e1470..27bdd36ca5 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs @@ -32,77 +32,76 @@ using OpenSearch.Stack.ArtifactsApi; using ProcNet; -namespace OpenSearch.OpenSearch.Managed.Configuration +namespace OpenSearch.OpenSearch.Managed.Configuration; + +public class NodeConfiguration { - public class NodeConfiguration - { - private Action _defaultStartArgs = s => { }; - - public NodeConfiguration(OpenSearchVersion version, int? port = null) : this(new ClusterConfiguration(version), - port) - { - } - - public NodeConfiguration(IClusterConfiguration clusterConfiguration, int? port = null, - string nodePrefix = null) - { - ClusterConfiguration = clusterConfiguration; - DesiredPort = port; - DesiredNodeName = CreateNodeName(port, nodePrefix) ?? clusterConfiguration.CreateNodeName(port); - Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings); - - if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName); - if (DesiredPort.HasValue) - Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture)); - } - - private IClusterConfiguration ClusterConfiguration { get; } - - public int? DesiredPort { get; } - public string DesiredNodeName { get; } - - public Action ModifyStartArguments - { - get => _defaultStartArgs; - set => _defaultStartArgs = value ?? (s => { }); - } - - /// - /// Wheter should continue to write output to console after it has started. - /// Defaults to true but useful to turn of if it proofs to be too noisy - /// - public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; - - /// - /// The expected duration of the shut down sequence for OpenSearch. After this wait duration a hard kill will occur. - /// - public TimeSpan WaitForShutdown { get; set; } = TimeSpan.FromSeconds(10); - - /// - /// Copy of . Add individual node settings here. - /// - public NodeSettings Settings { get; } - - public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; - public OpenSearchVersion Version => ClusterConfiguration.Version; - public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version); - - public void InitialMasterNodes(string initialMasterNodes) => - Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0"); - - public string AttributeKey(string attribute) - { - var attr = "attr."; - return $"node.{attr}{attribute}"; - } - - public void Add(string key, string value) => Settings.Add(key, value); - - private string CreateNodeName(int? node, string prefix = null) - { - if (prefix == null) return null; - var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); - return $"{prefix.Replace("Cluster", "").ToLowerInvariant()}-node-{suffix}{node}"; - } - } + private Action _defaultStartArgs = s => { }; + + public NodeConfiguration(OpenSearchVersion version, int? port = null) : this(new ClusterConfiguration(version), + port) + { + } + + public NodeConfiguration(IClusterConfiguration clusterConfiguration, int? port = null, + string nodePrefix = null) + { + ClusterConfiguration = clusterConfiguration; + DesiredPort = port; + DesiredNodeName = CreateNodeName(port, nodePrefix) ?? clusterConfiguration.CreateNodeName(port); + Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings); + + if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName); + if (DesiredPort.HasValue) + Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture)); + } + + private IClusterConfiguration ClusterConfiguration { get; } + + public int? DesiredPort { get; } + public string DesiredNodeName { get; } + + public Action ModifyStartArguments + { + get => _defaultStartArgs; + set => _defaultStartArgs = value ?? (s => { }); + } + + /// + /// Wheter should continue to write output to console after it has started. + /// Defaults to true but useful to turn of if it proofs to be too noisy + /// + public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; + + /// + /// The expected duration of the shut down sequence for OpenSearch. After this wait duration a hard kill will occur. + /// + public TimeSpan WaitForShutdown { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Copy of . Add individual node settings here. + /// + public NodeSettings Settings { get; } + + public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; + public OpenSearchVersion Version => ClusterConfiguration.Version; + public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version); + + public void InitialMasterNodes(string initialMasterNodes) => + Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0"); + + public string AttributeKey(string attribute) + { + var attr = "attr."; + return $"node.{attr}{attribute}"; + } + + public void Add(string key, string value) => Settings.Add(key, value); + + private string CreateNodeName(int? node, string prefix = null) + { + if (prefix == null) return null; + var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); + return $"{prefix.Replace("Cluster", "").ToLowerInvariant()}-node-{suffix}{node}"; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs index 8d2c8f40f2..8a38c13274 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs @@ -26,25 +26,24 @@ * under the License. */ -namespace OpenSearch.OpenSearch.Managed.Configuration +namespace OpenSearch.OpenSearch.Managed.Configuration; + +public struct NodeSetting { - public struct NodeSetting - { - public string Key { get; } - public string Value { get; } + public string Key { get; } + public string Value { get; } - /// - /// Stores for which opensearch version range this setting is applicable - /// - public string VersionRange { get; } + /// + /// Stores for which opensearch version range this setting is applicable + /// + public string VersionRange { get; } - public NodeSetting(string key, string value, string range) - { - Key = key; - Value = value; - VersionRange = range; - } + public NodeSetting(string key, string value, string range) + { + Key = key; + Value = value; + VersionRange = range; + } - public override string ToString() => $"{Key}={Value}"; - } + public override string ToString() => $"{Key}={Value}"; } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs index ea34663ca7..6a62456aa9 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs @@ -31,40 +31,39 @@ using System.Linq; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Managed.Configuration +namespace OpenSearch.OpenSearch.Managed.Configuration; + +public class NodeSettings : List { - public class NodeSettings : List - { - public NodeSettings() - { - } + public NodeSettings() + { + } - public NodeSettings(NodeSettings settings) : base(settings) - { - } + public NodeSettings(NodeSettings settings) : base(settings) + { + } - public void Add(string setting) - { - var s = setting.Split(new[] {'='}, 2, StringSplitOptions.RemoveEmptyEntries); - if (s.Length != 2) - throw new ArgumentException($"Can only add node settings in key=value from but received: {setting}"); - Add(new NodeSetting(s[0], s[1], null)); - } + public void Add(string setting) + { + var s = setting.Split(new[] { '=' }, 2, StringSplitOptions.RemoveEmptyEntries); + if (s.Length != 2) + throw new ArgumentException($"Can only add node settings in key=value from but received: {setting}"); + Add(new NodeSetting(s[0], s[1], null)); + } - public void Add(string key, string value) => Add(new NodeSetting(key, value, null)); + public void Add(string key, string value) => Add(new NodeSetting(key, value, null)); - public void Add(string key, string value, string versionRange) => - Add(new NodeSetting(key, value, versionRange)); + public void Add(string key, string value, string versionRange) => + Add(new NodeSetting(key, value, versionRange)); - public string[] ToCommandLineArguments(OpenSearchVersion version) => - this - //if a node setting is only applicable for a certain version make sure its filtered out - .Where(s => string.IsNullOrEmpty(s.VersionRange) || version.InRange(s.VersionRange)) - //allow additional settings to take precedence over already DefaultNodeSettings - //without relying on opensearch to dedup - .GroupBy(setting => setting.Key) - .Select(g => g.Last()) - .SelectMany(s => new[] { "-E", s.ToString() }) - .ToArray(); - } + public string[] ToCommandLineArguments(OpenSearchVersion version) => + this + //if a node setting is only applicable for a certain version make sure its filtered out + .Where(s => string.IsNullOrEmpty(s.VersionRange) || version.InRange(s.VersionRange)) + //allow additional settings to take precedence over already DefaultNodeSettings + //without relying on opensearch to dedup + .GroupBy(setting => setting.Key) + .Select(g => g.Last()) + .SelectMany(s => new[] { "-E", s.ToString() }) + .ToArray(); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs index 1f8e2911e7..993a40e15d 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs @@ -29,16 +29,15 @@ using System; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; + +public class ConsoleLineWriter : IConsoleLineHandler { - public class ConsoleLineWriter : IConsoleLineHandler - { - public void Handle(LineOut lineOut) - { - var w = lineOut.Error ? Console.Error : Console.Out; - w.WriteLine(lineOut); - } + public void Handle(LineOut lineOut) + { + var w = lineOut.Error ? Console.Error : Console.Out; + w.WriteLine(lineOut); + } - public void Handle(Exception e) => Console.Error.WriteLine(e); - } + public void Handle(Exception e) => Console.Error.WriteLine(e); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs index f54b8eb648..99c892e8a7 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs @@ -28,38 +28,37 @@ using System.Text.RegularExpressions; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; + +public static class ExceptionLineParser { - public static class ExceptionLineParser - { - private static readonly Regex CauseRegex = new Regex(@"^(?.*?Exception:)(?.*?)$"); + private static readonly Regex CauseRegex = new Regex(@"^(?.*?Exception:)(?.*?)$"); - private static readonly Regex - LocRegex = new Regex(@"^(?\s*?at )(?.*?)\((?.*?)\)(?.*?)$"); + private static readonly Regex + LocRegex = new Regex(@"^(?\s*?at )(?.*?)\((?.*?)\)(?.*?)$"); - public static bool TryParseCause(string line, out string cause, out string message) - { - cause = message = null; - if (string.IsNullOrEmpty(line)) return false; - var match = CauseRegex.Match(line); - if (!match.Success) return false; - cause = match.Groups["cause"].Value.Trim(); - message = match.Groups["message"].Value.Trim(); - return true; - } + public static bool TryParseCause(string line, out string cause, out string message) + { + cause = message = null; + if (string.IsNullOrEmpty(line)) return false; + var match = CauseRegex.Match(line); + if (!match.Success) return false; + cause = match.Groups["cause"].Value.Trim(); + message = match.Groups["message"].Value.Trim(); + return true; + } - public static bool TryParseStackTrace(string line, out string at, out string method, out string file, - out string jar) - { - at = method = file = jar = null; - if (string.IsNullOrEmpty(line)) return false; - var match = LocRegex.Match(line); - if (!match.Success) return false; - at = match.Groups["at"].Value; - method = match.Groups["method"].Value.Trim(); - file = match.Groups["file"].Value.Trim(); - jar = match.Groups["jar"].Value.Trim(); - return true; - } - } + public static bool TryParseStackTrace(string line, out string at, out string method, out string file, + out string jar) + { + at = method = file = jar = null; + if (string.IsNullOrEmpty(line)) return false; + var match = LocRegex.Match(line); + if (!match.Success) return false; + at = match.Groups["at"].Value; + method = match.Groups["method"].Value.Trim(); + file = match.Groups["file"].Value.Trim(); + jar = match.Groups["jar"].Value.Trim(); + return true; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs index 000c2c78b8..425b6eb416 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs @@ -29,25 +29,24 @@ using System; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; + +public static class LineWriterExtensions { - public static class LineWriterExtensions - { - public static void WriteDiagnostic(this IConsoleLineHandler writer, string message) => - writer.Handle(Info(message)); + public static void WriteDiagnostic(this IConsoleLineHandler writer, string message) => + writer.Handle(Info(message)); - public static void WriteDiagnostic(this IConsoleLineHandler writer, string message, string node) => - writer?.Handle(Info(node != null ? $"[{node}] {message}" : message)); + public static void WriteDiagnostic(this IConsoleLineHandler writer, string message, string node) => + writer?.Handle(Info(node != null ? $"[{node}] {message}" : message)); - public static void WriteError(this IConsoleLineHandler writer, string message) => writer.Handle(Error(message)); + public static void WriteError(this IConsoleLineHandler writer, string message) => writer.Handle(Error(message)); - public static void WriteError(this IConsoleLineHandler writer, string message, string node) => - writer?.Handle(Error(node != null ? $"[{node}] {message}" : message)); + public static void WriteError(this IConsoleLineHandler writer, string message, string node) => + writer?.Handle(Error(node != null ? $"[{node}] {message}" : message)); - private static string Format(bool error, string message) => - $"[{DateTime.UtcNow:yyyy-MM-ddThh:mm:ss,fff}][{(error ? "ERROR" : "INFO ")}][Managed OpenSearch\t] {message}"; + private static string Format(bool error, string message) => + $"[{DateTime.UtcNow:yyyy-MM-ddThh:mm:ss,fff}][{(error ? "ERROR" : "INFO ")}][Managed OpenSearch\t] {message}"; - private static LineOut Info(string message) => ConsoleOut.Out(Format(false, message)); - private static LineOut Error(string message) => ConsoleOut.ErrorOut(Format(true, message)); - } + private static LineOut Info(string message) => ConsoleOut.Out(Format(false, message)); + private static LineOut Error(string message) => ConsoleOut.ErrorOut(Format(true, message)); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs index 53e3ab08b7..c4a22266b9 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs @@ -32,220 +32,219 @@ using System.Linq; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; + +public class LineHighlightWriter : IConsoleLineHandler { - public class LineHighlightWriter : IConsoleLineHandler - { - private readonly LineOutParser _lineOutParser; - - private static readonly ConsoleColor[] AvailableNodeColors = - { - ConsoleColor.DarkGreen, ConsoleColor.DarkBlue, ConsoleColor.DarkRed, ConsoleColor.DarkCyan, - ConsoleColor.DarkYellow, ConsoleColor.Blue, - }; - - private static readonly object Lock = new object(); - - private static readonly char[] Anchors = {'[', ']', '{', '}'}; - - private readonly Dictionary _nodeColors; - - public LineHighlightWriter() - { - } - - public LineHighlightWriter(IList nodes, LineOutParser lineOutParser) - { - _lineOutParser = lineOutParser ?? throw new NullReferenceException(nameof(lineOutParser)); - if (nodes == null) throw new NullReferenceException(nameof(nodes)); - - var colors = new Dictionary(); - for (var i = 0; i < nodes.Count; i++) - { - var color = i % AvailableNodeColors.Length; - colors.Add(nodes[i], AvailableNodeColors[color]); - } - - _nodeColors = colors; - } - - public void Handle(Exception e) - { - lock (Lock) - { - Console.BackgroundColor = ConsoleColor.Red; - Console.ForegroundColor = ConsoleColor.White; - Console.Error.WriteLine(e); - Console.ResetColor(); - } - } - - public void Handle(LineOut lineOut) - { - var parsed = _lineOutParser.TryParse(lineOut.Line, out var date, out var level, out var section, - out var node, out var message, out _); - if (parsed) Write(lineOut.Error, date, level, section, node, message, NodeColor); - else if (ExceptionLineParser.TryParseCause(lineOut.Line, out var cause, out var causeMessage)) - WriteCausedBy(lineOut.Error, cause, causeMessage); - - else if (ExceptionLineParser.TryParseStackTrace(lineOut.Line, out var at, out var method, out var file, - out var jar)) - WriteStackTraceLine(lineOut.Error, at, method, file, jar); - - else Write(lineOut.Error, lineOut.Line); - } - - private ConsoleColor NodeColor(string node) => - _nodeColors != null && _nodeColors.TryGetValue(node, out var color) ? color : AvailableNodeColors[0]; - - private static void Write(bool error, string date, string level, string section, string node, string message, - Func getNodeColor = null) - { - lock (Lock) - { - var w = error ? Console.Error : Console.Out; - WriteBlock(w, ConsoleColor.DarkGray, date); - WriteBlock(w, LevelColor(level), level, 5); - - if (!string.IsNullOrWhiteSpace(section)) - { - WriteBlock(w, ConsoleColor.DarkCyan, section, 25); - WriteSpace(w); - } - - WriteBlock(w, getNodeColor?.Invoke(node) ?? ConsoleColor.DarkGreen, node); - WriteSpace(w); - - var messageColor = error || level == "ERROR" ? ConsoleColor.Red : ConsoleColor.White; - WriteMessage(w, messageColor, message); - - Console.ResetColor(); - w.Flush(); - } - } - - private static void WriteCausedBy(bool error, string cause, string causeMessage) - { - lock (Lock) - { - var w = error ? Console.Error : Console.Out; - Write(w, ConsoleColor.DarkRed, cause); - WriteSpace(w); - Write(w, ConsoleColor.Red, causeMessage); - w.WriteLine(); - Console.ResetColor(); - w.Flush(); - } - } - - private static void WriteStackTraceLine(bool error, string at, string method, string file, string jar) - { - lock (Lock) - { - var w = error ? Console.Error : Console.Out; - Write(w, ConsoleColor.DarkGray, at); - Write(w, ConsoleColor.DarkBlue, method); - Write(w, ConsoleColor.DarkGray, "("); - Write(w, ConsoleColor.Blue, file); - Write(w, ConsoleColor.DarkGray, ")"); - WriteSpace(w); - Write(w, ConsoleColor.Gray, jar); - w.WriteLine(); - - Console.ResetColor(); - w.Flush(); - } - } - - private static void Write(bool error, string message) - { - lock (Lock) - { - var w = error ? Console.Error : Console.Out; - var messageColor = error ? ConsoleColor.Red : ConsoleColor.White; - WriteMessage(w, messageColor, message); - Console.ResetColor(); - w.Flush(); - } - } - - private static ConsoleColor LevelColor(string level) - { - switch (level ?? "") - { - case "WARN": return ConsoleColor.Yellow; - case "FATAL": - case "ERROR": - return ConsoleColor.Red; - case "DEBUG": - case "TRACE": - return ConsoleColor.DarkGray; - default: - return ConsoleColor.Cyan; - } - } - - private static IEnumerable Parts(string s) - { - int start = 0, index; - while ((index = s.IndexOfAny(Anchors, start)) != -1) - { - if (index - start > 0) - yield return s.Substring(start, index - start); - - yield return s.Substring(index, 1); - start = index + 1; - } - - if (start < s.Length) - yield return s.Substring(start); - } - - private static void WriteMessage(TextWriter w, ConsoleColor color, string message) - { - var insideSquareBracket = 0; - var insideCurlyBracket = 0; - foreach (var p in Parts(message)) - { - if (p.Length == 0) continue; - if (p[0] == '[') insideSquareBracket++; - else if (p[0] == ']') insideSquareBracket--; - else if (p[0] == '{') insideCurlyBracket++; - else if (p[0] == '}') insideCurlyBracket--; - - if (Anchors.Contains(p[0])) Console.ForegroundColor = ConsoleColor.DarkGray; - else if (insideSquareBracket > 0) Console.ForegroundColor = ConsoleColor.Yellow; - else if (insideCurlyBracket > 0) Console.ForegroundColor = ConsoleColor.Blue; - else Console.ForegroundColor = color; - - w.Write(p); - } - - Console.ResetColor(); - w.WriteLine(); - } - - private static void WriteSpace(TextWriter w) => w.Write(" "); - - private static void WriteBlock(TextWriter w, ConsoleColor color, string block, int? pad = null) - { - if (string.IsNullOrEmpty(block)) return; - var b = pad != null ? block.PadRight(pad.Value) : block; - Console.ForegroundColor = ConsoleColor.DarkGray; - w.Write("["); - Console.ForegroundColor = color; - w.Write(b); - Console.ForegroundColor = ConsoleColor.DarkGray; - w.Write("]"); - } - - private static void Write(TextWriter w, ConsoleColor color, string block, int? pad = null) - { - var b = pad != null ? block.PadRight(pad.Value) : block; - var original = Console.ForegroundColor; - Console.ForegroundColor = color; - w.Write(b); - Console.ForegroundColor = original; - } - } + private readonly LineOutParser _lineOutParser; + + private static readonly ConsoleColor[] AvailableNodeColors = + { + ConsoleColor.DarkGreen, ConsoleColor.DarkBlue, ConsoleColor.DarkRed, ConsoleColor.DarkCyan, + ConsoleColor.DarkYellow, ConsoleColor.Blue, + }; + + private static readonly object Lock = new object(); + + private static readonly char[] Anchors = { '[', ']', '{', '}' }; + + private readonly Dictionary _nodeColors; + + public LineHighlightWriter() + { + } + + public LineHighlightWriter(IList nodes, LineOutParser lineOutParser) + { + _lineOutParser = lineOutParser ?? throw new NullReferenceException(nameof(lineOutParser)); + if (nodes == null) throw new NullReferenceException(nameof(nodes)); + + var colors = new Dictionary(); + for (var i = 0; i < nodes.Count; i++) + { + var color = i % AvailableNodeColors.Length; + colors.Add(nodes[i], AvailableNodeColors[color]); + } + + _nodeColors = colors; + } + + public void Handle(Exception e) + { + lock (Lock) + { + Console.BackgroundColor = ConsoleColor.Red; + Console.ForegroundColor = ConsoleColor.White; + Console.Error.WriteLine(e); + Console.ResetColor(); + } + } + + public void Handle(LineOut lineOut) + { + var parsed = _lineOutParser.TryParse(lineOut.Line, out var date, out var level, out var section, + out var node, out var message, out _); + if (parsed) Write(lineOut.Error, date, level, section, node, message, NodeColor); + else if (ExceptionLineParser.TryParseCause(lineOut.Line, out var cause, out var causeMessage)) + WriteCausedBy(lineOut.Error, cause, causeMessage); + + else if (ExceptionLineParser.TryParseStackTrace(lineOut.Line, out var at, out var method, out var file, + out var jar)) + WriteStackTraceLine(lineOut.Error, at, method, file, jar); + + else Write(lineOut.Error, lineOut.Line); + } + + private ConsoleColor NodeColor(string node) => + _nodeColors != null && _nodeColors.TryGetValue(node, out var color) ? color : AvailableNodeColors[0]; + + private static void Write(bool error, string date, string level, string section, string node, string message, + Func getNodeColor = null) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + WriteBlock(w, ConsoleColor.DarkGray, date); + WriteBlock(w, LevelColor(level), level, 5); + + if (!string.IsNullOrWhiteSpace(section)) + { + WriteBlock(w, ConsoleColor.DarkCyan, section, 25); + WriteSpace(w); + } + + WriteBlock(w, getNodeColor?.Invoke(node) ?? ConsoleColor.DarkGreen, node); + WriteSpace(w); + + var messageColor = error || level == "ERROR" ? ConsoleColor.Red : ConsoleColor.White; + WriteMessage(w, messageColor, message); + + Console.ResetColor(); + w.Flush(); + } + } + + private static void WriteCausedBy(bool error, string cause, string causeMessage) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + Write(w, ConsoleColor.DarkRed, cause); + WriteSpace(w); + Write(w, ConsoleColor.Red, causeMessage); + w.WriteLine(); + Console.ResetColor(); + w.Flush(); + } + } + + private static void WriteStackTraceLine(bool error, string at, string method, string file, string jar) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + Write(w, ConsoleColor.DarkGray, at); + Write(w, ConsoleColor.DarkBlue, method); + Write(w, ConsoleColor.DarkGray, "("); + Write(w, ConsoleColor.Blue, file); + Write(w, ConsoleColor.DarkGray, ")"); + WriteSpace(w); + Write(w, ConsoleColor.Gray, jar); + w.WriteLine(); + + Console.ResetColor(); + w.Flush(); + } + } + + private static void Write(bool error, string message) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + var messageColor = error ? ConsoleColor.Red : ConsoleColor.White; + WriteMessage(w, messageColor, message); + Console.ResetColor(); + w.Flush(); + } + } + + private static ConsoleColor LevelColor(string level) + { + switch (level ?? "") + { + case "WARN": return ConsoleColor.Yellow; + case "FATAL": + case "ERROR": + return ConsoleColor.Red; + case "DEBUG": + case "TRACE": + return ConsoleColor.DarkGray; + default: + return ConsoleColor.Cyan; + } + } + + private static IEnumerable Parts(string s) + { + int start = 0, index; + while ((index = s.IndexOfAny(Anchors, start)) != -1) + { + if (index - start > 0) + yield return s.Substring(start, index - start); + + yield return s.Substring(index, 1); + start = index + 1; + } + + if (start < s.Length) + yield return s.Substring(start); + } + + private static void WriteMessage(TextWriter w, ConsoleColor color, string message) + { + var insideSquareBracket = 0; + var insideCurlyBracket = 0; + foreach (var p in Parts(message)) + { + if (p.Length == 0) continue; + if (p[0] == '[') insideSquareBracket++; + else if (p[0] == ']') insideSquareBracket--; + else if (p[0] == '{') insideCurlyBracket++; + else if (p[0] == '}') insideCurlyBracket--; + + if (Anchors.Contains(p[0])) Console.ForegroundColor = ConsoleColor.DarkGray; + else if (insideSquareBracket > 0) Console.ForegroundColor = ConsoleColor.Yellow; + else if (insideCurlyBracket > 0) Console.ForegroundColor = ConsoleColor.Blue; + else Console.ForegroundColor = color; + + w.Write(p); + } + + Console.ResetColor(); + w.WriteLine(); + } + + private static void WriteSpace(TextWriter w) => w.Write(" "); + + private static void WriteBlock(TextWriter w, ConsoleColor color, string block, int? pad = null) + { + if (string.IsNullOrEmpty(block)) return; + var b = pad != null ? block.PadRight(pad.Value) : block; + Console.ForegroundColor = ConsoleColor.DarkGray; + w.Write("["); + Console.ForegroundColor = color; + w.Write(b); + Console.ForegroundColor = ConsoleColor.DarkGray; + w.Write("]"); + } + + private static void Write(TextWriter w, ConsoleColor color, string block, int? pad = null) + { + var b = pad != null ? block.PadRight(pad.Value) : block; + var original = Console.ForegroundColor; + Console.ForegroundColor = color; + w.Write(b); + Console.ForegroundColor = original; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs index 1acfc8c6a0..f49ff2f8a0 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs @@ -30,116 +30,115 @@ using System.Text.RegularExpressions; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters -{ - public class LineOutParser - { - private LineOutParser() { } - - public static readonly LineOutParser OpenSearch = new(shortNamePrefix: "o.o", fullNamePrefix: "org.opensearch", - securityPluginName: "OpenSearchSecurityPlugin"); +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; -/* -[2016-09-26T11:43:17,314][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initializing ... -[2016-09-26T11:43:17,470][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] using [1] data paths, mounts [[BOOTCAMP (C:)]], net usable_space [27.7gb], net total_space [129.7gb], spins? [unknown], types [NTFS] -[2016-09-26T11:43:17,471][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] heap size [1.9gb], compressed ordinary object pointers [true] -[2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] -[2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [aggs-matrix-stats] -[2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [ingest-common] -[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-expression] -[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-groovy] -[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-mustache] -[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-painless] -[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [percolator] -[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [reindex] -[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty3] -[2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty4] -[2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-attachment] -[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-geoip] -[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-attachments] -[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-murmur3] -[2016-09-26T11:43:19,374][WARN ][d.m.attachment ] [mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin. -[2016-09-26T11:43:22,179][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initialized -[2016-09-26T11:43:22,180][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] starting ... -*/ - private static readonly Regex ConsoleLineParser = - new Regex(@"\[(?.*?)\]\[(?.*?)\](?:\[(?
.*?)\])(?: \[(?.*?)\])? (?.+)"); - - private static readonly Regex PortParser = new Regex(@"bound_address(es)?(opensearch)? {.+\:(?\d+)}"); - - //[2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] - private static readonly Regex InfoParser = - new Regex(@"version\[(?.*)\], pid\[(?.*)\], build\[(?.+)\]"); - - private LineOutParser(string shortNamePrefix, string fullNamePrefix, string securityPluginName) : this() - { - _shortNamePrefix = shortNamePrefix; - _fullNamePrefix = fullNamePrefix; - _securityPluginRegex = new Regex(Regex.Escape(securityPluginName)); - } - - private readonly Regex _securityPluginRegex; - private readonly string _shortNamePrefix; - private readonly string _fullNamePrefix ; - - public bool TryParse(string line, - out string date, out string level, out string section, out string node, out string message, - out bool started) - { - date = level = section = node = message = null; - started = false; - if (string.IsNullOrEmpty(line)) return false; - - var match = ConsoleLineParser.Match(line); - if (!match.Success) return false; - date = match.Groups["date"].Value.Trim(); - level = match.Groups["level"].Value.Trim(); - section = match.Groups["section"].Value.Trim().Replace(_fullNamePrefix + ".", ""); - node = match.Groups["node"].Value.Trim(); - message = match.Groups["message"].Value.Trim(); - started = TryGetStartedConfirmation(section, message); - return true; - } - - private bool TryGetStartedConfirmation(string section, string message) => section == ShortName("n.Node") && message == "started"; - - public bool TryGetPortNumber(string section, string message, out int port) - { - port = 0; - var inHttpSection = - section == ShortName("h.HttpServer") - || section == "http" - || section == ShortName("h.AbstractHttpServerTransport") - || section == ShortName("h.n.Netty4HttpServerTransport") - || section == ShortName("x.s.t.n.SecurityNetty4HttpServerTransport"); - if (!inHttpSection) return false; - - if (string.IsNullOrWhiteSpace(message)) return false; - - var match = PortParser.Match(message); - if (!match.Success) return false; - - var portString = match.Groups["port"].Value.Trim(); - port = int.Parse(portString); - return true; - } - - public bool TryParseNodeInfo(string section, string message, out string version, out int? pid) - { - var inNodeSection = section == ShortName("n.Node") || section == "node"; - - version = null; - pid = null; - if (!inNodeSection) return false; - - var match = InfoParser.Match(message.Replace(Environment.NewLine, "")); - if (!match.Success) return false; - - version = match.Groups["version"].Value.Trim(); - pid = int.Parse(match.Groups["pid"].Value.Trim()); - return true; - } - - private string ShortName(string suffix) => $"{_shortNamePrefix}.{suffix}"; - } +public class LineOutParser +{ + private LineOutParser() { } + + public static readonly LineOutParser OpenSearch = new(shortNamePrefix: "o.o", fullNamePrefix: "org.opensearch", + securityPluginName: "OpenSearchSecurityPlugin"); + + /* + [2016-09-26T11:43:17,314][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initializing ... + [2016-09-26T11:43:17,470][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] using [1] data paths, mounts [[BOOTCAMP (C:)]], net usable_space [27.7gb], net total_space [129.7gb], spins? [unknown], types [NTFS] + [2016-09-26T11:43:17,471][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] heap size [1.9gb], compressed ordinary object pointers [true] + [2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] + [2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [aggs-matrix-stats] + [2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [ingest-common] + [2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-expression] + [2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-groovy] + [2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-mustache] + [2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-painless] + [2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [percolator] + [2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [reindex] + [2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty3] + [2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty4] + [2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-attachment] + [2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-geoip] + [2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-attachments] + [2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-murmur3] + [2016-09-26T11:43:19,374][WARN ][d.m.attachment ] [mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin. + [2016-09-26T11:43:22,179][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initialized + [2016-09-26T11:43:22,180][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] starting ... + */ + private static readonly Regex ConsoleLineParser = + new Regex(@"\[(?.*?)\]\[(?.*?)\](?:\[(?
.*?)\])(?: \[(?.*?)\])? (?.+)"); + + private static readonly Regex PortParser = new Regex(@"bound_address(es)?(opensearch)? {.+\:(?\d+)}"); + + //[2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] + private static readonly Regex InfoParser = + new Regex(@"version\[(?.*)\], pid\[(?.*)\], build\[(?.+)\]"); + + private LineOutParser(string shortNamePrefix, string fullNamePrefix, string securityPluginName) : this() + { + _shortNamePrefix = shortNamePrefix; + _fullNamePrefix = fullNamePrefix; + _securityPluginRegex = new Regex(Regex.Escape(securityPluginName)); + } + + private readonly Regex _securityPluginRegex; + private readonly string _shortNamePrefix; + private readonly string _fullNamePrefix; + + public bool TryParse(string line, + out string date, out string level, out string section, out string node, out string message, + out bool started) + { + date = level = section = node = message = null; + started = false; + if (string.IsNullOrEmpty(line)) return false; + + var match = ConsoleLineParser.Match(line); + if (!match.Success) return false; + date = match.Groups["date"].Value.Trim(); + level = match.Groups["level"].Value.Trim(); + section = match.Groups["section"].Value.Trim().Replace(_fullNamePrefix + ".", ""); + node = match.Groups["node"].Value.Trim(); + message = match.Groups["message"].Value.Trim(); + started = TryGetStartedConfirmation(section, message); + return true; + } + + private bool TryGetStartedConfirmation(string section, string message) => section == ShortName("n.Node") && message == "started"; + + public bool TryGetPortNumber(string section, string message, out int port) + { + port = 0; + var inHttpSection = + section == ShortName("h.HttpServer") + || section == "http" + || section == ShortName("h.AbstractHttpServerTransport") + || section == ShortName("h.n.Netty4HttpServerTransport") + || section == ShortName("x.s.t.n.SecurityNetty4HttpServerTransport"); + if (!inHttpSection) return false; + + if (string.IsNullOrWhiteSpace(message)) return false; + + var match = PortParser.Match(message); + if (!match.Success) return false; + + var portString = match.Groups["port"].Value.Trim(); + port = int.Parse(portString); + return true; + } + + public bool TryParseNodeInfo(string section, string message, out string version, out int? pid) + { + var inNodeSection = section == ShortName("n.Node") || section == "node"; + + version = null; + pid = null; + if (!inNodeSection) return false; + + var match = InfoParser.Match(message.Replace(Environment.NewLine, "")); + if (!match.Success) return false; + + version = match.Groups["version"].Value.Trim(); + pid = int.Parse(match.Groups["pid"].Value.Trim()); + return true; + } + + private string ShortName(string suffix) => $"{_shortNamePrefix}.{suffix}"; } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs index e8e24c3900..55347aedc0 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs @@ -29,18 +29,17 @@ using System; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters; + +internal class NoopConsoleLineWriter : IConsoleLineHandler { - internal class NoopConsoleLineWriter : IConsoleLineHandler - { - public static NoopConsoleLineWriter Instance { get; } = new NoopConsoleLineWriter(); + public static NoopConsoleLineWriter Instance { get; } = new NoopConsoleLineWriter(); - public void Handle(LineOut lineOut) - { - } + public void Handle(LineOut lineOut) + { + } - public void Handle(Exception e) - { - } - } + public void Handle(Exception e) + { + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs index ff71128c20..e3f7a955f1 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs @@ -26,54 +26,53 @@ * under the License. */ -namespace OpenSearch.OpenSearch.Managed.FileSystem +namespace OpenSearch.OpenSearch.Managed.FileSystem; + +/// +/// The file system for an OpenSearch node +/// +public interface INodeFileSystem { - /// - /// The file system for an OpenSearch node - /// - public interface INodeFileSystem - { - /// - /// The path to the script to start OpenSearch - /// - string Binary { get; } + /// + /// The path to the script to start OpenSearch + /// + string Binary { get; } - /// - /// The path to the script to manage plugins - /// - string PluginBinary { get; } + /// + /// The path to the script to manage plugins + /// + string PluginBinary { get; } - /// - /// The path to the home directory - /// - string OpenSearchHome { get; } + /// + /// The path to the home directory + /// + string OpenSearchHome { get; } - /// - /// The path to the config directory - /// - string ConfigPath { get; } + /// + /// The path to the config directory + /// + string ConfigPath { get; } - /// - /// The path to the data directory - /// - string DataPath { get; } + /// + /// The path to the data directory + /// + string DataPath { get; } - /// - /// The path to the logs directory - /// - string LogsPath { get; } + /// + /// The path to the logs directory + /// + string LogsPath { get; } - /// - /// The path to the repository directory - /// - string RepositoryPath { get; } + /// + /// The path to the repository directory + /// + string RepositoryPath { get; } - /// - /// The path to the directory in which this node resides - /// - string LocalFolder { get; } + /// + /// The path to the directory in which this node resides + /// + string LocalFolder { get; } - /// The config environment variable to use for this version - string ConfigEnvironmentVariableName { get; } - } + /// The config environment variable to use for this version + string ConfigEnvironmentVariableName { get; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs index 8a1c091bfb..e49ff3a851 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs @@ -32,69 +32,68 @@ using OpenSearch.Stack.ArtifactsApi; using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Managed.FileSystem +namespace OpenSearch.OpenSearch.Managed.FileSystem; + +/// +public class NodeFileSystem : INodeFileSystem { - /// - public class NodeFileSystem : INodeFileSystem - { - protected const string SubFolder = "OpenSearchManaged"; + protected const string SubFolder = "OpenSearchManaged"; - public NodeFileSystem(OpenSearchVersion version, string openSearchHome = null) - { - Version = version; - Artifact = version.Artifact(Product.OpenSearch); - LocalFolder = AppDataFolder(version); - OpenSearchHome = openSearchHome ?? - GetOpenSearchHomeVariable() ?? throw new ArgumentNullException(nameof(openSearchHome)); + public NodeFileSystem(OpenSearchVersion version, string openSearchHome = null) + { + Version = version; + Artifact = version.Artifact(Product.OpenSearch); + LocalFolder = AppDataFolder(version); + OpenSearchHome = openSearchHome ?? + GetOpenSearchHomeVariable() ?? throw new ArgumentNullException(nameof(openSearchHome)); - ConfigEnvironmentVariableName = "OPENSEARCH_PATH_CONF"; - } + ConfigEnvironmentVariableName = "OPENSEARCH_PATH_CONF"; + } - protected OpenSearchVersion Version { get; } - protected Artifact Artifact { get; } + protected OpenSearchVersion Version { get; } + protected Artifact Artifact { get; } - private static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; + private static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; - protected static string BinarySuffix => IsMono || Path.DirectorySeparatorChar == '/' ? "" : ".bat"; + protected static string BinarySuffix => IsMono || Path.DirectorySeparatorChar == '/' ? "" : ".bat"; - /// - public string Binary => Path.Combine(OpenSearchHome, "bin", "opensearch") + BinarySuffix; + /// + public string Binary => Path.Combine(OpenSearchHome, "bin", "opensearch") + BinarySuffix; - /// - public string PluginBinary => Path.Combine(OpenSearchHome, "bin", "opensearch-plugin") + BinarySuffix; + /// + public string PluginBinary => Path.Combine(OpenSearchHome, "bin", "opensearch-plugin") + BinarySuffix; - /// - public string OpenSearchHome { get; } + /// + public string OpenSearchHome { get; } - /// - public string LocalFolder { get; } + /// + public string LocalFolder { get; } - /// - public virtual string ConfigPath => null; + /// + public virtual string ConfigPath => null; - /// - public virtual string DataPath => null; + /// + public virtual string DataPath => null; - /// - public virtual string LogsPath => null; + /// + public virtual string LogsPath => null; - /// - public virtual string RepositoryPath => null; + /// + public virtual string RepositoryPath => null; - public string ConfigEnvironmentVariableName { get; } + public string ConfigEnvironmentVariableName { get; } - protected static string AppDataFolder(OpenSearchVersion version) - { - var appData = GetApplicationDataDirectory(); - return Path.Combine(appData, SubFolder, version.Artifact(Product.OpenSearch).LocalFolderName); - } + protected static string AppDataFolder(OpenSearchVersion version) + { + var appData = GetApplicationDataDirectory(); + return Path.Combine(appData, SubFolder, version.Artifact(Product.OpenSearch).LocalFolderName); + } - protected static string GetOpenSearchHomeVariable() => Environment.GetEnvironmentVariable("OPENSEARCH_HOME"); + protected static string GetOpenSearchHomeVariable() => Environment.GetEnvironmentVariable("OPENSEARCH_HOME"); - protected static string GetApplicationDataDirectory() => - RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Environment.GetEnvironmentVariable("LocalAppData") - : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, - Environment.SpecialFolderOption.Create); - } + protected static string GetApplicationDataDirectory() => + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Environment.GetEnvironmentVariable("LocalAppData") + : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.Create); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs index 4200fa39d5..05f415d703 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs @@ -28,16 +28,15 @@ using System; -namespace OpenSearch.OpenSearch.Managed +namespace OpenSearch.OpenSearch.Managed; + +public class OpenSearchCleanExitException : Exception { - public class OpenSearchCleanExitException : Exception - { - public OpenSearchCleanExitException(string message) : base(message) - { - } + public OpenSearchCleanExitException(string message) : base(message) + { + } - public OpenSearchCleanExitException(string message, Exception innerException) : base(message, innerException) - { - } - } + public OpenSearchCleanExitException(string message, Exception innerException) : base(message, innerException) + { + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs index 97456a376b..561b845c58 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs @@ -28,12 +28,11 @@ using OpenSearch.OpenSearch.Managed.Configuration; -namespace OpenSearch.OpenSearch.Managed +namespace OpenSearch.OpenSearch.Managed; + +public class OpenSearchCluster : ClusterBase { - public class OpenSearchCluster : ClusterBase - { - public OpenSearchCluster(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) - { - } - } + public OpenSearchCluster(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs index da8250901a..a5af441d31 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs @@ -39,246 +39,245 @@ using ProcNet; using ProcNet.Std; -namespace OpenSearch.OpenSearch.Managed +namespace OpenSearch.OpenSearch.Managed; + +public class OpenSearchNode : ObservableProcess { - public class OpenSearchNode : ObservableProcess - { - private readonly ManualResetEvent _startedHandle = new ManualResetEvent(false); - - public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null) - : this(new NodeConfiguration(new ClusterConfiguration(version, (v, s) => new NodeFileSystem(v, openSearchHome)))) { } - - public OpenSearchNode(NodeConfiguration config) : base(StartArgs(config)) => NodeConfiguration = config; - - public string Version { get; private set; } - public int? Port { get; private set; } - public bool NodeStarted { get; private set; } - public NodeConfiguration NodeConfiguration { get; } - - private int? JavaProcessId { get; set; } - public override int? ProcessId => JavaProcessId ?? base.ProcessId; - public int? HostProcessId => base.ProcessId; - - /// - /// Set this true if you want the node to go into assumed started state as soon as its waiting for more nodes to start - /// doing the election. - /// Useful to speed up starting multi node clusters - /// - public bool AssumeStartedOnNotEnoughMasterPing { get; set; } - - internal IConsoleLineHandler Writer { get; private set; } - - public Exception LastSeenException { get; set; } - public WaitHandle StartedHandle => _startedHandle; - - private static StartArguments StartArgs(NodeConfiguration config) - { - //var args = new[] {config.FileSystem.Binary}.Concat(config.CommandLineArguments); - - var startArguments = new StartArguments(config.FileSystem.Binary, config.CommandLineArguments) - { - SendControlCFirst = true, - Environment = EnvVars(config), - WaitForExit = config.WaitForShutdown, - WaitForStreamReadersTimeout = config.WaitForShutdown - }; - config.ModifyStartArguments(startArguments); - return startArguments; - } - - private static Dictionary EnvVars(NodeConfiguration config) - { - var javaOpts = new List { "-Xms1g", "-Xmx1g" }; - - var environmentVariables = new Dictionary { { "OPENSEARCH_JAVA_OPTS", string.Join(" ", javaOpts) } }; - - if (!string.IsNullOrWhiteSpace(config.FileSystem.ConfigPath)) - environmentVariables.Add(config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath); - - if (!string.IsNullOrWhiteSpace(config.FileSystem.OpenSearchHome)) - environmentVariables.Add("OPENSEARCH_HOME", config.FileSystem.OpenSearchHome); - - var knnLibDir = Path.Combine(config.FileSystem.OpenSearchHome, "plugins", "opensearch-knn", config.Version >= "1.3.10" ? "lib" : "knnlib"); - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - AppendPathEnvVar("JAVA_LIBRARY_PATH", knnLibDir); - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - AppendPathEnvVar("LD_LIBRARY_PATH", knnLibDir); - - return environmentVariables; - } - - private static void AppendPathEnvVar(string name, string value) - { - var previous = Environment.GetEnvironmentVariable(name); - Environment.SetEnvironmentVariable(name, - string.IsNullOrWhiteSpace(previous) - ? value - : $"{previous}{Path.PathSeparator}{value}" - ); - } - - private bool AssumedStartedStateChecker(string section, string message) - { - if (AssumeStartedOnNotEnoughMasterPing - && section.Contains("ZenDiscovery") - && message.Contains("not enough master nodes discovered during pinging")) - return true; - - return false; - } - - public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); - - public IDisposable Start(TimeSpan waitForStarted) => Start(new LineHighlightWriter(), waitForStarted); - - public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) - { - var node = NodeConfiguration.DesiredNodeName; - var subscription = SubscribeLines(writer); - if (WaitForStarted(waitForStarted)) return subscription; - - subscription.Dispose(); - throw new OpenSearchCleanExitException( - $"Failed to start node: {node} before the configured timeout of: {waitForStarted}"); - } - - public IDisposable SubscribeLines() => SubscribeLines(new LineHighlightWriter()); - - public IDisposable SubscribeLines(IConsoleLineHandler writer) => - SubscribeLines(writer, delegate { }, delegate { }, delegate { }); - - public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext) => - SubscribeLines(writer, onNext, delegate { }, delegate { }); - - public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, - Action onError - ) => - SubscribeLines(writer, onNext, onError, delegate { }); - - public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, Action onError, - Action onCompleted - ) - { - Writer = writer; - var node = NodeConfiguration.DesiredNodeName; - writer?.WriteDiagnostic($"OpenSearch location: [{Binary}]", node); - writer?.WriteDiagnostic($"Settings: {{{string.Join(" ", NodeConfiguration.CommandLineArguments)}}}", node); - - var envVarName = "JAVA_HOME"; - var javaHome = Environment.GetEnvironmentVariable(envVarName); - writer?.WriteDiagnostic($"{envVarName}: {{{javaHome}}}", node); - Process.StartInfo.Environment[envVarName] = javaHome; - - return SubscribeLines( - l => - { - writer?.Handle(l); - onNext?.Invoke(l); - }, - e => - { - LastSeenException = e; - writer?.Handle(e); - onError?.Invoke(e); - _startedHandle.Set(); - }, - () => - { - onCompleted?.Invoke(); - _startedHandle.Set(); - }); - } - - public bool WaitForStarted(TimeSpan timeout) => _startedHandle.WaitOne(timeout); - - protected override void OnBeforeSetCompletedHandle() - { - _startedHandle.Set(); - base.OnBeforeSetCompletedHandle(); - } - - protected override void OnBeforeWaitForEndOfStreamsError(TimeSpan waited) - { - // The wait for streams finished before streams were fully read. - // this usually indicates the process is still running. - // Proc will successfully kill the host but will leave the JavaProcess the bat file starts running - // The opensearch jar is closing down so won't leak but might prevent EphemeralClusterComposer to do its clean up. - // We do a hard kill on both here to make sure both processes are gone. - HardKill(HostProcessId); - HardKill(JavaProcessId); - } - - private static void HardKill(int? processId) - { - if (!processId.HasValue) return; - - try - { - var p = Process.GetProcessById(processId.Value); - p.Kill(); - } - catch (Exception) { } - } - - protected override bool ContinueReadingFromProcessReaders() - { - if (!NodeStarted) return true; - - return true; - - // some how if we return false here it leads to Task starvation in Proc and tests in e.g will OpenSearch.OpenSearch.Xunit will start - // to timeout. This makes little sense to me now, so leaving this performance optimization out for now. Hopefully another fresh look will yield - // to (not so) obvious. - //return this.NodeConfiguration.ShowOpenSearchOutputAfterStarted; - } - - protected override bool KeepBufferingLines(LineOut c) - { - var lineOutParser = LineOutParser.OpenSearch; - //if the node is already started only keep buffering lines while we have a writer and the nodeconfiguration wants output after started - if (NodeStarted) - { - var keepBuffering = Writer != null && NodeConfiguration.ShowOpenSearchOutputAfterStarted; - if (!keepBuffering) CancelAsyncReads(); - return keepBuffering; - } - - var parsed = lineOutParser.TryParse(c?.Line, out _, out _, out var section, out _, out var message, - out var started); - - if (!parsed) return Writer != null; - - if (JavaProcessId == null && lineOutParser.TryParseNodeInfo(section, message, out var version, out var pid)) - { - JavaProcessId = pid; - Version = version; - } - - else if (lineOutParser.TryGetPortNumber(section, message, out var port)) - { - Port = port; - var dp = NodeConfiguration.DesiredPort; - if (dp.HasValue && Port != dp.Value) - throw new OpenSearchCleanExitException( - $"Node started on port {port} but {dp.Value} was requested"); - } - - if (!started) started = AssumedStartedStateChecker(section, message); - if (started) - { - if (!Port.HasValue) - throw new OpenSearchCleanExitException( - $"Node started but OpenSearchNode did not grab its port number"); - - NodeStarted = true; - _startedHandle.Set(); - } - - // if we have dont a writer always return true - if (Writer != null) return true; - - //otherwise only keep buffering if we are not started - return !started; - } - } + private readonly ManualResetEvent _startedHandle = new ManualResetEvent(false); + + public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null) + : this(new NodeConfiguration(new ClusterConfiguration(version, (v, s) => new NodeFileSystem(v, openSearchHome)))) { } + + public OpenSearchNode(NodeConfiguration config) : base(StartArgs(config)) => NodeConfiguration = config; + + public string Version { get; private set; } + public int? Port { get; private set; } + public bool NodeStarted { get; private set; } + public NodeConfiguration NodeConfiguration { get; } + + private int? JavaProcessId { get; set; } + public override int? ProcessId => JavaProcessId ?? base.ProcessId; + public int? HostProcessId => base.ProcessId; + + /// + /// Set this true if you want the node to go into assumed started state as soon as its waiting for more nodes to start + /// doing the election. + /// Useful to speed up starting multi node clusters + /// + public bool AssumeStartedOnNotEnoughMasterPing { get; set; } + + internal IConsoleLineHandler Writer { get; private set; } + + public Exception LastSeenException { get; set; } + public WaitHandle StartedHandle => _startedHandle; + + private static StartArguments StartArgs(NodeConfiguration config) + { + //var args = new[] {config.FileSystem.Binary}.Concat(config.CommandLineArguments); + + var startArguments = new StartArguments(config.FileSystem.Binary, config.CommandLineArguments) + { + SendControlCFirst = true, + Environment = EnvVars(config), + WaitForExit = config.WaitForShutdown, + WaitForStreamReadersTimeout = config.WaitForShutdown + }; + config.ModifyStartArguments(startArguments); + return startArguments; + } + + private static Dictionary EnvVars(NodeConfiguration config) + { + var javaOpts = new List { "-Xms1g", "-Xmx1g" }; + + var environmentVariables = new Dictionary { { "OPENSEARCH_JAVA_OPTS", string.Join(" ", javaOpts) } }; + + if (!string.IsNullOrWhiteSpace(config.FileSystem.ConfigPath)) + environmentVariables.Add(config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath); + + if (!string.IsNullOrWhiteSpace(config.FileSystem.OpenSearchHome)) + environmentVariables.Add("OPENSEARCH_HOME", config.FileSystem.OpenSearchHome); + + var knnLibDir = Path.Combine(config.FileSystem.OpenSearchHome, "plugins", "opensearch-knn", config.Version >= "1.3.10" ? "lib" : "knnlib"); + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + AppendPathEnvVar("JAVA_LIBRARY_PATH", knnLibDir); + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + AppendPathEnvVar("LD_LIBRARY_PATH", knnLibDir); + + return environmentVariables; + } + + private static void AppendPathEnvVar(string name, string value) + { + var previous = Environment.GetEnvironmentVariable(name); + Environment.SetEnvironmentVariable(name, + string.IsNullOrWhiteSpace(previous) + ? value + : $"{previous}{Path.PathSeparator}{value}" + ); + } + + private bool AssumedStartedStateChecker(string section, string message) + { + if (AssumeStartedOnNotEnoughMasterPing + && section.Contains("ZenDiscovery") + && message.Contains("not enough master nodes discovered during pinging")) + return true; + + return false; + } + + public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); + + public IDisposable Start(TimeSpan waitForStarted) => Start(new LineHighlightWriter(), waitForStarted); + + public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) + { + var node = NodeConfiguration.DesiredNodeName; + var subscription = SubscribeLines(writer); + if (WaitForStarted(waitForStarted)) return subscription; + + subscription.Dispose(); + throw new OpenSearchCleanExitException( + $"Failed to start node: {node} before the configured timeout of: {waitForStarted}"); + } + + public IDisposable SubscribeLines() => SubscribeLines(new LineHighlightWriter()); + + public IDisposable SubscribeLines(IConsoleLineHandler writer) => + SubscribeLines(writer, delegate { }, delegate { }, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext) => + SubscribeLines(writer, onNext, delegate { }, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, + Action onError + ) => + SubscribeLines(writer, onNext, onError, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, Action onError, + Action onCompleted + ) + { + Writer = writer; + var node = NodeConfiguration.DesiredNodeName; + writer?.WriteDiagnostic($"OpenSearch location: [{Binary}]", node); + writer?.WriteDiagnostic($"Settings: {{{string.Join(" ", NodeConfiguration.CommandLineArguments)}}}", node); + + var envVarName = "JAVA_HOME"; + var javaHome = Environment.GetEnvironmentVariable(envVarName); + writer?.WriteDiagnostic($"{envVarName}: {{{javaHome}}}", node); + Process.StartInfo.Environment[envVarName] = javaHome; + + return SubscribeLines( + l => + { + writer?.Handle(l); + onNext?.Invoke(l); + }, + e => + { + LastSeenException = e; + writer?.Handle(e); + onError?.Invoke(e); + _startedHandle.Set(); + }, + () => + { + onCompleted?.Invoke(); + _startedHandle.Set(); + }); + } + + public bool WaitForStarted(TimeSpan timeout) => _startedHandle.WaitOne(timeout); + + protected override void OnBeforeSetCompletedHandle() + { + _startedHandle.Set(); + base.OnBeforeSetCompletedHandle(); + } + + protected override void OnBeforeWaitForEndOfStreamsError(TimeSpan waited) + { + // The wait for streams finished before streams were fully read. + // this usually indicates the process is still running. + // Proc will successfully kill the host but will leave the JavaProcess the bat file starts running + // The opensearch jar is closing down so won't leak but might prevent EphemeralClusterComposer to do its clean up. + // We do a hard kill on both here to make sure both processes are gone. + HardKill(HostProcessId); + HardKill(JavaProcessId); + } + + private static void HardKill(int? processId) + { + if (!processId.HasValue) return; + + try + { + var p = Process.GetProcessById(processId.Value); + p.Kill(); + } + catch (Exception) { } + } + + protected override bool ContinueReadingFromProcessReaders() + { + if (!NodeStarted) return true; + + return true; + + // some how if we return false here it leads to Task starvation in Proc and tests in e.g will OpenSearch.OpenSearch.Xunit will start + // to timeout. This makes little sense to me now, so leaving this performance optimization out for now. Hopefully another fresh look will yield + // to (not so) obvious. + //return this.NodeConfiguration.ShowOpenSearchOutputAfterStarted; + } + + protected override bool KeepBufferingLines(LineOut c) + { + var lineOutParser = LineOutParser.OpenSearch; + //if the node is already started only keep buffering lines while we have a writer and the nodeconfiguration wants output after started + if (NodeStarted) + { + var keepBuffering = Writer != null && NodeConfiguration.ShowOpenSearchOutputAfterStarted; + if (!keepBuffering) CancelAsyncReads(); + return keepBuffering; + } + + var parsed = lineOutParser.TryParse(c?.Line, out _, out _, out var section, out _, out var message, + out var started); + + if (!parsed) return Writer != null; + + if (JavaProcessId == null && lineOutParser.TryParseNodeInfo(section, message, out var version, out var pid)) + { + JavaProcessId = pid; + Version = version; + } + + else if (lineOutParser.TryGetPortNumber(section, message, out var port)) + { + Port = port; + var dp = NodeConfiguration.DesiredPort; + if (dp.HasValue && Port != dp.Value) + throw new OpenSearchCleanExitException( + $"Node started on port {port} but {dp.Value} was requested"); + } + + if (!started) started = AssumedStartedStateChecker(section, message); + if (started) + { + if (!Port.HasValue) + throw new OpenSearchCleanExitException( + $"Node started but OpenSearchNode did not grab its port number"); + + NodeStarted = true; + _startedHandle.Set(); + } + + // if we have dont a writer always return true + if (Writer != null) return true; + + //otherwise only keep buffering if we are not started + return !started; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs index 4efb622b98..030b93a59b 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs @@ -28,30 +28,29 @@ using System; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +/// +/// An assembly attribute that specifies the +/// for Xunit tests within the assembly. +/// +[AttributeUsage(AttributeTargets.Assembly)] +public class OpenSearchXunitConfigurationAttribute : Attribute { - /// - /// An assembly attribute that specifies the - /// for Xunit tests within the assembly. - /// - [AttributeUsage(AttributeTargets.Assembly)] - public class OpenSearchXunitConfigurationAttribute : Attribute - { - /// - /// Creates a new instance of - /// - /// - /// A type deriving from that specifies the run options - /// - public OpenSearchXunitConfigurationAttribute(Type type) - { - var options = Activator.CreateInstance(type) as OpenSearchXunitRunOptions; - Options = options ?? new OpenSearchXunitRunOptions(); - } + /// + /// Creates a new instance of + /// + /// + /// A type deriving from that specifies the run options + /// + public OpenSearchXunitConfigurationAttribute(Type type) + { + var options = Activator.CreateInstance(type) as OpenSearchXunitRunOptions; + Options = options ?? new OpenSearchXunitRunOptions(); + } - /// - /// The run options - /// - public OpenSearchXunitRunOptions Options { get; } - } + /// + /// The run options + /// + public OpenSearchXunitRunOptions Options { get; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs index ff841b76c3..f234d7b678 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs @@ -33,65 +33,64 @@ using OpenSearch.OpenSearch.Xunit.XunitPlumbing; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +/// +/// The Xunit test runner options +/// +public class OpenSearchXunitRunOptions { - /// - /// The Xunit test runner options - /// - public class OpenSearchXunitRunOptions - { - /// - /// Informs the runner whether we expect to run integration tests. Defaults to true - /// - public bool RunIntegrationTests { get; set; } = true; + /// + /// Informs the runner whether we expect to run integration tests. Defaults to true + /// + public bool RunIntegrationTests { get; set; } = true; - /// - /// Setting this to true will assume the cluster that is currently running was started for the purpose of these tests - /// Defaults to false - /// - public bool IntegrationTestsMayUseAlreadyRunningNode { get; set; } = false; + /// + /// Setting this to true will assume the cluster that is currently running was started for the purpose of these tests + /// Defaults to false + /// + public bool IntegrationTestsMayUseAlreadyRunningNode { get; set; } = false; - /// - /// Informs the runner whether unit tests will be run. Defaults to false. - /// If set to true and is false, the runner will run all the - /// tests in parallel with the maximum degree of parallelism - /// - public bool RunUnitTests { get; set; } + /// + /// Informs the runner whether unit tests will be run. Defaults to false. + /// If set to true and is false, the runner will run all the + /// tests in parallel with the maximum degree of parallelism + /// + public bool RunUnitTests { get; set; } - /// - /// A global test filter that can be used to only run certain tests. - /// Accepts a comma separated list of filters - /// - public string TestFilter { get; set; } + /// + /// A global test filter that can be used to only run certain tests. + /// Accepts a comma separated list of filters + /// + public string TestFilter { get; set; } - /// - /// A global cluster filter that can be used to only run certain cluster's tests. - /// Accepts a comma separated list of filters - /// - public string ClusterFilter { get; set; } + /// + /// A global cluster filter that can be used to only run certain cluster's tests. + /// Accepts a comma separated list of filters + /// + public string ClusterFilter { get; set; } - /// - /// Informs the runner what version of OpenSearch is under test. Required for - /// to kick in - /// - public OpenSearchVersion Version { get; set; } + /// + /// Informs the runner what version of OpenSearch is under test. Required for + /// to kick in + /// + public OpenSearchVersion Version { get; set; } - /// - /// Called when the tests have finished running successfully - /// - /// Per cluster timings of the total test time, including starting OpenSearch - /// All collection of failed cluster, failed tests tuples - public virtual void OnTestsFinished(Dictionary runnerClusterTotals, - ConcurrentBag> runnerFailedCollections) - { - } + /// + /// Called when the tests have finished running successfully + /// + /// Per cluster timings of the total test time, including starting OpenSearch + /// All collection of failed cluster, failed tests tuples + public virtual void OnTestsFinished(Dictionary runnerClusterTotals, + ConcurrentBag> runnerFailedCollections) + { + } - /// - /// Called before tests run. An ideal place to perform actions such as writing information to - /// . - /// - public virtual void OnBeforeTestsRun() - { - } - } + /// + /// Called before tests run. An ideal place to perform actions such as writing information to + /// . + /// + public virtual void OnBeforeTestsRun() + { + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs index b9ba0fec81..ed7b32685e 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs @@ -28,10 +28,9 @@ using OpenSearch.OpenSearch.Ephemeral; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +public static class OpenSearchXunitRunner { - public static class OpenSearchXunitRunner - { - public static IEphemeralCluster CurrentCluster { get; internal set; } - } + public static IEphemeralCluster CurrentCluster { get; internal set; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs index eea8e4f381..1c8e31fe8a 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs @@ -30,18 +30,17 @@ using OpenSearch.OpenSearch.Ephemeral.Tasks; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +/// +/// A task that writes a diagnostic message to indicate that tests will now run +/// +public class PrintXunitAfterStartedTask : ClusterComposeTask { - /// - /// A task that writes a diagnostic message to indicate that tests will now run - /// - public class PrintXunitAfterStartedTask : ClusterComposeTask - { - /// - public override void Run(IEphemeralCluster cluster) - { - var name = cluster.GetType().Name; - cluster.Writer.WriteDiagnostic($"All good! kicking off [{name}] tests now"); - } - } + /// + public override void Run(IEphemeralCluster cluster) + { + var name = cluster.GetType().Name; + cluster.Writer.WriteDiagnostic($"All good! kicking off [{name}] tests now"); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs index 8e2f4c61ff..108ca1d2e1 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs @@ -32,18 +32,17 @@ using System.Linq; using System.Threading.Tasks; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +internal static class ForEachAsyncExtensions { - internal static class ForEachAsyncExtensions - { - internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => - Task.WhenAll( - from partition in Partitioner.Create(source).GetPartitions(dop) - select Task.Run(async delegate - { - using (partition) - while (partition.MoveNext()) - await body(partition.Current).ConfigureAwait(false); - })); - } + internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => + Task.WhenAll( + from partition in Partitioner.Create(source).GetPartitions(dop) + select Task.Run(async delegate + { + using (partition) + while (partition.MoveNext()) + await body(partition.Current).ConfigureAwait(false); + })); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs index 621b6f0d84..b1b814efeb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs @@ -30,27 +30,26 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +public class OpenSearchTestFramework : XunitTestFramework { - public class OpenSearchTestFramework : XunitTestFramework - { - public OpenSearchTestFramework(IMessageSink messageSink) : base(messageSink) - { - } + public OpenSearchTestFramework(IMessageSink messageSink) : base(messageSink) + { + } - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => - new OpenSearchTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => + new OpenSearchTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) - { - var assembly = Assembly.Load(assemblyName); - var options = assembly.GetCustomAttribute()?.Options ?? - new OpenSearchXunitRunOptions(); + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + { + var assembly = Assembly.Load(assemblyName); + var options = assembly.GetCustomAttribute()?.Options ?? + new OpenSearchXunitRunOptions(); - return new TestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink) - { - Options = options - }; - } - } + return new TestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink) + { + Options = options + }; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs index b59692b347..32d5650abe 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs @@ -30,36 +30,35 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +public class OpenSearchTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer { - public class OpenSearchTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer - { - public OpenSearchTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, - IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) : base( - assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) - { - var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); - var options = a.GetCustomAttribute()?.Options ?? - new OpenSearchXunitRunOptions(); - Options = options; - } + public OpenSearchTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, + IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) : base( + assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) + { + var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); + var options = a.GetCustomAttribute()?.Options ?? + new OpenSearchXunitRunOptions(); + Options = options; + } - /// - /// The options for - /// - public OpenSearchXunitRunOptions Options { get; } + /// + /// The options for + /// + public OpenSearchXunitRunOptions Options { get; } - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, - IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) - { - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); - return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); - } - } + protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, + IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) + { + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs index 1b8f46420b..401fad4ecb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -40,268 +40,267 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +internal class TestAssemblyRunner : XunitTestAssemblyRunner { - internal class TestAssemblyRunner : XunitTestAssemblyRunner - { - private readonly Dictionary> _assemblyFixtureMappings = - new Dictionary>(); - - private readonly List, GroupedByCluster>> _grouped; - - public TestAssemblyRunner(ITestAssembly testAssembly, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) - { - var tests = OrderTestCollections(); - RunIntegrationTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); - IntegrationTestsMayUseAlreadyRunningNode = - executionOptions.GetValue(nameof(OpenSearchXunitRunOptions - .IntegrationTestsMayUseAlreadyRunningNode)); - RunUnitTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); - TestFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.TestFilter)); - ClusterFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter)); - - //bit side effecty, sets up _assemblyFixtureMappings before possibly letting xunit do its regular concurrency thing - _grouped = (from c in tests - let cluster = ClusterFixture(c.Item2.First().TestMethod.TestClass) - let testcase = new GroupedByCluster {Collection = c.Item1, TestCases = c.Item2, Cluster = cluster} - group testcase by testcase.Cluster - into g - orderby g.Count() descending - select g).ToList(); - } - - public ConcurrentBag Summaries { get; } = new ConcurrentBag(); - - public ConcurrentBag> FailedCollections { get; } = - new ConcurrentBag>(); - - public Dictionary ClusterTotals { get; } = new Dictionary(); - - private bool RunIntegrationTests { get; } - private bool IntegrationTestsMayUseAlreadyRunningNode { get; } - private bool RunUnitTests { get; } - private string TestFilter { get; } - private string ClusterFilter { get; } - - protected override Task RunTestCollectionAsync(IMessageBus b, ITestCollection c, - IEnumerable t, CancellationTokenSource s) - { - var aggregator = new ExceptionAggregator(Aggregator); - var fixtureObjects = new Dictionary(); - foreach (var kv in _assemblyFixtureMappings) fixtureObjects.Add(kv.Key, kv.Value); - return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, - s) - .RunAsync(); - } - - protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, - CancellationTokenSource cancellationTokenSource) - { - //threading guess - var defaultMaxConcurrency = Environment.ProcessorCount * 4; - - if (RunUnitTests && !RunIntegrationTests) - return await UnitTestPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) - .ConfigureAwait(false); - - return await IntegrationPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) - .ConfigureAwait(false); - } - - - private async Task UnitTestPipeline(int defaultMaxConcurrency, IMessageBus messageBus, - CancellationTokenSource ctx) - { - //make sure all clusters go in started state (won't actually start clusters in unit test mode) - //foreach (var g in this._grouped) g.Key?.Start(); - - var testFilters = CreateTestFilters(TestFilter); - await _grouped.SelectMany(g => g) - .ForEachAsync(defaultMaxConcurrency, - async g => { await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); }) - .ConfigureAwait(false); - //foreach (var g in this._grouped) g.Key?.Dispose(); - - return new RunSummary - { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; - } - - private async Task IntegrationPipeline(int defaultMaxConcurrency, IMessageBus messageBus, - CancellationTokenSource ctx) - { - var testFilters = CreateTestFilters(TestFilter); - foreach (var group in _grouped) - { - OpenSearchXunitRunner.CurrentCluster = @group.Key; - if (@group.Key == null) - { - var testCount = @group.SelectMany(q => q.TestCases).Count(); - Console.WriteLine($" -> Several tests skipped because they have no cluster associated"); - Summaries.Add(new RunSummary {Total = testCount, Skipped = testCount}); - continue; - } - - var type = @group.Key.GetType(); - var clusterName = type.Name.Replace("Cluster", string.Empty) ?? "UNKNOWN"; - if (!MatchesClusterFilter(clusterName)) continue; - - var dop = @group.Key.ClusterConfiguration?.MaxConcurrency ?? defaultMaxConcurrency; - dop = dop <= 0 ? defaultMaxConcurrency : dop; - - var timeout = @group.Key.ClusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); - - var skipReasons = @group.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); - var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); - if (allSkipped) - { - Console.WriteLine($" -> All tests from {clusterName} are skipped under the current configuration"); - Summaries.Add(new RunSummary {Total = skipReasons.Count, Skipped = skipReasons.Count}); - continue; - } - - ClusterTotals.Add(clusterName, Stopwatch.StartNew()); - - bool ValidateRunningVersion() - { - try - { - var t = new ValidateRunningVersion(); - t.Run(@group.Key); - return true; - } - catch (Exception) - { - return false; - } - } - - using (@group.Key) - { - if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion()) - @group.Key?.Start(timeout); - - await @group.ForEachAsync(dop, - async g => - { - await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); - }) - .ConfigureAwait(false); - } - - ClusterTotals[clusterName].Stop(); - } - - return new RunSummary - { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; - } - - private async Task RunTestCollections(IMessageBus messageBus, CancellationTokenSource ctx, GroupedByCluster g, - string[] testFilters) - { - var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); - if (!MatchesATestFilter(test, testFilters)) return; - if (testFilters.Length > 0) Console.WriteLine(" -> " + test); - - try - { - var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) - .ConfigureAwait(false); - var type = g.Cluster?.GetType(); - var clusterName = type?.Name.Replace("Cluster", "") ?? "UNKNOWN"; - if (summary.Failed > 0) - FailedCollections.Add(Tuple.Create(clusterName, test)); - Summaries.Add(summary); - } - catch (TaskCanceledException) - { - // TODO: What should happen here? - } - } - - private static string[] CreateTestFilters(string testFilters) => - testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() - ?? new string[] { }; - - private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) - { - if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; - return testFilters - .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private bool MatchesClusterFilter(string cluster) - { - if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(ClusterFilter)) return true; - return ClusterFilter - .Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries) - .Select(c => c.Trim()) - .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private IEphemeralCluster ClusterFixture(ITestClass testMethodTestClass) - { - var clusterType = GetClusterForClass(testMethodTestClass.Class); - if (clusterType == null) return null; - - if (_assemblyFixtureMappings.TryGetValue(clusterType, out var cluster)) return cluster; - Aggregator.Run(() => - { - var o = Activator.CreateInstance(clusterType); - cluster = o as IEphemeralCluster; - }); - _assemblyFixtureMappings.Add(clusterType, cluster); - return cluster; - } - - public static bool IsAnIntegrationTestClusterType(Type type) => - typeof(XunitClusterBase).GetTypeInfo().IsAssignableFrom(type.GetTypeInfo()) - || IsSubclassOfRawGeneric(typeof(XunitClusterBase<>), type); - - public static Type GetClusterForClass(ITypeInfo testClass) => - GetClusterFromClassClusterFixture(testClass) ?? GetClusterFromIntegrationAttribute(testClass); - - private static Type GetClusterFromClassClusterFixture(ITypeInfo testClass) => ( - from i in testClass.Interfaces - where i.IsGenericType - from a in i.GetGenericArguments() - select a.ToRuntimeType() - ).FirstOrDefault(IsAnIntegrationTestClusterType); - - private static Type GetClusterFromIntegrationAttribute(ITypeInfo testClass) => - testClass.GetCustomAttributes(typeof(IntegrationTestClusterAttribute)) - .FirstOrDefault()?.GetNamedArgument(nameof(IntegrationTestClusterAttribute.ClusterType)); - - private static bool IsSubclassOfRawGeneric(Type generic, Type toCheck) - { - while (toCheck != null && toCheck != typeof(object)) - { - var cur = toCheck.GetTypeInfo().IsGenericType ? toCheck.GetGenericTypeDefinition() : toCheck; - if (generic == cur) return true; - - toCheck = toCheck.GetTypeInfo().BaseType; - } - - return false; - } - - private class GroupedByCluster - { - public IEphemeralCluster Cluster { get; set; } - public ITestCollection Collection { get; set; } - public List TestCases { get; set; } - } - } + private readonly Dictionary> _assemblyFixtureMappings = + new Dictionary>(); + + private readonly List, GroupedByCluster>> _grouped; + + public TestAssemblyRunner(ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + { + var tests = OrderTestCollections(); + RunIntegrationTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); + IntegrationTestsMayUseAlreadyRunningNode = + executionOptions.GetValue(nameof(OpenSearchXunitRunOptions + .IntegrationTestsMayUseAlreadyRunningNode)); + RunUnitTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); + TestFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.TestFilter)); + ClusterFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter)); + + //bit side effecty, sets up _assemblyFixtureMappings before possibly letting xunit do its regular concurrency thing + _grouped = (from c in tests + let cluster = ClusterFixture(c.Item2.First().TestMethod.TestClass) + let testcase = new GroupedByCluster { Collection = c.Item1, TestCases = c.Item2, Cluster = cluster } + group testcase by testcase.Cluster + into g + orderby g.Count() descending + select g).ToList(); + } + + public ConcurrentBag Summaries { get; } = new ConcurrentBag(); + + public ConcurrentBag> FailedCollections { get; } = + new ConcurrentBag>(); + + public Dictionary ClusterTotals { get; } = new Dictionary(); + + private bool RunIntegrationTests { get; } + private bool IntegrationTestsMayUseAlreadyRunningNode { get; } + private bool RunUnitTests { get; } + private string TestFilter { get; } + private string ClusterFilter { get; } + + protected override Task RunTestCollectionAsync(IMessageBus b, ITestCollection c, + IEnumerable t, CancellationTokenSource s) + { + var aggregator = new ExceptionAggregator(Aggregator); + var fixtureObjects = new Dictionary(); + foreach (var kv in _assemblyFixtureMappings) fixtureObjects.Add(kv.Key, kv.Value); + return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, + s) + .RunAsync(); + } + + protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, + CancellationTokenSource cancellationTokenSource) + { + //threading guess + var defaultMaxConcurrency = Environment.ProcessorCount * 4; + + if (RunUnitTests && !RunIntegrationTests) + return await UnitTestPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + .ConfigureAwait(false); + + return await IntegrationPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + .ConfigureAwait(false); + } + + + private async Task UnitTestPipeline(int defaultMaxConcurrency, IMessageBus messageBus, + CancellationTokenSource ctx) + { + //make sure all clusters go in started state (won't actually start clusters in unit test mode) + //foreach (var g in this._grouped) g.Key?.Start(); + + var testFilters = CreateTestFilters(TestFilter); + await _grouped.SelectMany(g => g) + .ForEachAsync(defaultMaxConcurrency, + async g => { await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); }) + .ConfigureAwait(false); + //foreach (var g in this._grouped) g.Key?.Dispose(); + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + private async Task IntegrationPipeline(int defaultMaxConcurrency, IMessageBus messageBus, + CancellationTokenSource ctx) + { + var testFilters = CreateTestFilters(TestFilter); + foreach (var group in _grouped) + { + OpenSearchXunitRunner.CurrentCluster = @group.Key; + if (@group.Key == null) + { + var testCount = @group.SelectMany(q => q.TestCases).Count(); + Console.WriteLine($" -> Several tests skipped because they have no cluster associated"); + Summaries.Add(new RunSummary { Total = testCount, Skipped = testCount }); + continue; + } + + var type = @group.Key.GetType(); + var clusterName = type.Name.Replace("Cluster", string.Empty) ?? "UNKNOWN"; + if (!MatchesClusterFilter(clusterName)) continue; + + var dop = @group.Key.ClusterConfiguration?.MaxConcurrency ?? defaultMaxConcurrency; + dop = dop <= 0 ? defaultMaxConcurrency : dop; + + var timeout = @group.Key.ClusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); + + var skipReasons = @group.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); + var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); + if (allSkipped) + { + Console.WriteLine($" -> All tests from {clusterName} are skipped under the current configuration"); + Summaries.Add(new RunSummary { Total = skipReasons.Count, Skipped = skipReasons.Count }); + continue; + } + + ClusterTotals.Add(clusterName, Stopwatch.StartNew()); + + bool ValidateRunningVersion() + { + try + { + var t = new ValidateRunningVersion(); + t.Run(@group.Key); + return true; + } + catch (Exception) + { + return false; + } + } + + using (@group.Key) + { + if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion()) + @group.Key?.Start(timeout); + + await @group.ForEachAsync(dop, + async g => + { + await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); + }) + .ConfigureAwait(false); + } + + ClusterTotals[clusterName].Stop(); + } + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + private async Task RunTestCollections(IMessageBus messageBus, CancellationTokenSource ctx, GroupedByCluster g, + string[] testFilters) + { + var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); + if (!MatchesATestFilter(test, testFilters)) return; + if (testFilters.Length > 0) Console.WriteLine(" -> " + test); + + try + { + var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) + .ConfigureAwait(false); + var type = g.Cluster?.GetType(); + var clusterName = type?.Name.Replace("Cluster", "") ?? "UNKNOWN"; + if (summary.Failed > 0) + FailedCollections.Add(Tuple.Create(clusterName, test)); + Summaries.Add(summary); + } + catch (TaskCanceledException) + { + // TODO: What should happen here? + } + } + + private static string[] CreateTestFilters(string testFilters) => + testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() + ?? new string[] { }; + + private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) + { + if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; + return testFilters + .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private bool MatchesClusterFilter(string cluster) + { + if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(ClusterFilter)) return true; + return ClusterFilter + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Select(c => c.Trim()) + .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private IEphemeralCluster ClusterFixture(ITestClass testMethodTestClass) + { + var clusterType = GetClusterForClass(testMethodTestClass.Class); + if (clusterType == null) return null; + + if (_assemblyFixtureMappings.TryGetValue(clusterType, out var cluster)) return cluster; + Aggregator.Run(() => + { + var o = Activator.CreateInstance(clusterType); + cluster = o as IEphemeralCluster; + }); + _assemblyFixtureMappings.Add(clusterType, cluster); + return cluster; + } + + public static bool IsAnIntegrationTestClusterType(Type type) => + typeof(XunitClusterBase).GetTypeInfo().IsAssignableFrom(type.GetTypeInfo()) + || IsSubclassOfRawGeneric(typeof(XunitClusterBase<>), type); + + public static Type GetClusterForClass(ITypeInfo testClass) => + GetClusterFromClassClusterFixture(testClass) ?? GetClusterFromIntegrationAttribute(testClass); + + private static Type GetClusterFromClassClusterFixture(ITypeInfo testClass) => ( + from i in testClass.Interfaces + where i.IsGenericType + from a in i.GetGenericArguments() + select a.ToRuntimeType() + ).FirstOrDefault(IsAnIntegrationTestClusterType); + + private static Type GetClusterFromIntegrationAttribute(ITypeInfo testClass) => + testClass.GetCustomAttributes(typeof(IntegrationTestClusterAttribute)) + .FirstOrDefault()?.GetNamedArgument(nameof(IntegrationTestClusterAttribute.ClusterType)); + + private static bool IsSubclassOfRawGeneric(Type generic, Type toCheck) + { + while (toCheck != null && toCheck != typeof(object)) + { + var cur = toCheck.GetTypeInfo().IsGenericType ? toCheck.GetGenericTypeDefinition() : toCheck; + if (generic == cur) return true; + + toCheck = toCheck.GetTypeInfo().BaseType; + } + + return false; + } + + private class GroupedByCluster + { + public IEphemeralCluster Cluster { get; set; } + public ITestCollection Collection { get; set; } + public List TestCases { get; set; } + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs index 0f9711cc0e..6631a6e227 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs @@ -33,40 +33,39 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +internal class TestCollectionRunner : XunitTestCollectionRunner { - internal class TestCollectionRunner : XunitTestCollectionRunner - { - private readonly Dictionary _assemblyFixtureMappings; - private readonly IMessageSink _diagnosticMessageSink; + private readonly Dictionary _assemblyFixtureMappings; + private readonly IMessageSink _diagnosticMessageSink; - public TestCollectionRunner(Dictionary assemblyFixtureMappings, - ITestCollection testCollection, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - ITestCaseOrderer testCaseOrderer, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, - cancellationTokenSource) - { - _assemblyFixtureMappings = assemblyFixtureMappings; - _diagnosticMessageSink = diagnosticMessageSink; - } + public TestCollectionRunner(Dictionary assemblyFixtureMappings, + ITestCollection testCollection, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, + cancellationTokenSource) + { + _assemblyFixtureMappings = assemblyFixtureMappings; + _diagnosticMessageSink = diagnosticMessageSink; + } - protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, - IEnumerable testCases) - { - // whats this doing exactly?? - var combinedFixtures = new Dictionary(_assemblyFixtureMappings); - foreach (var kvp in CollectionFixtureMappings) - combinedFixtures[kvp.Key] = kvp.Value; + protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, + IEnumerable testCases) + { + // whats this doing exactly?? + var combinedFixtures = new Dictionary(_assemblyFixtureMappings); + foreach (var kvp in CollectionFixtureMappings) + combinedFixtures[kvp.Key] = kvp.Value; - // We've done everything we need, so hand back off to default Xunit implementation for class runner - return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, - TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) - .RunAsync(); - } - } + // We've done everything we need, so hand back off to default Xunit implementation for class runner + return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, + TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) + .RunAsync(); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs index 2ec6b3df2e..e1e446d93f 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs @@ -34,82 +34,81 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.Sdk +namespace OpenSearch.OpenSearch.Xunit.Sdk; + +internal class TestFrameworkExecutor : XunitTestFrameworkExecutor { - internal class TestFrameworkExecutor : XunitTestFrameworkExecutor - { - public TestFrameworkExecutor(AssemblyName a, ISourceInformationProvider sip, IMessageSink d) : base(a, sip, d) - { - } + public TestFrameworkExecutor(AssemblyName a, ISourceInformationProvider sip, IMessageSink d) : base(a, sip, d) + { + } - public OpenSearchXunitRunOptions Options { get; set; } + public OpenSearchXunitRunOptions Options { get; set; } - public override void RunAll(IMessageSink executionMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions, - ITestFrameworkExecutionOptions executionOptions) - { - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); - discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + public override void RunAll(IMessageSink executionMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions, + ITestFrameworkExecutionOptions executionOptions) + { + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); - base.RunAll(executionMessageSink, discoveryOptions, executionOptions); - } + base.RunAll(executionMessageSink, discoveryOptions, executionOptions); + } - public override void RunTests(IEnumerable testCases, IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - { - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); - executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); - base.RunTests(testCases, executionMessageSink, executionOptions); - } + public override void RunTests(IEnumerable testCases, IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + { + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + base.RunTests(testCases, executionMessageSink, executionOptions); + } - protected override async void RunTestCases(IEnumerable testCases, IMessageSink sink, - ITestFrameworkExecutionOptions options) - { - options.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); - options.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - options.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - options.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); - options.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); - options.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); - try - { - using (var runner = - new TestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, sink, options)) - { - Options.OnBeforeTestsRun(); - await runner.RunAsync().ConfigureAwait(false); - Options.OnTestsFinished(runner.ClusterTotals, runner.FailedCollections); - } - } - catch (Exception e) - { - if (e is OpenSearchCleanExitException || e is AggregateException ae && - ae.Flatten().InnerException is OpenSearchCleanExitException) - sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, - new OpenSearchCleanExitException("Node failed to start", e))); - else - sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); - throw; - } - } - } + protected override async void RunTestCases(IEnumerable testCases, IMessageSink sink, + ITestFrameworkExecutionOptions options) + { + options.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + options.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + options.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + options.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + options.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + options.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + try + { + using (var runner = + new TestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, sink, options)) + { + Options.OnBeforeTestsRun(); + await runner.RunAsync().ConfigureAwait(false); + Options.OnTestsFinished(runner.ClusterTotals, runner.FailedCollections); + } + } + catch (Exception e) + { + if (e is OpenSearchCleanExitException || e is AggregateException ae && + ae.Flatten().InnerException is OpenSearchCleanExitException) + sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, + new OpenSearchCleanExitException("Node failed to start", e))); + else + sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); + throw; + } + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs index 2c7f5d15b7..9e9a22a73f 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs @@ -28,26 +28,25 @@ using OpenSearch.OpenSearch.Ephemeral; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +/// +/// Base class for a cluster that integrates with Xunit tests +/// +public abstract class XunitClusterBase : XunitClusterBase { - /// - /// Base class for a cluster that integrates with Xunit tests - /// - public abstract class XunitClusterBase : XunitClusterBase - { - protected XunitClusterBase(XunitClusterConfiguration configuration) : base(configuration) - { - } - } + protected XunitClusterBase(XunitClusterConfiguration configuration) : base(configuration) + { + } +} - /// - /// Base class for a cluster that integrates with Xunit tests - /// - public abstract class XunitClusterBase : EphemeralCluster - where TConfiguration : XunitClusterConfiguration - { - protected XunitClusterBase(TConfiguration configuration) : base(configuration) - { - } - } +/// +/// Base class for a cluster that integrates with Xunit tests +/// +public abstract class XunitClusterBase : EphemeralCluster + where TConfiguration : XunitClusterConfiguration +{ + protected XunitClusterBase(TConfiguration configuration) : base(configuration) + { + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs index f852ee7820..09501eb6cf 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs @@ -31,29 +31,28 @@ using OpenSearch.OpenSearch.Ephemeral.Plugins; using OpenSearch.Stack.ArtifactsApi; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +public class XunitClusterConfiguration : EphemeralClusterConfiguration { - public class XunitClusterConfiguration : EphemeralClusterConfiguration - { - public XunitClusterConfiguration( - OpenSearchVersion version, - ClusterFeatures features = ClusterFeatures.None, - OpenSearchPlugins plugins = null, - int numberOfNodes = 1) - : base(version, features, plugins, numberOfNodes) => - AdditionalAfterStartedTasks.Add(new PrintXunitAfterStartedTask()); + public XunitClusterConfiguration( + OpenSearchVersion version, + ClusterFeatures features = ClusterFeatures.None, + OpenSearchPlugins plugins = null, + int numberOfNodes = 1) + : base(version, features, plugins, numberOfNodes) => + AdditionalAfterStartedTasks.Add(new PrintXunitAfterStartedTask()); - /// - protected override string NodePrefix => "xunit"; + /// + protected override string NodePrefix => "xunit"; - /// - /// The maximum number of tests that can run concurrently against a cluster using this configuration. - /// - public int MaxConcurrency { get; set; } + /// + /// The maximum number of tests that can run concurrently against a cluster using this configuration. + /// + public int MaxConcurrency { get; set; } - /// - /// The maximum amount of time a cluster can run using this configuration. - /// - public TimeSpan? Timeout { get; set; } - } + /// + /// The maximum amount of time a cluster can run using this configuration. + /// + public TimeSpan? Timeout { get; set; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs index f27ca8208f..5ed7f9f59d 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs @@ -30,24 +30,23 @@ using System.Collections.Concurrent; using OpenSearch.OpenSearch.Ephemeral; -namespace OpenSearch.OpenSearch.Xunit +namespace OpenSearch.OpenSearch.Xunit; + +/// +/// Extension methods for +/// +public static class XunitClusterExtensions { - /// - /// Extension methods for - /// - public static class XunitClusterExtensions - { - private static readonly ConcurrentDictionary Clients = - new ConcurrentDictionary(); + private static readonly ConcurrentDictionary Clients = + new ConcurrentDictionary(); - /// - /// Gets a client for the cluster if one exists, or creates a new client if one doesn't. - /// - /// the cluster to create a client for - /// A delegate to create a client, given the cluster to create it for - /// the type of the client - /// An instance of a client - public static T GetOrAddClient(this IEphemeralCluster cluster, Func getOrAdd) => - (T) Clients.GetOrAdd(cluster, c => getOrAdd(c)); - } + /// + /// Gets a client for the cluster if one exists, or creates a new client if one doesn't. + /// + /// the cluster to create a client for + /// A delegate to create a client, given the cluster to create it for + /// the type of the client + /// An instance of a client + public static T GetOrAddClient(this IEphemeralCluster cluster, Func getOrAdd) => + (T)Clients.GetOrAdd(cluster, c => getOrAdd(c)); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs index 852f47775b..20c21ecd92 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs @@ -29,12 +29,11 @@ using OpenSearch.OpenSearch.Ephemeral; using OpenSearch.OpenSearch.Managed; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +// ReSharper disable once UnusedTypeParameter +// used by the runner to new() the proper cluster +public interface IClusterFixture + where TCluster : ICluster, new() { - // ReSharper disable once UnusedTypeParameter - // used by the runner to new() the proper cluster - public interface IClusterFixture - where TCluster : ICluster, new() - { - } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs index 0f074e6d3b..2153a11fb8 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs @@ -29,19 +29,18 @@ using System; using OpenSearch.OpenSearch.Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class IntegrationTestClusterAttribute : Attribute { - [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] - public class IntegrationTestClusterAttribute : Attribute - { - public IntegrationTestClusterAttribute(Type clusterType) - { - if (!TestAssemblyRunner.IsAnIntegrationTestClusterType(clusterType)) - throw new ArgumentException( - $"Cluster must be subclass of {nameof(XunitClusterBase)} or {nameof(XunitClusterBase)}<>"); - ClusterType = clusterType; - } + public IntegrationTestClusterAttribute(Type clusterType) + { + if (!TestAssemblyRunner.IsAnIntegrationTestClusterType(clusterType)) + throw new ArgumentException( + $"Cluster must be subclass of {nameof(XunitClusterBase)} or {nameof(XunitClusterBase)}<>"); + ClusterType = clusterType; + } - public Type ClusterType { get; } - } + public Type ClusterType { get; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs index 9e84b0ee01..a26626f07f 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs @@ -37,100 +37,99 @@ using Xunit.Sdk; using Enumerable = System.Linq.Enumerable; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +/// +/// An Xunit test that should be skipped, and a reason why. +/// +public abstract class SkipTestAttributeBase : Attribute { - /// - /// An Xunit test that should be skipped, and a reason why. - /// - public abstract class SkipTestAttributeBase : Attribute - { - /// - /// Whether the test should be skipped - /// - public abstract bool Skip { get; } + /// + /// Whether the test should be skipped + /// + public abstract bool Skip { get; } - /// - /// The reason why the test should be skipped - /// - public abstract string Reason { get; } - } + /// + /// The reason why the test should be skipped + /// + public abstract string Reason { get; } +} - /// - /// An Xunit integration test - /// - [XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer", - "OpenSearch.OpenSearch.Xunit")] - public class I : FactAttribute - { - } +/// +/// An Xunit integration test +/// +[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer", + "OpenSearch.OpenSearch.Xunit")] +public class I : FactAttribute +{ +} - /// - /// A test discoverer used to discover integration tests cases attached - /// to test methods that are attributed with attribute - /// - public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer - { - public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) - { - } +/// +/// A test discoverer used to discover integration tests cases attached +/// to test methods that are attributed with attribute +/// +public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer +{ + public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) + { + } - /// - protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, - out string skipReason) - { - skipReason = null; - var runIntegrationTests = - discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); - if (!runIntegrationTests) return true; + /// + protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + var runIntegrationTests = + discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); + if (!runIntegrationTests) return true; - var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class); - if (cluster == null) - { - skipReason += - $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}"; - return true; - } + var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class); + if (cluster == null) + { + skipReason += + $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}"; + return true; + } - var openSearchVersion = - discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.Version)); + var openSearchVersion = + discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.Version)); - // Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges) - var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes(testMethod)); - if (skipVersionAttribute != null) - { - var skipVersionRanges = - skipVersionAttribute.GetNamedArgument>(nameof(SkipVersionAttribute.Ranges)) ?? - new List(); - if (openSearchVersion == null && skipVersionRanges.Count > 0) - { - skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " + - $"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}"; - return true; - } + // Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges) + var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes(testMethod)); + if (skipVersionAttribute != null) + { + var skipVersionRanges = + skipVersionAttribute.GetNamedArgument>(nameof(SkipVersionAttribute.Ranges)) ?? + new List(); + if (openSearchVersion == null && skipVersionRanges.Count > 0) + { + skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " + + $"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}"; + return true; + } - if (openSearchVersion != null) - { - var reason = skipVersionAttribute.GetNamedArgument(nameof(SkipVersionAttribute.Reason)); - for (var index = 0; index < skipVersionRanges.Count; index++) - { - var range = skipVersionRanges[index]; - // inrange takes prereleases into account - if (!openSearchVersion.InRange(range)) continue; - skipReason = - $"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies"; - if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}"; - return true; - } - } - } + if (openSearchVersion != null) + { + var reason = skipVersionAttribute.GetNamedArgument(nameof(SkipVersionAttribute.Reason)); + for (var index = 0; index < skipVersionRanges.Count; index++) + { + var range = skipVersionRanges[index]; + // inrange takes prereleases into account + if (!openSearchVersion.InRange(range)) continue; + skipReason = + $"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies"; + if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}"; + return true; + } + } + } - var skipTests = GetAttributes(testMethod) - .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); + var skipTests = GetAttributes(testMethod) + .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); - if (skipTests == null) return false; + if (skipTests == null) return false; - skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); - return true; - } - } + skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); + return true; + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs index 41f694101d..6caa999387 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs @@ -32,80 +32,79 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +/// +/// Base test discoverer used to discover tests cases attached +/// to test methods that are attributed with (or a subclass). +/// +public abstract class OpenSearchTestCaseDiscoverer : IXunitTestCaseDiscoverer { - /// - /// Base test discoverer used to discover tests cases attached - /// to test methods that are attributed with (or a subclass). - /// - public abstract class OpenSearchTestCaseDiscoverer : IXunitTestCaseDiscoverer - { - protected readonly IMessageSink DiagnosticMessageSink; - private readonly FactDiscoverer _factDiscoverer; + protected readonly IMessageSink DiagnosticMessageSink; + private readonly FactDiscoverer _factDiscoverer; - protected OpenSearchTestCaseDiscoverer(IMessageSink diagnosticMessageSink) - { - DiagnosticMessageSink = diagnosticMessageSink; - _factDiscoverer = new FactDiscoverer(diagnosticMessageSink); - } + protected OpenSearchTestCaseDiscoverer(IMessageSink diagnosticMessageSink) + { + DiagnosticMessageSink = diagnosticMessageSink; + _factDiscoverer = new FactDiscoverer(diagnosticMessageSink); + } - /// - public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, IAttributeInfo factAttribute) => - SkipMethod(discoveryOptions, testMethod, out var skipReason) - ? string.IsNullOrEmpty(skipReason) - ? Enumerable.Empty() - : new IXunitTestCase[] {new SkippingTestCase(skipReason, testMethod, null)} - : DiscoverImpl(discoveryOptions, testMethod, factAttribute); + /// + public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, IAttributeInfo factAttribute) => + SkipMethod(discoveryOptions, testMethod, out var skipReason) + ? string.IsNullOrEmpty(skipReason) + ? Enumerable.Empty() + : new IXunitTestCase[] { new SkippingTestCase(skipReason, testMethod, null) } + : DiscoverImpl(discoveryOptions, testMethod, factAttribute); - protected virtual IEnumerable DiscoverImpl(ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, IAttributeInfo factAttribute - ) => _factDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); + protected virtual IEnumerable DiscoverImpl(ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, IAttributeInfo factAttribute + ) => _factDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - /// - /// Detemines whether a test method should be skipped, and the reason why - /// - /// The discovery options - /// The test method - /// The reason to skip - /// - protected virtual bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, - out string skipReason) - { - skipReason = null; - return false; - } + /// + /// Detemines whether a test method should be skipped, and the reason why + /// + /// The discovery options + /// The test method + /// The reason to skip + /// + protected virtual bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + return false; + } - protected static TValue GetAttribute(ITestMethod testMethod, string propertyName) - where TAttribute : Attribute - { - var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - var attribute = classAttributes.Concat(methodAttributes).FirstOrDefault(); - return attribute == null ? default(TValue) : attribute.GetNamedArgument(propertyName); - } + protected static TValue GetAttribute(ITestMethod testMethod, string propertyName) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var attribute = classAttributes.Concat(methodAttributes).FirstOrDefault(); + return attribute == null ? default(TValue) : attribute.GetNamedArgument(propertyName); + } - protected static IList GetAttributes(ITestMethod testMethod) - where TAttribute : Attribute - { - var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - return classAttributes.Concat(methodAttributes).ToList(); - } + protected static IList GetAttributes(ITestMethod testMethod) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + return classAttributes.Concat(methodAttributes).ToList(); + } - protected static IEnumerable GetAttributes(ITestMethod testMethod, - string propertyName) - where TAttribute : Attribute - { - var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)); - var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)); - return classAttributes - .Concat(methodAttributes) - .Select(a => a.GetNamedArgument(propertyName)); - } - } + protected static IEnumerable GetAttributes(ITestMethod testMethod, + string propertyName) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)); + return classAttributes + .Concat(methodAttributes) + .Select(a => a.GetNamedArgument(propertyName)); + } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs index cfeec7b8da..26d174c6ab 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs @@ -31,35 +31,34 @@ using System.Linq; using SemanticVersioning; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +/// +/// An Xunit test that should be skipped for given OpenSearch versions, and a reason why. +/// +public class SkipVersionAttribute : Attribute { - /// - /// An Xunit test that should be skipped for given OpenSearch versions, and a reason why. - /// - public class SkipVersionAttribute : Attribute - { - // ReSharper disable once UnusedParameter.Local - // reason is used to allow the test its used on to self document why its been put in place - public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason) - { - Reason = reason; - Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma) - ? new List() - : skipVersionRangesSeparatedByComma.Split(',') - .Select(r => r.Trim()) - .Where(r => !string.IsNullOrWhiteSpace(r)) - .Select(r => new Range(r)) - .ToList(); - } + // ReSharper disable once UnusedParameter.Local + // reason is used to allow the test its used on to self document why its been put in place + public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason) + { + Reason = reason; + Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma) + ? new List() + : skipVersionRangesSeparatedByComma.Split(',') + .Select(r => r.Trim()) + .Where(r => !string.IsNullOrWhiteSpace(r)) + .Select(r => new Range(r)) + .ToList(); + } - /// - /// The reason why the test should be skipped - /// - public string Reason { get; } + /// + /// The reason why the test should be skipped + /// + public string Reason { get; } - /// - /// The version ranges for which the test should be skipped - /// - public IList Ranges { get; } - } + /// + /// The version ranges for which the test should be skipped + /// + public IList Ranges { get; } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs index b5c8b44e09..995de00e16 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs @@ -31,41 +31,40 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +public class SkippingTestCase : TestMethodTestCase, IXunitTestCase { - public class SkippingTestCase : TestMethodTestCase, IXunitTestCase - { - /// Used for de-serialization. - public SkippingTestCase() - { - } + /// Used for de-serialization. + public SkippingTestCase() + { + } - /// - /// Initializes a new instance of the class. - /// - /// The test method this test case belongs to. - /// The arguments for the test method. - public SkippingTestCase(string skipReason, ITestMethod testMethod, object[] testMethodArguments = null) - : base(TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod, testMethodArguments) => - SkipReason = skipReason ?? "skipped"; + /// + /// Initializes a new instance of the class. + /// + /// The test method this test case belongs to. + /// The arguments for the test method. + public SkippingTestCase(string skipReason, ITestMethod testMethod, object[] testMethodArguments = null) + : base(TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod, testMethodArguments) => + SkipReason = skipReason ?? "skipped"; - public int Timeout => 0; + public int Timeout => 0; - /// - public Task RunAsync( - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - object[] constructorArguments, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) => - new XunitTestCaseRunner( - this, - DisplayName, - SkipReason, - constructorArguments, - TestMethodArguments, - messageBus, - aggregator, - cancellationTokenSource).RunAsync(); - } + /// + public Task RunAsync( + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + object[] constructorArguments, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) => + new XunitTestCaseRunner( + this, + DisplayName, + SkipReason, + constructorArguments, + TestMethodArguments, + messageBus, + aggregator, + cancellationTokenSource).RunAsync(); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/TheoryUnitTestDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/TheoryUnitTestDiscoverer.cs index 29ef1b6475..b0e761247b 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/TheoryUnitTestDiscoverer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/TheoryUnitTestDiscoverer.cs @@ -10,26 +10,25 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +/// +/// An Xunit theory unit test +/// +[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.TheoryUnitTestDiscoverer", + "OpenSearch.OpenSearch.Xunit")] +public class TU : TheoryAttribute { - /// - /// An Xunit theory unit test - /// - [XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.TheoryUnitTestDiscoverer", - "OpenSearch.OpenSearch.Xunit")] - public class TU : TheoryAttribute - { - } +} - public class TheoryUnitTestDiscoverer : UnitTestDiscoverer - { - private readonly TheoryDiscoverer _discoverer; +public class TheoryUnitTestDiscoverer : UnitTestDiscoverer +{ + private readonly TheoryDiscoverer _discoverer; - public TheoryUnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) => - _discoverer = new TheoryDiscoverer(diagnosticMessageSink); + public TheoryUnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) => + _discoverer = new TheoryDiscoverer(diagnosticMessageSink); - protected override IEnumerable DiscoverImpl(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, - IAttributeInfo factAttribute - ) => _discoverer.Discover(discoveryOptions, testMethod, factAttribute); - } + protected override IEnumerable DiscoverImpl(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + IAttributeInfo factAttribute + ) => _discoverer.Discover(discoveryOptions, testMethod, factAttribute); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs index 763a385c48..9d128c8b71 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs @@ -31,42 +31,41 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing; + +/// +/// An Xunit unit test +/// +[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.UnitTestDiscoverer", + "OpenSearch.OpenSearch.Xunit")] +public class U : FactAttribute { - /// - /// An Xunit unit test - /// - [XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.UnitTestDiscoverer", - "OpenSearch.OpenSearch.Xunit")] - public class U : FactAttribute - { - } +} - /// - /// A test discoverer used to discover unit tests cases attached - /// to test methods that are attributed with attribute - /// - public class UnitTestDiscoverer : OpenSearchTestCaseDiscoverer - { - public UnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) - { - } +/// +/// A test discoverer used to discover unit tests cases attached +/// to test methods that are attributed with attribute +/// +public class UnitTestDiscoverer : OpenSearchTestCaseDiscoverer +{ + public UnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) + { + } - /// - protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, - out string skipReason) - { - skipReason = null; - var runUnitTests = discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); - if (!runUnitTests) return true; + /// + protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + var runUnitTests = discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); + if (!runUnitTests) return true; - var skipTests = GetAttributes(testMethod) - .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); + var skipTests = GetAttributes(testMethod) + .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); - if (skipTests == null) return false; + if (skipTests == null) return false; - skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); - return true; - } - } + skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); + return true; + } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs index c18e2980e8..ad40d64784 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs @@ -32,63 +32,62 @@ using OpenSearch.Stack.ArtifactsApi.Resolvers; using Version = SemanticVersioning.Version; -namespace OpenSearch.Stack.ArtifactsApi +namespace OpenSearch.Stack.ArtifactsApi; + +public class Artifact { - public class Artifact - { - private static readonly Uri BaseUri = new Uri("http://localhost"); + private static readonly Uri BaseUri = new Uri("http://localhost"); - internal Artifact(Product product, Version version, string downloadUrl, string buildHash) - { - ProductName = product?.ProductName ?? "opensearch"; - Version = version; - DownloadUrl = downloadUrl; - BuildHash = buildHash; - } + internal Artifact(Product product, Version version, string downloadUrl, string buildHash) + { + ProductName = product?.ProductName ?? "opensearch"; + Version = version; + DownloadUrl = downloadUrl; + BuildHash = buildHash; + } - internal Artifact(Product product, Version version, SearchPackage package, string buildHash = null) - { - ProductName = product.ProductName; - Version = version; - DownloadUrl = package.DownloadUrl; - ShaUrl = package.ShaUrl; - AscUrl = package.AscUrl; - BuildHash = buildHash; - } + internal Artifact(Product product, Version version, SearchPackage package, string buildHash = null) + { + ProductName = product.ProductName; + Version = version; + DownloadUrl = package.DownloadUrl; + ShaUrl = package.ShaUrl; + AscUrl = package.AscUrl; + BuildHash = buildHash; + } - public string LocalFolderName - { - get - { - var hashed = !Version.IsPreRelease || string.IsNullOrWhiteSpace(BuildHash) - ? string.Empty - : $"-build-{BuildHash}"; - return $"{ProductName}-{Version}{hashed}"; - } - } + public string LocalFolderName + { + get + { + var hashed = !Version.IsPreRelease || string.IsNullOrWhiteSpace(BuildHash) + ? string.Empty + : $"-build-{BuildHash}"; + return $"{ProductName}-{Version}{hashed}"; + } + } - public string FolderInZip => $"{ProductName}-{Version}"; + public string FolderInZip => $"{ProductName}-{Version}"; - public string Archive - { - get - { - if (!Uri.TryCreate(DownloadUrl, UriKind.Absolute, out var uri)) - uri = new Uri(BaseUri, DownloadUrl); + public string Archive + { + get + { + if (!Uri.TryCreate(DownloadUrl, UriKind.Absolute, out var uri)) + uri = new Uri(BaseUri, DownloadUrl); - return Path.GetFileName(uri.LocalPath); - } - } + return Path.GetFileName(uri.LocalPath); + } + } - // ReSharper disable UnusedAutoPropertyAccessor.Global - public string ProductName { get; } - public Version Version { get; } - public string DownloadUrl { get; } + // ReSharper disable UnusedAutoPropertyAccessor.Global + public string ProductName { get; } + public Version Version { get; } + public string DownloadUrl { get; } - public string BuildHash { get; } - public string ShaUrl { get; } + public string BuildHash { get; } + public string ShaUrl { get; } - public string AscUrl { get; } - // ReSharper restore UnusedAutoPropertyAccessor.Global - } + public string AscUrl { get; } + // ReSharper restore UnusedAutoPropertyAccessor.Global } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs index c54251faaa..53be7cb204 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs @@ -35,97 +35,96 @@ using SemanticVersioning; using Version = SemanticVersioning.Version; -namespace OpenSearch.Stack.ArtifactsApi +namespace OpenSearch.Stack.ArtifactsApi; + +public class OpenSearchVersion : Version, IComparable { - public class OpenSearchVersion : Version, IComparable - { - private readonly ConcurrentDictionary _resolved = new(); + private readonly ConcurrentDictionary _resolved = new(); - protected OpenSearchVersion(string version, string buildHash = null) : base(version) => BuildHash = buildHash; + protected OpenSearchVersion(string version, string buildHash = null) : base(version) => BuildHash = buildHash; - private string BuildHash { get; } + private string BuildHash { get; } - public bool IsSnapshot => PreRelease == "SNAPSHOT"; + public bool IsSnapshot => PreRelease == "SNAPSHOT"; - public int CompareTo(string other) - { - var v = (OpenSearchVersion)other; - return CompareTo(v); - } + public int CompareTo(string other) + { + var v = (OpenSearchVersion)other; + return CompareTo(v); + } - public Artifact Artifact(Product product) - { - var cacheKey = product.ToString(); - if (_resolved.TryGetValue(cacheKey, out var artifact)) - return artifact; + public Artifact Artifact(Product product) + { + var cacheKey = product.ToString(); + if (_resolved.TryGetValue(cacheKey, out var artifact)) + return artifact; - var currentPlatform = OsMonikers.CurrentPlatform(); + var currentPlatform = OsMonikers.CurrentPlatform(); - ReleasedVersionResolver.TryResolve(product, this, currentPlatform, RuntimeInformation.OSArchitecture, out artifact); + ReleasedVersionResolver.TryResolve(product, this, currentPlatform, RuntimeInformation.OSArchitecture, out artifact); - _resolved.TryAdd(cacheKey, artifact); + _resolved.TryAdd(cacheKey, artifact); - return artifact; - } + return artifact; + } - /// - /// Resolves an OpenSearch version using either format '$version' or '$ServerType-$version', where version could be 'x.y.z' or 'latest' or even 'latest-x' - /// - public static OpenSearchVersion From(string managedVersionString) => - // TODO resolve `latest` and `latest-x` for OpenSearch - managedVersionString == null ? null : new OpenSearchVersion(managedVersionString, ""); + /// + /// Resolves an OpenSearch version using either format '$version' or '$ServerType-$version', where version could be 'x.y.z' or 'latest' or even 'latest-x' + /// + public static OpenSearchVersion From(string managedVersionString) => + // TODO resolve `latest` and `latest-x` for OpenSearch + managedVersionString == null ? null : new OpenSearchVersion(managedVersionString, ""); - public bool InRange(string range) - { - var versionRange = new Range(range); - return InRange(versionRange); - } + public bool InRange(string range) + { + var versionRange = new Range(range); + return InRange(versionRange); + } - public bool InRange(Range versionRange) - { - var satisfied = versionRange.IsSatisfied(this); - if (satisfied) - return true; + public bool InRange(Range versionRange) + { + var satisfied = versionRange.IsSatisfied(this); + if (satisfied) + return true; - //Semver can only match snapshot version with ranges on the same major and minor - //anything else fails but we want to know e.g 1.0.0-SNAPSHOT satisfied by 1.0.0; - var wholeVersion = $"{Major}.{Minor}.{Patch}"; - return versionRange.IsSatisfied(wholeVersion); - } + //Semver can only match snapshot version with ranges on the same major and minor + //anything else fails but we want to know e.g 1.0.0-SNAPSHOT satisfied by 1.0.0; + var wholeVersion = $"{Major}.{Minor}.{Patch}"; + return versionRange.IsSatisfied(wholeVersion); + } - public static implicit operator OpenSearchVersion(string version) => From(version); + public static implicit operator OpenSearchVersion(string version) => From(version); - public static bool operator <(OpenSearchVersion first, string second) => first < (OpenSearchVersion)second; + public static bool operator <(OpenSearchVersion first, string second) => first < (OpenSearchVersion)second; - public static bool operator >(OpenSearchVersion first, string second) => first > (OpenSearchVersion)second; + public static bool operator >(OpenSearchVersion first, string second) => first > (OpenSearchVersion)second; - public static bool operator <(string first, OpenSearchVersion second) => (OpenSearchVersion)first < second; + public static bool operator <(string first, OpenSearchVersion second) => (OpenSearchVersion)first < second; - public static bool operator >(string first, OpenSearchVersion second) => (OpenSearchVersion)first > second; + public static bool operator >(string first, OpenSearchVersion second) => (OpenSearchVersion)first > second; - public static bool operator <=(OpenSearchVersion first, string second) => first <= (OpenSearchVersion)second; + public static bool operator <=(OpenSearchVersion first, string second) => first <= (OpenSearchVersion)second; - public static bool operator >=(OpenSearchVersion first, string second) => first >= (OpenSearchVersion)second; + public static bool operator >=(OpenSearchVersion first, string second) => first >= (OpenSearchVersion)second; - public static bool operator <=(string first, OpenSearchVersion second) => (OpenSearchVersion)first <= second; + public static bool operator <=(string first, OpenSearchVersion second) => (OpenSearchVersion)first <= second; - public static bool operator >=(string first, OpenSearchVersion second) => (OpenSearchVersion)first >= second; + public static bool operator >=(string first, OpenSearchVersion second) => (OpenSearchVersion)first >= second; - public static bool operator ==(OpenSearchVersion first, string second) => first == (OpenSearchVersion)second; + public static bool operator ==(OpenSearchVersion first, string second) => first == (OpenSearchVersion)second; - public static bool operator !=(OpenSearchVersion first, string second) => first != (OpenSearchVersion)second; + public static bool operator !=(OpenSearchVersion first, string second) => first != (OpenSearchVersion)second; - public static bool operator ==(string first, OpenSearchVersion second) => (OpenSearchVersion)first == second; + public static bool operator ==(string first, OpenSearchVersion second) => (OpenSearchVersion)first == second; - public static bool operator !=(string first, OpenSearchVersion second) => (OpenSearchVersion)first != second; + public static bool operator !=(string first, OpenSearchVersion second) => (OpenSearchVersion)first != second; - // ReSharper disable once UnusedMember.Local - private bool Equals(OpenSearchVersion other) => base.Equals(other); + // ReSharper disable once UnusedMember.Local + private bool Equals(OpenSearchVersion other) => base.Equals(other); - public override bool Equals(object obj) => base.Equals(obj); + public override bool Equals(object obj) => base.Equals(obj); - public override int GetHashCode() => base.GetHashCode(); - } + public override int GetHashCode() => base.GetHashCode(); } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs index 2ecdc77ebc..db36a15b0f 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs @@ -29,93 +29,92 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -namespace OpenSearch.Stack.ArtifactsApi.Platform +namespace OpenSearch.Stack.ArtifactsApi.Platform; + +internal static class OsMonikers { - internal static class OsMonikers - { - public static readonly string Windows = "windows"; - public static readonly string Linux = "linux"; - public static readonly string OSX = "darwin"; + public static readonly string Windows = "windows"; + public static readonly string Linux = "linux"; + public static readonly string OSX = "darwin"; - public static string From(OSPlatform platform) - { - if (platform == OSPlatform.Windows) return Windows; - if (platform == OSPlatform.Linux) return Linux; - if (platform == OSPlatform.OSX) return OSX; - return "unknown"; - } + public static string From(OSPlatform platform) + { + if (platform == OSPlatform.Windows) return Windows; + if (platform == OSPlatform.Linux) return Linux; + if (platform == OSPlatform.OSX) return OSX; + return "unknown"; + } - public static OSPlatform CurrentPlatform() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return OSPlatform.Windows; - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return OSPlatform.OSX; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return OSPlatform.Linux; - throw new Exception( - $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); - } + public static OSPlatform CurrentPlatform() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return OSPlatform.Windows; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return OSPlatform.OSX; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return OSPlatform.Linux; + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); + } - /// - /// Maps from current os and architecture to a suffix used for tagging platform-specific OpenSearch releases. - /// - /// - public static string GetOpenSearchPlatformMoniker(OSPlatform platform, Architecture architecture) - { - if (platform == OSPlatform.Windows) - { - if (architecture == Architecture.X64) return "windows-x64"; - if (architecture == Architecture.X86) return "windows-x86"; - } + /// + /// Maps from current os and architecture to a suffix used for tagging platform-specific OpenSearch releases. + /// + /// + public static string GetOpenSearchPlatformMoniker(OSPlatform platform, Architecture architecture) + { + if (platform == OSPlatform.Windows) + { + if (architecture == Architecture.X64) return "windows-x64"; + if (architecture == Architecture.X86) return "windows-x86"; + } - if (platform == OSPlatform.OSX) - { - if (architecture == Architecture.X64) return "macos-x64"; - if (architecture == Architecture.Arm64) return "macos-arm64"; + if (platform == OSPlatform.OSX) + { + if (architecture == Architecture.X64) return "macos-x64"; + if (architecture == Architecture.Arm64) return "macos-arm64"; - } + } - if (platform == OSPlatform.Linux) - { - if (architecture == Architecture.X64) return "linux-x64"; - if (architecture == Architecture.Arm64) return "linux-arm64"; - } - throw new Exception( - $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); - } + if (platform == OSPlatform.Linux) + { + if (architecture == Architecture.X64) return "linux-x64"; + if (architecture == Architecture.Arm64) return "linux-arm64"; + } + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); + } - public static string GetPlatformArchiveExtension(OSPlatform platform) - { - if (platform == OSPlatform.Linux) return "tar.gz"; - if (platform == OSPlatform.OSX) return "tar.gz"; - if (platform == OSPlatform.Windows) return "zip"; - throw new Exception( - $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); + public static string GetPlatformArchiveExtension(OSPlatform platform) + { + if (platform == OSPlatform.Linux) return "tar.gz"; + if (platform == OSPlatform.OSX) return "tar.gz"; + if (platform == OSPlatform.Windows) return "zip"; + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); - } - public static string CurrentPlatformArchiveExtension() - { - var platform = CurrentPlatform(); - return GetPlatformArchiveExtension(platform); - } + } + public static string CurrentPlatformArchiveExtension() + { + var platform = CurrentPlatform(); + return GetPlatformArchiveExtension(platform); + } - public static string CurrentPlatformPackageSuffix() - { - var intelX86Suffix = "x86_64"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return $"{Windows}-{intelX86Suffix}"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return $"{OSX}-{intelX86Suffix}"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return $"{Linux}-{intelX86Suffix}"; + public static string CurrentPlatformPackageSuffix() + { + var intelX86Suffix = "x86_64"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return $"{Windows}-{intelX86Suffix}"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return $"{OSX}-{intelX86Suffix}"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return $"{Linux}-{intelX86Suffix}"; - throw new Exception( - $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); - } + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); + } - internal static string CurrentPlatformSearchFilter() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "zip"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "tar"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "tar"; + internal static string CurrentPlatformSearchFilter() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "zip"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "tar"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "tar"; - throw new Exception( - $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); - } - } + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net"); + } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs index 536cebb3ee..1a44cc5f62 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs @@ -28,58 +28,57 @@ using System; -namespace OpenSearch.Stack.ArtifactsApi.Products +namespace OpenSearch.Stack.ArtifactsApi.Products; + +/// An OpenSearch plugin +public class OpenSearchPlugin : SubProduct { - /// An OpenSearch plugin - public class OpenSearchPlugin : SubProduct - { - public OpenSearchPlugin(string plugin, Func isValid = null, - Func listName = null) - : base(plugin, isValid, listName) => - PlatformDependent = false; // ReSharper disable InconsistentNaming - public static OpenSearchPlugin AnalysisIcu { get; } = new("analysis-icu"); - public static OpenSearchPlugin AnalysisKuromoji { get; } = new("analysis-kuromoji"); - public static OpenSearchPlugin AnalysisNori { get; } = new("analysis-nori"); - public static OpenSearchPlugin AnalysisPhonetic { get; } = new("analysis-phonetic"); - public static OpenSearchPlugin AnalysisSmartCn { get; } = new("analysis-smartcn"); - public static OpenSearchPlugin AnalysisStempel { get; } = new("analysis-stempel"); - public static OpenSearchPlugin AnalysisUkrainian { get; } = new("analysis-ukrainian"); + public OpenSearchPlugin(string plugin, Func isValid = null, + Func listName = null) + : base(plugin, isValid, listName) => + PlatformDependent = false; // ReSharper disable InconsistentNaming + public static OpenSearchPlugin AnalysisIcu { get; } = new("analysis-icu"); + public static OpenSearchPlugin AnalysisKuromoji { get; } = new("analysis-kuromoji"); + public static OpenSearchPlugin AnalysisNori { get; } = new("analysis-nori"); + public static OpenSearchPlugin AnalysisPhonetic { get; } = new("analysis-phonetic"); + public static OpenSearchPlugin AnalysisSmartCn { get; } = new("analysis-smartcn"); + public static OpenSearchPlugin AnalysisStempel { get; } = new("analysis-stempel"); + public static OpenSearchPlugin AnalysisUkrainian { get; } = new("analysis-ukrainian"); - public static OpenSearchPlugin DiscoveryAzureClassic { get; } = new("discovery-azure-classic"); + public static OpenSearchPlugin DiscoveryAzureClassic { get; } = new("discovery-azure-classic"); - public static OpenSearchPlugin DiscoveryEC2 { get; } = new("discovery-ec2"); - public static OpenSearchPlugin DiscoveryFile { get; } = new("discovery-file"); - public static OpenSearchPlugin DiscoveryGCE { get; } = new("discovery-gce"); + public static OpenSearchPlugin DiscoveryEC2 { get; } = new("discovery-ec2"); + public static OpenSearchPlugin DiscoveryFile { get; } = new("discovery-file"); + public static OpenSearchPlugin DiscoveryGCE { get; } = new("discovery-gce"); - public static OpenSearchPlugin IngestAttachment { get; } = new("ingest-attachment", version => version >= "1.0.0"); + public static OpenSearchPlugin IngestAttachment { get; } = new("ingest-attachment", version => version >= "1.0.0"); - public static OpenSearchPlugin IngestGeoIp { get; } = - new("ingest-geoip", version => version >= "1.0.0") - { - ShippedByDefaultAsOf = "1.0.0" - }; + public static OpenSearchPlugin IngestGeoIp { get; } = + new("ingest-geoip", version => version >= "1.0.0") + { + ShippedByDefaultAsOf = "1.0.0" + }; - public static OpenSearchPlugin IngestUserAgent { get; } = - new("ingest-user-agent", version => version >= "1.0.0") - { - ShippedByDefaultAsOf = "1.0.0" - }; + public static OpenSearchPlugin IngestUserAgent { get; } = + new("ingest-user-agent", version => version >= "1.0.0") + { + ShippedByDefaultAsOf = "1.0.0" + }; - public static OpenSearchPlugin MapperAttachment { get; } = new("mapper-attachments"); - public static OpenSearchPlugin MapperMurmur3 { get; } = new("mapper-murmur3"); - public static OpenSearchPlugin MapperSize { get; } = new("mapper-size"); + public static OpenSearchPlugin MapperAttachment { get; } = new("mapper-attachments"); + public static OpenSearchPlugin MapperMurmur3 { get; } = new("mapper-murmur3"); + public static OpenSearchPlugin MapperSize { get; } = new("mapper-size"); - public static OpenSearchPlugin RepositoryAzure { get; } = new("repository-azure"); - public static OpenSearchPlugin RepositoryGCS { get; } = new("repository-gcs"); - public static OpenSearchPlugin RepositoryHDFS { get; } = new("repository-hdfs"); - public static OpenSearchPlugin RepositoryS3 { get; } = new("repository-s3"); + public static OpenSearchPlugin RepositoryAzure { get; } = new("repository-azure"); + public static OpenSearchPlugin RepositoryGCS { get; } = new("repository-gcs"); + public static OpenSearchPlugin RepositoryHDFS { get; } = new("repository-hdfs"); + public static OpenSearchPlugin RepositoryS3 { get; } = new("repository-s3"); - public static OpenSearchPlugin Security { get; } = new("opensearch-security"); + public static OpenSearchPlugin Security { get; } = new("opensearch-security"); - public static OpenSearchPlugin StoreSMB { get; } = new("store-smb"); + public static OpenSearchPlugin StoreSMB { get; } = new("store-smb"); - public static OpenSearchPlugin DeleteByQuery { get; } = new("delete-by-query", version => version < "1.0.0"); + public static OpenSearchPlugin DeleteByQuery { get; } = new("delete-by-query", version => version < "1.0.0"); - public static OpenSearchPlugin Knn { get; } = new("opensearch-knn"); - } + public static OpenSearchPlugin Knn { get; } = new("opensearch-knn"); } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs index 0ab56e4bde..aa11291d12 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs @@ -29,46 +29,45 @@ using System.Collections.Concurrent; using OpenSearch.Stack.ArtifactsApi.Platform; -namespace OpenSearch.Stack.ArtifactsApi.Products +namespace OpenSearch.Stack.ArtifactsApi.Products; + +public class Product { - public class Product - { - private static readonly ConcurrentDictionary CachedProducts = - new ConcurrentDictionary(); + private static readonly ConcurrentDictionary CachedProducts = + new ConcurrentDictionary(); - private static readonly OpenSearchVersion DefaultIncludePlatformSuffix = OpenSearchVersion.From("1.0.0"); + private static readonly OpenSearchVersion DefaultIncludePlatformSuffix = OpenSearchVersion.From("1.0.0"); - private Product(string productName) => ProductName = productName; + private Product(string productName) => ProductName = productName; - protected Product(string productName, SubProduct relatedProduct, OpenSearchVersion platformVersionSuffixAfter = null) : this(productName) - { - SubProduct = relatedProduct; - PlatformSuffixAfter = platformVersionSuffixAfter ?? DefaultIncludePlatformSuffix; - } + protected Product(string productName, SubProduct relatedProduct, OpenSearchVersion platformVersionSuffixAfter = null) : this(productName) + { + SubProduct = relatedProduct; + PlatformSuffixAfter = platformVersionSuffixAfter ?? DefaultIncludePlatformSuffix; + } - public SubProduct SubProduct { get; } + public SubProduct SubProduct { get; } - public string Moniker => SubProduct?.SubProductName ?? ProductName; + public string Moniker => SubProduct?.SubProductName ?? ProductName; - public string Extension => PlatformDependent ? OsMonikers.CurrentPlatformArchiveExtension() : "zip"; + public string Extension => PlatformDependent ? OsMonikers.CurrentPlatformArchiveExtension() : "zip"; - public string ProductName { get; } + public string ProductName { get; } - public bool PlatformDependent => SubProduct?.PlatformDependent ?? true; + public bool PlatformDependent => SubProduct?.PlatformDependent ?? true; - public OpenSearchVersion PlatformSuffixAfter { get; } + public OpenSearchVersion PlatformSuffixAfter { get; } - public static Product OpenSearch { get; } = From("opensearch"); + public static Product OpenSearch { get; } = From("opensearch"); - public static Product OpenSearchDashboards { get; } = From("opensearch-dashboards", platformInZipAfter: "1.0.0"); + public static Product OpenSearchDashboards { get; } = From("opensearch-dashboards", platformInZipAfter: "1.0.0"); - public static Product From(string product, SubProduct subProduct = null, OpenSearchVersion platformInZipAfter = null) => - CachedProducts.GetOrAdd(subProduct == null ? $"{product}" : $"{product}/{subProduct.SubProductName}", - k => new Product(product, subProduct, platformInZipAfter)); + public static Product From(string product, SubProduct subProduct = null, OpenSearchVersion platformInZipAfter = null) => + CachedProducts.GetOrAdd(subProduct == null ? $"{product}" : $"{product}/{subProduct.SubProductName}", + k => new Product(product, subProduct, platformInZipAfter)); - public static Product OpenSearchPlugin(OpenSearchPlugin plugin) => From("opensearch-plugins", plugin); + public static Product OpenSearchPlugin(OpenSearchPlugin plugin) => From("opensearch-plugins", plugin); - public override string ToString() => - SubProduct != null ? $"{ProductName}/{SubProduct.SubProductName}" : ProductName; - } + public override string ToString() => + SubProduct != null ? $"{ProductName}/{SubProduct.SubProductName}" : ProductName; } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs index 0f8d713731..cae127b1f3 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs @@ -28,38 +28,37 @@ using System; -namespace OpenSearch.Stack.ArtifactsApi.Products +namespace OpenSearch.Stack.ArtifactsApi.Products; + +public class SubProduct { - public class SubProduct - { - private readonly Func _getExistsMoniker; + private readonly Func _getExistsMoniker; - private readonly Func _isValid; + private readonly Func _isValid; - public SubProduct(string subProject, Func isValid = null, - Func listName = null) - { - SubProductName = subProject; - _isValid = isValid ?? (v => true); - _getExistsMoniker = listName ?? (v => subProject); - } + public SubProduct(string subProject, Func isValid = null, + Func listName = null) + { + SubProductName = subProject; + _isValid = isValid ?? (v => true); + _getExistsMoniker = listName ?? (v => subProject); + } - public string SubProductName { get; } + public string SubProductName { get; } - public OpenSearchVersion ShippedByDefaultAsOf { get; set; } + public OpenSearchVersion ShippedByDefaultAsOf { get; set; } - public bool PlatformDependent { get; protected set; } + public bool PlatformDependent { get; protected set; } - /// what moniker to use when asserting the sub product is already present - public string GetExistsMoniker(OpenSearchVersion version) => _getExistsMoniker(version); + /// what moniker to use when asserting the sub product is already present + public string GetExistsMoniker(OpenSearchVersion version) => _getExistsMoniker(version); - /// Whether the sub project is included in the distribution out of the box for the given version - public bool IsIncludedOutOfTheBox(OpenSearchVersion version) => - ShippedByDefaultAsOf != null && version >= ShippedByDefaultAsOf; + /// Whether the sub project is included in the distribution out of the box for the given version + public bool IsIncludedOutOfTheBox(OpenSearchVersion version) => + ShippedByDefaultAsOf != null && version >= ShippedByDefaultAsOf; - /// Whether the subProject is valid for the given version - public bool IsValid(OpenSearchVersion version) => IsIncludedOutOfTheBox(version) || _isValid(version); + /// Whether the subProject is valid for the given version + public bool IsValid(OpenSearchVersion version) => IsIncludedOutOfTheBox(version) || _isValid(version); - public override string ToString() => SubProductName; - } + public override string ToString() => SubProductName; } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs index e48793fd0b..ef189834c0 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs @@ -37,91 +37,90 @@ using System.Text.Json.Serialization; using System.Text.RegularExpressions; -namespace OpenSearch.Stack.ArtifactsApi.Resolvers +namespace OpenSearch.Stack.ArtifactsApi.Resolvers; + +public static class ApiResolver { - public static class ApiResolver - { - // TODO: update string when working on artifacts API - private const string ArtifactsApiUrl = "https://artifacts-api.opensearch.org/v1/"; + // TODO: update string when working on artifacts API + private const string ArtifactsApiUrl = "https://artifacts-api.opensearch.org/v1/"; - private static readonly ConcurrentDictionary Releases = new ConcurrentDictionary(); + private static readonly ConcurrentDictionary Releases = new ConcurrentDictionary(); - private static HttpClient HttpClient { get; } = - new HttpClient(new HttpClientHandler {SslProtocols = SslProtocols.Tls12}) - { - BaseAddress = new Uri(ArtifactsApiUrl) - }; + private static HttpClient HttpClient { get; } = + new HttpClient(new HttpClientHandler { SslProtocols = SslProtocols.Tls12 }) + { + BaseAddress = new Uri(ArtifactsApiUrl) + }; - private static Regex BuildHashRegex { get; } = - // TODO: update string when working on artifacts API - new Regex(@"https://(?:snapshots|staging).opensearch.org/(\d+\.\d+\.\d+-([^/]+)?)"); + private static Regex BuildHashRegex { get; } = + // TODO: update string when working on artifacts API + new Regex(@"https://(?:snapshots|staging).opensearch.org/(\d+\.\d+\.\d+-([^/]+)?)"); - public static string FetchJson(string path) - { - using (var stream = HttpClient.GetStreamAsync(path).GetAwaiter().GetResult()) - using (var fileStream = new StreamReader(stream)) - return fileStream.ReadToEnd(); - } + public static string FetchJson(string path) + { + using (var stream = HttpClient.GetStreamAsync(path).GetAwaiter().GetResult()) + using (var fileStream = new StreamReader(stream)) + return fileStream.ReadToEnd(); + } - public static bool IsReleasedVersion(string version) - { - if (Releases.TryGetValue(version, out var released)) return released; - var versionPath = "https://github.com/opensearch-project/opensearch/releases/tag/" + version; - var message = new HttpRequestMessage {Method = HttpMethod.Head, RequestUri = new Uri(versionPath)}; + public static bool IsReleasedVersion(string version) + { + if (Releases.TryGetValue(version, out var released)) return released; + var versionPath = "https://github.com/opensearch-project/opensearch/releases/tag/" + version; + var message = new HttpRequestMessage { Method = HttpMethod.Head, RequestUri = new Uri(versionPath) }; - using (var response = HttpClient.SendAsync(message).GetAwaiter().GetResult()) - { - released = response.IsSuccessStatusCode; - Releases.TryAdd(version, released); - return released; - } - } + using (var response = HttpClient.SendAsync(message).GetAwaiter().GetResult()) + { + released = response.IsSuccessStatusCode; + Releases.TryAdd(version, released); + return released; + } + } - public static string LatestBuildHash(string version) - { - var json = FetchJson($"search/{version}/msi"); - try - { - // if packages is empty it turns into an array[] otherwise its a dictionary :/ - var packages = JsonSerializer.Deserialize(json).Packages; - if (packages.Count == 0) - throw new Exception("Can not get build hash for: " + version); - return GetBuildHash(packages.First().Value.DownloadUrl); - } - catch - { - throw new Exception("Can not get build hash for: " + version); - } - } + public static string LatestBuildHash(string version) + { + var json = FetchJson($"search/{version}/msi"); + try + { + // if packages is empty it turns into an array[] otherwise its a dictionary :/ + var packages = JsonSerializer.Deserialize(json).Packages; + if (packages.Count == 0) + throw new Exception("Can not get build hash for: " + version); + return GetBuildHash(packages.First().Value.DownloadUrl); + } + catch + { + throw new Exception("Can not get build hash for: " + version); + } + } - public static string GetBuildHash(string url) - { - var tokens = BuildHashRegex.Split(url).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); - if (tokens.Length < 2) - throw new Exception("Can not parse build hash from: " + url); + public static string GetBuildHash(string url) + { + var tokens = BuildHashRegex.Split(url).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); + if (tokens.Length < 2) + throw new Exception("Can not parse build hash from: " + url); - return tokens[1]; - } - } + return tokens[1]; + } +} - internal class ArtifactsVersionsResponse - { - [JsonPropertyName("versions")] public List Versions { get; set; } - } +internal class ArtifactsVersionsResponse +{ + [JsonPropertyName("versions")] public List Versions { get; set; } +} - internal class ArtifactsSearchResponse - { - [JsonPropertyName("packages")] public Dictionary Packages { get; set; } - } +internal class ArtifactsSearchResponse +{ + [JsonPropertyName("packages")] public Dictionary Packages { get; set; } +} - internal class SearchPackage - { - [JsonPropertyName("url")] public string DownloadUrl { get; set; } - [JsonPropertyName("sha_url")] public string ShaUrl { get; set; } - [JsonPropertyName("asc_url")] public string AscUrl { get; set; } - [JsonPropertyName("type")] public string Type { get; set; } - [JsonPropertyName("architecture")] public string Architecture { get; set; } - [JsonPropertyName("os")] public string[] OperatingSystem { get; set; } - [JsonPropertyName("classifier")] public string Classifier { get; set; } - } +internal class SearchPackage +{ + [JsonPropertyName("url")] public string DownloadUrl { get; set; } + [JsonPropertyName("sha_url")] public string ShaUrl { get; set; } + [JsonPropertyName("asc_url")] public string AscUrl { get; set; } + [JsonPropertyName("type")] public string Type { get; set; } + [JsonPropertyName("architecture")] public string Architecture { get; set; } + [JsonPropertyName("os")] public string[] OperatingSystem { get; set; } + [JsonPropertyName("classifier")] public string Classifier { get; set; } } diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs index acb2130192..7471ba8092 100644 --- a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs @@ -31,23 +31,22 @@ using OpenSearch.Stack.ArtifactsApi.Products; using Version = SemanticVersioning.Version; -namespace OpenSearch.Stack.ArtifactsApi.Resolvers +namespace OpenSearch.Stack.ArtifactsApi.Resolvers; + +public static class ReleasedVersionResolver { - public static class ReleasedVersionResolver - { - private const string ArtifactsUrl = "https://artifacts.opensearch.org"; + private const string ArtifactsUrl = "https://artifacts.opensearch.org"; - public static bool TryResolve(Product product, Version version, OSPlatform platform, Architecture architecture, out Artifact artifact) - { - var productMoniker = product.Moniker; - var platformMoniker = OsMonikers.GetOpenSearchPlatformMoniker(platform, architecture); - var downloadPath = $"{ArtifactsUrl}/releases/bundle/{product}/{version}"; - var extension = OsMonikers.GetPlatformArchiveExtension(platform); - var archive = $"{productMoniker}-{version}-{platformMoniker}.{extension}"; - var downloadUrl = $"{downloadPath}/{archive}"; + public static bool TryResolve(Product product, Version version, OSPlatform platform, Architecture architecture, out Artifact artifact) + { + var productMoniker = product.Moniker; + var platformMoniker = OsMonikers.GetOpenSearchPlatformMoniker(platform, architecture); + var downloadPath = $"{ArtifactsUrl}/releases/bundle/{product}/{version}"; + var extension = OsMonikers.GetPlatformArchiveExtension(platform); + var archive = $"{productMoniker}-{version}-{platformMoniker}.{extension}"; + var downloadUrl = $"{downloadPath}/{archive}"; - artifact = new Artifact(product, version, downloadUrl, null); - return true; - } - } + artifact = new Artifact(product, version, downloadUrl, null); + return true; + } } diff --git a/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs b/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs index e67cf97164..8a58a0970f 100644 --- a/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs +++ b/abstractions/tests/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs @@ -33,44 +33,43 @@ using OpenSearch.Stack.ArtifactsApi; using Xunit; -namespace OpenSearch.OpenSearch.EphemeralTests +namespace OpenSearch.OpenSearch.EphemeralTests; + +public class EphemeralClusterTests { - public class EphemeralClusterTests - { - [TU] - [ClassData(typeof(SampleClusters))] - public void TestEphemeralCluster(OpenSearchVersion version, ClusterFeatures features) - { - const int numberOfNodes = 1; - var clusterConfiguration = - new EphemeralClusterConfiguration(version, features, null, numberOfNodes) - { - ShowOpenSearchOutputAfterStarted = true - }; - using var cluster = new EphemeralCluster(clusterConfiguration); - var timeout = new System.TimeSpan(0, 5, 30); - using (cluster.Start(timeout)) - { - Assert.True(cluster.Started, "OpenSearch cluster started"); - foreach (var n in cluster.Nodes) - { - n.SendControlC(); - Assert.True(n.WaitForCompletion(timeout), $"Failed to stop node {n.ProcessId}"); - } - } - } + [TU] + [ClassData(typeof(SampleClusters))] + public void TestEphemeralCluster(OpenSearchVersion version, ClusterFeatures features) + { + const int numberOfNodes = 1; + var clusterConfiguration = + new EphemeralClusterConfiguration(version, features, null, numberOfNodes) + { + ShowOpenSearchOutputAfterStarted = true + }; + using var cluster = new EphemeralCluster(clusterConfiguration); + var timeout = new System.TimeSpan(0, 5, 30); + using (cluster.Start(timeout)) + { + Assert.True(cluster.Started, "OpenSearch cluster started"); + foreach (var n in cluster.Nodes) + { + n.SendControlC(); + Assert.True(n.WaitForCompletion(timeout), $"Failed to stop node {n.ProcessId}"); + } + } + } - private class SampleClusters : IEnumerable - { - public IEnumerator GetEnumerator() - { - yield return new object[] {OpenSearchVersion.From("1.2.0"), ClusterFeatures.None}; - yield return new object[] {OpenSearchVersion.From("2.2.0"), ClusterFeatures.None}; - yield return new object[] {OpenSearchVersion.From("1.2.0"), ClusterFeatures.SSL}; - yield return new object[] {OpenSearchVersion.From("2.2.0"), ClusterFeatures.SSL}; - } + private class SampleClusters : IEnumerable + { + public IEnumerator GetEnumerator() + { + yield return new object[] { OpenSearchVersion.From("1.2.0"), ClusterFeatures.None }; + yield return new object[] { OpenSearchVersion.From("2.2.0"), ClusterFeatures.None }; + yield return new object[] { OpenSearchVersion.From("1.2.0"), ClusterFeatures.SSL }; + yield return new object[] { OpenSearchVersion.From("2.2.0"), ClusterFeatures.SSL }; + } - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } - } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } } diff --git a/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs b/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs index a490c62cf6..47e672a6a0 100644 --- a/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs +++ b/abstractions/tests/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs @@ -38,39 +38,40 @@ using Xunit; using Xunit.Abstractions; -namespace OpenSearch.Stack.ArtifactsApiTests +namespace OpenSearch.Stack.ArtifactsApiTests; + +public class ReleasedVersionResolverTests { - public class ReleasedVersionResolverTests - { - public ReleasedVersionResolverTests(ITestOutputHelper traceSink) => _traceSink = traceSink ?? throw new NullReferenceException(nameof(traceSink)); + public ReleasedVersionResolverTests(ITestOutputHelper traceSink) => _traceSink = traceSink ?? throw new NullReferenceException(nameof(traceSink)); - private readonly ITestOutputHelper _traceSink; + private readonly ITestOutputHelper _traceSink; - [U] - public void Does_Resolver_Construct_Valid_DownloadUrl_Test() + [U] + public void Does_Resolver_Construct_Valid_DownloadUrl_Test() + { + var httpClient = new HttpClient { - var httpClient = new HttpClient(); - httpClient.Timeout = TimeSpan.FromSeconds(3); + Timeout = TimeSpan.FromSeconds(3) + }; - var testCases = new[] - { - new {Product = Product.OpenSearch, Version = "1.2.3", Platform = OSPlatform.Linux, Architecture = Architecture.X64}, - new {Product = Product.OpenSearch, Version = "1.2.4", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64}, - new {Product = Product.OpenSearch, Version = "1.0.0", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64} - }; - foreach (var testCase in testCases) - { - var version = OpenSearchVersion.From(testCase.Version); - var resolveSucceeded = ReleasedVersionResolver.TryResolve(testCase.Product, version, testCase.Platform, - testCase.Architecture, out var artifact); - Assert.True(resolveSucceeded); - _traceSink.WriteLine($"Checking URL {artifact.DownloadUrl}"); + var testCases = new[] + { + new {Product = Product.OpenSearch, Version = "1.2.3", Platform = OSPlatform.Linux, Architecture = Architecture.X64}, + new {Product = Product.OpenSearch, Version = "1.2.4", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64}, + new {Product = Product.OpenSearch, Version = "1.0.0", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64} + }; + foreach (var testCase in testCases) + { + var version = OpenSearchVersion.From(testCase.Version); + var resolveSucceeded = ReleasedVersionResolver.TryResolve(testCase.Product, version, testCase.Platform, + testCase.Architecture, out var artifact); + Assert.True(resolveSucceeded); + _traceSink.WriteLine($"Checking URL {artifact.DownloadUrl}"); - using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(artifact.DownloadUrl)); - using var response = httpClient.Send(request); + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(artifact.DownloadUrl)); + using var response = httpClient.Send(request); - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - } + Assert.Equal(HttpStatusCode.OK, response.StatusCode); } } } diff --git a/samples/Samples/IndexTemplate/IndexTemplateSample.cs b/samples/Samples/IndexTemplate/IndexTemplateSample.cs index eb89ed1344..9b4d5da7ce 100644 --- a/samples/Samples/IndexTemplate/IndexTemplateSample.cs +++ b/samples/Samples/IndexTemplate/IndexTemplateSample.cs @@ -12,161 +12,161 @@ namespace Samples.IndexTemplate; public class IndexTemplateSample : Sample { - public IndexTemplateSample() - : base("index-template", "A sample demonstrating how to use the client to create and manage index templates") { } - - protected override async Task Run(IOpenSearchClient client) - { - // Create index template - - var putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d - .IndexPatterns("books-*") - .Priority(0) - .Template(t => t - .Settings(s => s - .NumberOfShards(3) - .NumberOfReplicas(0)) - .Map(m => m - .Properties(p => p - .Text(f => f.Name(b => b.Title)) - .Text(f => f.Name(b => b.Author)) - .Date(f => f.Name(b => b.PublishedOn)) - .Number(f => f.Name(b => b.Pages).Type(NumberType.Integer)) - )))); - Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); - Console.WriteLine($"Put Template: {putTemplate.IsValid}"); - - // Confirm mapping - - var createIndex = await client.Indices.CreateAsync("books-nonfiction"); - Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); - Console.WriteLine($"Create Index: {createIndex.IsValid}"); - - var getIndex = await client.Indices.GetAsync("books-nonfiction"); - Debug.Assert( - getIndex.Indices["books-nonfiction"].Mappings.Properties["pages"].Type == "integer", - "`pages` property should have `integer` type"); - Console.WriteLine($"`pages` property type: {getIndex.Indices["books-nonfiction"].Mappings.Properties["pages"].Type}"); - - // Multiple index templates - - putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d - .IndexPatterns("books-*") - .Priority(0) - .Template(t => t - .Settings(s => s - .NumberOfShards(3) - .NumberOfReplicas(0)))); - Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); - Console.WriteLine($"Put Template: {putTemplate.IsValid}"); - - putTemplate = await client.Indices.PutComposableTemplateAsync("books-fiction", d => d - .IndexPatterns("books-fiction-*") - .Priority(1) // higher priority than the `books` template - .Template(t => t - .Settings(s => s - .NumberOfShards(1) - .NumberOfReplicas(1)))); - Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); - Console.WriteLine($"Put Template: {putTemplate.IsValid}"); - - // Validate settings - - createIndex = await client.Indices.CreateAsync("books-fiction-romance"); - Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); - Console.WriteLine($"Create Index: {createIndex.IsValid}"); - - getIndex = await client.Indices.GetAsync("books-fiction-romance"); - Debug.Assert( - getIndex.Indices["books-fiction-romance"].Settings.NumberOfShards == 1, - "`books-fiction-romance` index should have 1 shard"); - Console.WriteLine($"Number of shards: {getIndex.Indices["books-fiction-romance"].Settings.NumberOfShards}"); - - // Component templates - - var putComponentTemplate = await client.Cluster.PutComponentTemplateAsync("books_mappings", d => d - .Template(t => t - .Map(m => m - .Properties(p => p - .Text(f => f.Name(b => b.Title)) - .Text(f => f.Name(b => b.Author)) - .Date(f => f.Name(b => b.PublishedOn)) - .Number(f => f.Name(b => b.Pages).Type(NumberType.Integer)) - )))); - Debug.Assert(putComponentTemplate.IsValid, putComponentTemplate.DebugInformation); - Console.WriteLine($"Put Component Template: {putComponentTemplate.IsValid}"); - - putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d - .IndexPatterns("books-*") - .Priority(0) - .ComposedOf("books_mappings") - .Template(t => t - .Settings(s => s - .NumberOfShards(3) - .NumberOfReplicas(0)))); - Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); - Console.WriteLine($"Put Template: {putTemplate.IsValid}"); - - putTemplate = await client.Indices.PutComposableTemplateAsync("books-fiction", d => d - .IndexPatterns("books-fiction-*") - .Priority(1) // higher priority than the `books` template - .ComposedOf("books_mappings") - .Template(t => t - .Settings(s => s - .NumberOfShards(1) - .NumberOfReplicas(1)))); - Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); - Console.WriteLine($"Put Template: {putTemplate.IsValid}"); - - // Validate settings & mappings - createIndex = await client.Indices.CreateAsync("books-fiction-horror"); - Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); - Console.WriteLine($"Create Index: {createIndex.IsValid}"); - - getIndex = await client.Indices.GetAsync("books-fiction-horror"); - Debug.Assert( - getIndex.Indices["books-fiction-horror"].Settings.NumberOfShards == 1, - "`books-fiction-horror` index should have 1 shard"); - Debug.Assert( - getIndex.Indices["books-fiction-horror"].Mappings.Properties["pages"].Type == "integer", - "`pages` property should have `integer` type"); - Console.WriteLine($"Number of shards: {getIndex.Indices["books-fiction-horror"].Settings.NumberOfShards}"); - Console.WriteLine($"`pages` property type: {getIndex.Indices["books-fiction-horror"].Mappings.Properties["pages"].Type}"); - - // Get index template - - var getTemplate = await client.Indices.GetComposableTemplateAsync("books"); - Debug.Assert( - getTemplate.IndexTemplates.First().IndexTemplate.IndexPatterns.First() == "books-*", - "First index pattern should be `books-*`"); - Console.WriteLine($"First index pattern: {getTemplate.IndexTemplates.First().IndexTemplate.IndexPatterns.First()}"); - - // Delete index template - - var deleteTemplate = await client.Indices.DeleteComposableTemplateAsync("books"); - Debug.Assert(deleteTemplate.IsValid, deleteTemplate.DebugInformation); - Console.WriteLine($"Delete Template: {deleteTemplate.IsValid}"); - - // Cleanup - - var deleteIndex = await client.Indices.DeleteAsync("books-*"); - Debug.Assert(deleteIndex.IsValid, deleteIndex.DebugInformation); - Console.WriteLine($"Delete Index: {deleteIndex.IsValid}"); - - deleteTemplate = await client.Indices.DeleteComposableTemplateAsync("books-fiction"); - Debug.Assert(deleteTemplate.IsValid, deleteTemplate.DebugInformation); - Console.WriteLine($"Delete Template: {deleteTemplate.IsValid}"); - - var deleteComponentTemplate = await client.Cluster.DeleteComponentTemplateAsync("books_mappings"); - Debug.Assert(deleteComponentTemplate.IsValid, deleteComponentTemplate.DebugInformation); - Console.WriteLine($"Delete Component Template: {deleteComponentTemplate.IsValid}"); - } - - private class Book - { - public string? Title { get; set; } - public string? Author { get; set; } - public DateTime? PublishedOn { get; set; } - public int? Pages { get; set; } - } + public IndexTemplateSample() + : base("index-template", "A sample demonstrating how to use the client to create and manage index templates") { } + + protected override async Task Run(IOpenSearchClient client) + { + // Create index template + + var putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d + .IndexPatterns("books-*") + .Priority(0) + .Template(t => t + .Settings(s => s + .NumberOfShards(3) + .NumberOfReplicas(0)) + .Map(m => m + .Properties(p => p + .Text(f => f.Name(b => b.Title)) + .Text(f => f.Name(b => b.Author)) + .Date(f => f.Name(b => b.PublishedOn)) + .Number(f => f.Name(b => b.Pages).Type(NumberType.Integer)) + )))); + Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); + Console.WriteLine($"Put Template: {putTemplate.IsValid}"); + + // Confirm mapping + + var createIndex = await client.Indices.CreateAsync("books-nonfiction"); + Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); + Console.WriteLine($"Create Index: {createIndex.IsValid}"); + + var getIndex = await client.Indices.GetAsync("books-nonfiction"); + Debug.Assert( + getIndex.Indices["books-nonfiction"].Mappings.Properties["pages"].Type == "integer", + "`pages` property should have `integer` type"); + Console.WriteLine($"`pages` property type: {getIndex.Indices["books-nonfiction"].Mappings.Properties["pages"].Type}"); + + // Multiple index templates + + putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d + .IndexPatterns("books-*") + .Priority(0) + .Template(t => t + .Settings(s => s + .NumberOfShards(3) + .NumberOfReplicas(0)))); + Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); + Console.WriteLine($"Put Template: {putTemplate.IsValid}"); + + putTemplate = await client.Indices.PutComposableTemplateAsync("books-fiction", d => d + .IndexPatterns("books-fiction-*") + .Priority(1) // higher priority than the `books` template + .Template(t => t + .Settings(s => s + .NumberOfShards(1) + .NumberOfReplicas(1)))); + Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); + Console.WriteLine($"Put Template: {putTemplate.IsValid}"); + + // Validate settings + + createIndex = await client.Indices.CreateAsync("books-fiction-romance"); + Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); + Console.WriteLine($"Create Index: {createIndex.IsValid}"); + + getIndex = await client.Indices.GetAsync("books-fiction-romance"); + Debug.Assert( + getIndex.Indices["books-fiction-romance"].Settings.NumberOfShards == 1, + "`books-fiction-romance` index should have 1 shard"); + Console.WriteLine($"Number of shards: {getIndex.Indices["books-fiction-romance"].Settings.NumberOfShards}"); + + // Component templates + + var putComponentTemplate = await client.Cluster.PutComponentTemplateAsync("books_mappings", d => d + .Template(t => t + .Map(m => m + .Properties(p => p + .Text(f => f.Name(b => b.Title)) + .Text(f => f.Name(b => b.Author)) + .Date(f => f.Name(b => b.PublishedOn)) + .Number(f => f.Name(b => b.Pages).Type(NumberType.Integer)) + )))); + Debug.Assert(putComponentTemplate.IsValid, putComponentTemplate.DebugInformation); + Console.WriteLine($"Put Component Template: {putComponentTemplate.IsValid}"); + + putTemplate = await client.Indices.PutComposableTemplateAsync("books", d => d + .IndexPatterns("books-*") + .Priority(0) + .ComposedOf("books_mappings") + .Template(t => t + .Settings(s => s + .NumberOfShards(3) + .NumberOfReplicas(0)))); + Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); + Console.WriteLine($"Put Template: {putTemplate.IsValid}"); + + putTemplate = await client.Indices.PutComposableTemplateAsync("books-fiction", d => d + .IndexPatterns("books-fiction-*") + .Priority(1) // higher priority than the `books` template + .ComposedOf("books_mappings") + .Template(t => t + .Settings(s => s + .NumberOfShards(1) + .NumberOfReplicas(1)))); + Debug.Assert(putTemplate.IsValid, putTemplate.DebugInformation); + Console.WriteLine($"Put Template: {putTemplate.IsValid}"); + + // Validate settings & mappings + createIndex = await client.Indices.CreateAsync("books-fiction-horror"); + Debug.Assert(createIndex.IsValid, createIndex.DebugInformation); + Console.WriteLine($"Create Index: {createIndex.IsValid}"); + + getIndex = await client.Indices.GetAsync("books-fiction-horror"); + Debug.Assert( + getIndex.Indices["books-fiction-horror"].Settings.NumberOfShards == 1, + "`books-fiction-horror` index should have 1 shard"); + Debug.Assert( + getIndex.Indices["books-fiction-horror"].Mappings.Properties["pages"].Type == "integer", + "`pages` property should have `integer` type"); + Console.WriteLine($"Number of shards: {getIndex.Indices["books-fiction-horror"].Settings.NumberOfShards}"); + Console.WriteLine($"`pages` property type: {getIndex.Indices["books-fiction-horror"].Mappings.Properties["pages"].Type}"); + + // Get index template + + var getTemplate = await client.Indices.GetComposableTemplateAsync("books"); + Debug.Assert( + getTemplate.IndexTemplates.First().IndexTemplate.IndexPatterns.First() == "books-*", + "First index pattern should be `books-*`"); + Console.WriteLine($"First index pattern: {getTemplate.IndexTemplates.First().IndexTemplate.IndexPatterns.First()}"); + + // Delete index template + + var deleteTemplate = await client.Indices.DeleteComposableTemplateAsync("books"); + Debug.Assert(deleteTemplate.IsValid, deleteTemplate.DebugInformation); + Console.WriteLine($"Delete Template: {deleteTemplate.IsValid}"); + + // Cleanup + + var deleteIndex = await client.Indices.DeleteAsync("books-*"); + Debug.Assert(deleteIndex.IsValid, deleteIndex.DebugInformation); + Console.WriteLine($"Delete Index: {deleteIndex.IsValid}"); + + deleteTemplate = await client.Indices.DeleteComposableTemplateAsync("books-fiction"); + Debug.Assert(deleteTemplate.IsValid, deleteTemplate.DebugInformation); + Console.WriteLine($"Delete Template: {deleteTemplate.IsValid}"); + + var deleteComponentTemplate = await client.Cluster.DeleteComponentTemplateAsync("books_mappings"); + Debug.Assert(deleteComponentTemplate.IsValid, deleteComponentTemplate.DebugInformation); + Console.WriteLine($"Delete Component Template: {deleteComponentTemplate.IsValid}"); + } + + private class Book + { + public string? Title { get; set; } + public string? Author { get; set; } + public DateTime? PublishedOn { get; set; } + public int? Pages { get; set; } + } } diff --git a/samples/Samples/NeuralSearch/NeuralSearchSample.cs b/samples/Samples/NeuralSearch/NeuralSearchSample.cs index dc459d5418..c84f1fb4a3 100644 --- a/samples/Samples/NeuralSearch/NeuralSearchSample.cs +++ b/samples/Samples/NeuralSearch/NeuralSearchSample.cs @@ -58,9 +58,9 @@ protected override async Task Run(IOpenSearchClient client) description = $"A model group for the opensearch-net {SampleName} sample", access_mode = "public" })); - Debug.Assert(registerModelGroupResp.Success && (string) registerModelGroupResp.Body.status == "CREATED", registerModelGroupResp.DebugInformation); + Debug.Assert(registerModelGroupResp.Success && (string)registerModelGroupResp.Body.status == "CREATED", registerModelGroupResp.DebugInformation); Console.WriteLine($"Model group named {MlModelGroupName} {registerModelGroupResp.Body.status}: {registerModelGroupResp.Body.model_group_id}"); - _modelGroupId = (string) registerModelGroupResp.Body.model_group_id; + _modelGroupId = (string)registerModelGroupResp.Body.model_group_id; // Register the ML model var registerModelResp = await client.Http.PostAsync( @@ -72,16 +72,16 @@ protected override async Task Run(IOpenSearchClient client) model_group_id = _modelGroupId, model_format = "TORCH_SCRIPT" })); - Debug.Assert(registerModelResp.Success && (string) registerModelResp.Body.status == "CREATED", registerModelResp.DebugInformation); + Debug.Assert(registerModelResp.Success && (string)registerModelResp.Body.status == "CREATED", registerModelResp.DebugInformation); Console.WriteLine($"Model registration task {registerModelResp.Body.status}: {registerModelResp.Body.task_id}"); - _modelRegistrationTaskId = (string) registerModelResp.Body.task_id; + _modelRegistrationTaskId = (string)registerModelResp.Body.task_id; // Wait for ML model registration to complete while (true) { var getTaskResp = await client.Http.GetAsync($"/_plugins/_ml/tasks/{_modelRegistrationTaskId}"); Console.WriteLine($"Model registration: {getTaskResp.Body.state}"); - Debug.Assert(getTaskResp.Success && (string) getTaskResp.Body.state != "FAILED", getTaskResp.DebugInformation); + Debug.Assert(getTaskResp.Success && (string)getTaskResp.Body.state != "FAILED", getTaskResp.DebugInformation); if (((string)getTaskResp.Body.state).StartsWith("COMPLETED")) { _modelId = getTaskResp.Body.model_id; @@ -93,16 +93,16 @@ protected override async Task Run(IOpenSearchClient client) // Deploy the ML model var deployModelResp = await client.Http.PostAsync($"/_plugins/_ml/models/{_modelId}/_deploy"); - Debug.Assert(deployModelResp.Success && (string) deployModelResp.Body.status == "CREATED", deployModelResp.DebugInformation); + Debug.Assert(deployModelResp.Success && (string)deployModelResp.Body.status == "CREATED", deployModelResp.DebugInformation); Console.WriteLine($"Model deployment task {deployModelResp.Body.status}: {deployModelResp.Body.task_id}"); - _modelDeployTaskId = (string) deployModelResp.Body.task_id; + _modelDeployTaskId = (string)deployModelResp.Body.task_id; // Wait for ML model deployment to complete while (true) { var getTaskResp = await client.Http.GetAsync($"/_plugins/_ml/tasks/{_modelDeployTaskId}"); Console.WriteLine($"Model deployment: {getTaskResp.Body.state}"); - Debug.Assert(getTaskResp.Success && (string) getTaskResp.Body.state != "FAILED", getTaskResp.DebugInformation); + Debug.Assert(getTaskResp.Success && (string)getTaskResp.Body.state != "FAILED", getTaskResp.DebugInformation); if (((string)getTaskResp.Body.state).StartsWith("COMPLETED")) break; await Task.Delay(10000); } @@ -221,7 +221,7 @@ protected override async Task Cleanup(IOpenSearchClient client) { // Cleanup the model deployment task var deleteModelDeployTaskResp = await client.Http.DeleteAsync($"/_plugins/_ml/tasks/{_modelDeployTaskId}"); - Debug.Assert(deleteModelDeployTaskResp.Success && (string) deleteModelDeployTaskResp.Body.result == "deleted", deleteModelDeployTaskResp.DebugInformation); + Debug.Assert(deleteModelDeployTaskResp.Success && (string)deleteModelDeployTaskResp.Body.result == "deleted", deleteModelDeployTaskResp.DebugInformation); Console.WriteLine($"Deleted model deployment task: {deleteModelDeployTaskResp.Body.result}"); } @@ -251,7 +251,7 @@ protected override async Task Cleanup(IOpenSearchClient client) { // Cleanup the model registration task var deleteModelRegistrationTaskResp = await client.Http.DeleteAsync($"/_plugins/_ml/tasks/{_modelRegistrationTaskId}"); - Debug.Assert(deleteModelRegistrationTaskResp.Success && (string) deleteModelRegistrationTaskResp.Body.result == "deleted", deleteModelRegistrationTaskResp.DebugInformation); + Debug.Assert(deleteModelRegistrationTaskResp.Success && (string)deleteModelRegistrationTaskResp.Body.result == "deleted", deleteModelRegistrationTaskResp.DebugInformation); Console.WriteLine($"Deleted model registration task: {deleteModelRegistrationTaskResp.Body.result}"); } @@ -259,7 +259,7 @@ protected override async Task Cleanup(IOpenSearchClient client) { // Cleanup the model group var deleteModelGroupResp = await client.Http.DeleteAsync($"/_plugins/_ml/model_groups/{_modelGroupId}"); - Debug.Assert(deleteModelGroupResp.Success && (string) deleteModelGroupResp.Body.result == "deleted", deleteModelGroupResp.DebugInformation); + Debug.Assert(deleteModelGroupResp.Success && (string)deleteModelGroupResp.Body.result == "deleted", deleteModelGroupResp.DebugInformation); Console.WriteLine($"Deleted model group: {deleteModelGroupResp.Body.result}"); } } diff --git a/samples/Samples/Program.cs b/samples/Samples/Program.cs index e4174840de..d7f1485489 100644 --- a/samples/Samples/Program.cs +++ b/samples/Samples/Program.cs @@ -12,13 +12,13 @@ namespace Samples; public static class Program { - public static async Task Main(string[] args) - { - var rootCommand = new RootCommand("A collection of samples demonstrating how to use the OpenSearch .NET client"); - var clientDescriptor = rootCommand.AddOpenSearchClientOptions(); + public static async Task Main(string[] args) + { + var rootCommand = new RootCommand("A collection of samples demonstrating how to use the OpenSearch .NET client"); + var clientDescriptor = rootCommand.AddOpenSearchClientOptions(); - foreach (var sample in Sample.GetAllSamples()) rootCommand.AddCommand(sample.AsCommand(clientDescriptor)); + foreach (var sample in Sample.GetAllSamples()) rootCommand.AddCommand(sample.AsCommand(clientDescriptor)); - await rootCommand.InvokeAsync(args); - } + await rootCommand.InvokeAsync(args); + } } diff --git a/samples/Samples/RawJson/RawJsonHighLevelSample.cs b/samples/Samples/RawJson/RawJsonHighLevelSample.cs index b229918559..38f0118af5 100644 --- a/samples/Samples/RawJson/RawJsonHighLevelSample.cs +++ b/samples/Samples/RawJson/RawJsonHighLevelSample.cs @@ -13,64 +13,64 @@ namespace Samples.RawJson; public class RawJsonHighLevelSample : Sample { - public RawJsonHighLevelSample() : base("raw-json-high-level", "A sample demonstrating how to use the high-level client to perform raw JSON requests") { } + public RawJsonHighLevelSample() : base("raw-json-high-level", "A sample demonstrating how to use the high-level client to perform raw JSON requests") { } protected override async Task Run(IOpenSearchClient client) { var info = await client.Http.GetAsync("/"); - Debug.Assert(info.Success, info.DebugInformation); + Debug.Assert(info.Success, info.DebugInformation); Console.WriteLine($"Welcome to {info.Body.version.distribution} {info.Body.version.number}!"); - const string indexName = "movies"; + const string indexName = "movies"; - // Check if the index already exists + // Check if the index already exists - var indexExists = await client.Http.HeadAsync($"/{indexName}"); - Debug.Assert(indexExists.HttpStatusCode == 404, indexExists.DebugInformation); - Console.WriteLine($"Index Exists: {indexExists.HttpStatusCode == 200}"); + var indexExists = await client.Http.HeadAsync($"/{indexName}"); + Debug.Assert(indexExists.HttpStatusCode == 404, indexExists.DebugInformation); + Console.WriteLine($"Index Exists: {indexExists.HttpStatusCode == 200}"); - // Create an index + // Create an index - var indexBody = new { settings = new { index = new { number_of_shards = 4 } } }; + var indexBody = new { settings = new { index = new { number_of_shards = 4 } } }; var createIndex = await client.Http.PutAsync($"/{indexName}", d => d.SerializableBody(indexBody)); - Debug.Assert(createIndex.Success && (bool)createIndex.Body.acknowledged, createIndex.DebugInformation); - Console.WriteLine($"Create Index: {createIndex.Success && (bool)createIndex.Body.acknowledged}"); + Debug.Assert(createIndex.Success && (bool)createIndex.Body.acknowledged, createIndex.DebugInformation); + Console.WriteLine($"Create Index: {createIndex.Success && (bool)createIndex.Body.acknowledged}"); - // Add a document to the index - var document = new { title = "Moneyball", director = "Bennett Miller", year = 2011 }; + // Add a document to the index + var document = new { title = "Moneyball", director = "Bennett Miller", year = 2011 }; - const string id = "1"; + const string id = "1"; - var addDocument = await client.Http.PutAsync( - $"/{indexName}/_doc/{id}", - d => d - .SerializableBody(document) - .QueryString(qs => qs.Add("refresh", true))); - Debug.Assert(addDocument.Success, addDocument.DebugInformation); + var addDocument = await client.Http.PutAsync( + $"/{indexName}/_doc/{id}", + d => d + .SerializableBody(document) + .QueryString(qs => qs.Add("refresh", true))); + Debug.Assert(addDocument.Success, addDocument.DebugInformation); // Search for a document const string q = "miller"; - var query = new - { - size = 5, - query = new { multi_match = new { query = q, fields = new[] { "title^2", "director" } } } - }; + var query = new + { + size = 5, + query = new { multi_match = new { query = q, fields = new[] { "title^2", "director" } } } + }; var search = await client.Http.PostAsync($"/{indexName}/_search", d => d.SerializableBody(query)); - Debug.Assert(search.Success, search.DebugInformation); + Debug.Assert(search.Success, search.DebugInformation); - foreach (var hit in search.Body.hits.hits) Console.WriteLine($"Search Hit: {hit["_source"]["title"]}"); + foreach (var hit in search.Body.hits.hits) Console.WriteLine($"Search Hit: {hit["_source"]["title"]}"); - // Delete the document - var deleteDocument = await client.Http.DeleteAsync($"/{indexName}/_doc/{id}"); - Debug.Assert(deleteDocument.Success, deleteDocument.DebugInformation); - Console.WriteLine($"Delete Document: {deleteDocument.Success}"); + // Delete the document + var deleteDocument = await client.Http.DeleteAsync($"/{indexName}/_doc/{id}"); + Debug.Assert(deleteDocument.Success, deleteDocument.DebugInformation); + Console.WriteLine($"Delete Document: {deleteDocument.Success}"); - // Delete the index + // Delete the index var deleteIndex = await client.Http.DeleteAsync($"/{indexName}"); - Debug.Assert(deleteIndex.Success && (bool)deleteIndex.Body.acknowledged, deleteIndex.DebugInformation); - Console.WriteLine($"Delete Index: {deleteIndex.Success && (bool)deleteIndex.Body.acknowledged}"); + Debug.Assert(deleteIndex.Success && (bool)deleteIndex.Body.acknowledged, deleteIndex.DebugInformation); + Console.WriteLine($"Delete Index: {deleteIndex.Success && (bool)deleteIndex.Body.acknowledged}"); } } diff --git a/samples/Samples/RawJson/RawJsonLowLevelSample.cs b/samples/Samples/RawJson/RawJsonLowLevelSample.cs index e8248de1d6..5046ae1373 100644 --- a/samples/Samples/RawJson/RawJsonLowLevelSample.cs +++ b/samples/Samples/RawJson/RawJsonLowLevelSample.cs @@ -14,63 +14,63 @@ namespace Samples.RawJson; public class RawJsonLowLevelSample : Sample { - public RawJsonLowLevelSample() : base("raw-json-low-level", "A sample demonstrating how to use the low-level client to perform raw JSON requests") { } + public RawJsonLowLevelSample() : base("raw-json-low-level", "A sample demonstrating how to use the low-level client to perform raw JSON requests") { } protected override async Task Run(IOpenSearchClient client) { var info = await client.LowLevel.Http.GetAsync("/"); - Debug.Assert(info.Success, info.DebugInformation); + Debug.Assert(info.Success, info.DebugInformation); Console.WriteLine($"Welcome to {info.Body.version.distribution} {info.Body.version.number}!"); - const string indexName = "movies"; + const string indexName = "movies"; - // Check if the index already exists + // Check if the index already exists - var indexExists = await client.LowLevel.Http.HeadAsync($"/{indexName}"); - Debug.Assert(indexExists.HttpStatusCode == 404, indexExists.DebugInformation); - Console.WriteLine($"Index Exists: {indexExists.HttpStatusCode == 200}"); + var indexExists = await client.LowLevel.Http.HeadAsync($"/{indexName}"); + Debug.Assert(indexExists.HttpStatusCode == 404, indexExists.DebugInformation); + Console.WriteLine($"Index Exists: {indexExists.HttpStatusCode == 200}"); - // Create an index + // Create an index - var indexBody = new { settings = new { index = new { number_of_shards = 4 } } }; + var indexBody = new { settings = new { index = new { number_of_shards = 4 } } }; var createIndex = await client.LowLevel.Http.PutAsync($"/{indexName}", PostData.Serializable(indexBody)); - Debug.Assert(createIndex.Success && (bool)createIndex.Body.acknowledged, createIndex.DebugInformation); - Console.WriteLine($"Create Index: {createIndex.Success && (bool)createIndex.Body.acknowledged}"); + Debug.Assert(createIndex.Success && (bool)createIndex.Body.acknowledged, createIndex.DebugInformation); + Console.WriteLine($"Create Index: {createIndex.Success && (bool)createIndex.Body.acknowledged}"); - // Add a document to the index - var document = new { title = "Moneyball", director = "Bennett Miller", year = 2011}; + // Add a document to the index + var document = new { title = "Moneyball", director = "Bennett Miller", year = 2011 }; - const string id = "1"; + const string id = "1"; - var addDocument = await client.LowLevel.Http.PutAsync( - $"/{indexName}/_doc/{id}", - PostData.Serializable(document), - new HttpPutRequestParameters{ QueryString = {{"refresh", true}} }); - Debug.Assert(addDocument.Success, addDocument.DebugInformation); + var addDocument = await client.LowLevel.Http.PutAsync( + $"/{indexName}/_doc/{id}", + PostData.Serializable(document), + new HttpPutRequestParameters { QueryString = { { "refresh", true } } }); + Debug.Assert(addDocument.Success, addDocument.DebugInformation); // Search for a document const string q = "miller"; - var query = new - { - size = 5, - query = new { multi_match = new { query = q, fields = new[] { "title^2", "director" } } } - }; + var query = new + { + size = 5, + query = new { multi_match = new { query = q, fields = new[] { "title^2", "director" } } } + }; var search = await client.LowLevel.Http.PostAsync($"/{indexName}/_search", PostData.Serializable(query)); - Debug.Assert(search.Success, search.DebugInformation); + Debug.Assert(search.Success, search.DebugInformation); - foreach (var hit in search.Body.hits.hits) Console.WriteLine($"Search Hit: {hit["_source"]["title"]}"); + foreach (var hit in search.Body.hits.hits) Console.WriteLine($"Search Hit: {hit["_source"]["title"]}"); - // Delete the document - var deleteDocument = await client.LowLevel.Http.DeleteAsync($"/{indexName}/_doc/{id}"); - Debug.Assert(deleteDocument.Success, deleteDocument.DebugInformation); - Console.WriteLine($"Delete Document: {deleteDocument.Success}"); + // Delete the document + var deleteDocument = await client.LowLevel.Http.DeleteAsync($"/{indexName}/_doc/{id}"); + Debug.Assert(deleteDocument.Success, deleteDocument.DebugInformation); + Console.WriteLine($"Delete Document: {deleteDocument.Success}"); - // Delete the index + // Delete the index var deleteIndex = await client.LowLevel.Http.DeleteAsync($"/{indexName}"); - Debug.Assert(deleteIndex.Success && (bool)deleteIndex.Body.acknowledged, deleteIndex.DebugInformation); - Console.WriteLine($"Delete Index: {deleteIndex.Success && (bool)deleteIndex.Body.acknowledged}"); + Debug.Assert(deleteIndex.Success && (bool)deleteIndex.Body.acknowledged, deleteIndex.DebugInformation); + Console.WriteLine($"Delete Index: {deleteIndex.Success && (bool)deleteIndex.Body.acknowledged}"); } } diff --git a/samples/Samples/Sample.cs b/samples/Samples/Sample.cs index e683b98935..9202c801c7 100644 --- a/samples/Samples/Sample.cs +++ b/samples/Samples/Sample.cs @@ -13,27 +13,27 @@ namespace Samples; public abstract class Sample { - public static IEnumerable GetAllSamples() => - typeof(Sample) - .Assembly - .GetTypes() - .Where(t => !t.IsAbstract && t.IsSubclassOf(typeof(Sample))) - .Select(t => (Sample) Activator.CreateInstance(t)!); - - private readonly string _name; - private readonly string _description; - - protected Sample(string name, string description) - { - _name = name; - _description = description; - } - - public Command AsCommand(IValueDescriptor clientDescriptor) - { - var command = new Command(_name, _description); - - command.SetHandler(async client => + public static IEnumerable GetAllSamples() => + typeof(Sample) + .Assembly + .GetTypes() + .Where(t => !t.IsAbstract && t.IsSubclassOf(typeof(Sample))) + .Select(t => (Sample)Activator.CreateInstance(t)!); + + private readonly string _name; + private readonly string _description; + + protected Sample(string name, string description) + { + _name = name; + _description = description; + } + + public Command AsCommand(IValueDescriptor clientDescriptor) + { + var command = new Command(_name, _description); + + command.SetHandler(async client => { try { @@ -52,10 +52,10 @@ public Command AsCommand(IValueDescriptor clientDescriptor) } }, clientDescriptor); - return command; - } + return command; + } - protected abstract Task Run(IOpenSearchClient client); + protected abstract Task Run(IOpenSearchClient client); protected virtual Task Cleanup(IOpenSearchClient client) => Task.CompletedTask; } diff --git a/samples/Samples/Utils/OpenSearchClientOptions.cs b/samples/Samples/Utils/OpenSearchClientOptions.cs index ae42bdb387..947a6e8cb5 100644 --- a/samples/Samples/Utils/OpenSearchClientOptions.cs +++ b/samples/Samples/Utils/OpenSearchClientOptions.cs @@ -14,46 +14,46 @@ namespace Samples.Utils; public static class OpenSearchClientOptions { - public static IValueDescriptor AddOpenSearchClientOptions(this Command command, bool global = true) - { - Option host = new("--host", () => new Uri("https://localhost:9200"), "The OpenSearch host to connect to"); - Option username = new("--username", () => "admin", "The username to use for authentication"); - Option password = new("--password", () => "admin", "The password to use for authentication"); - - Action