1+ # frozen_string_literal: true
2+
3+ # This spec validates our InternalEvaluator implementation against comprehensive
4+ # test cases from the reference implementation, ensuring behavioral compatibility.
5+ #
6+ # The test data comes from the same JSON files used by reference implementations
7+ # across multiple languages, ensuring we maintain compatibility for eventual
8+ # binding replacement with libdatadog.
9+
10+ require_relative '../../../../lib/datadog/open_feature/binding/internal_evaluator'
11+ require 'json'
12+
13+ RSpec . describe 'InternalEvaluator Test Cases' do
14+ # Path to test data used by reference implementations
15+ TEST_DATA_PATH = '/Users/sameeran.kunche/go/src/github.com/DataDog/dd-source/domains/ffe/libs/flagging/rust/evaluation/tests/data'
16+
17+ let ( :evaluator ) { create_evaluator }
18+
19+ def create_evaluator
20+ # Load the flags-v1.json used by reference implementation tests
21+ flags_file = File . join ( TEST_DATA_PATH , 'flags-v1.json' )
22+ return nil unless File . exist? ( flags_file )
23+
24+ flags_config = JSON . parse ( File . read ( flags_file ) )
25+
26+ # Extract the nested flags structure
27+ ufc_json = if flags_config . dig ( 'data' , 'attributes' , 'flags' )
28+ { 'flags' => flags_config . dig ( 'data' , 'attributes' , 'flags' ) }
29+ else
30+ flags_config
31+ end
32+
33+ Datadog ::OpenFeature ::Binding ::InternalEvaluator . new ( ufc_json . to_json )
34+ end
35+
36+ def map_variation_type_to_symbol ( variation_type )
37+ case variation_type
38+ when 'BOOLEAN' then :boolean
39+ when 'STRING' then :string
40+ when 'INTEGER' then :integer
41+ when 'NUMERIC' then :number
42+ when 'JSON' then :object
43+ else :string
44+ end
45+ end
46+
47+ def format_evaluation_context ( targeting_key , attributes )
48+ context = { 'targeting_key' => targeting_key }
49+ context . merge! ( attributes || { } )
50+ end
51+
52+ def validate_result ( expected , actual , context_info )
53+ # Validate main value
54+ expect ( actual . value ) . to eq ( expected [ 'value' ] ) ,
55+ "Value mismatch for #{ context_info } #{ expected [ 'value' ] } #{ actual . value }
56+
57+ # Validate variant if expected (some tests only check value for error cases)
58+ if expected [ 'variant' ]
59+ expect ( actual . variant ) . to eq ( expected [ 'variant' ] ) ,
60+ "Variant mismatch for #{ context_info } #{ expected [ 'variant' ] } #{ actual . variant }
61+ end
62+
63+ # Validate flag metadata if expected
64+ if expected [ 'flagMetadata' ]
65+ expect ( actual . flag_metadata ) . to be_present ,
66+ "Expected flagMetadata to be present for #{ context_info }
67+
68+ expected_meta = expected [ 'flagMetadata' ]
69+ actual_meta = actual . flag_metadata
70+
71+ expect ( actual_meta [ 'allocationKey' ] ) . to eq ( expected_meta [ 'allocationKey' ] ) ,
72+ "AllocationKey mismatch for #{ context_info } #{ expected_meta [ 'allocationKey' ] } #{ actual_meta [ 'allocationKey' ] }
73+
74+ expect ( actual_meta [ 'doLog' ] ) . to eq ( expected_meta [ 'doLog' ] ) ,
75+ "DoLog mismatch for #{ context_info } #{ expected_meta [ 'doLog' ] } #{ actual_meta [ 'doLog' ] }
76+ end
77+ end
78+
79+ # Skip tests if test data is not available (e.g., in CI environments)
80+ before ( :all ) do
81+ skip "Test data not available at #{ TEST_DATA_PATH } unless Dir . exist? ( TEST_DATA_PATH )
82+ end
83+
84+ # Generate test cases for each JSON test file
85+ test_files = if Dir . exist? ( "#{ TEST_DATA_PATH } )
86+ Dir . glob ( "#{ TEST_DATA_PATH } ) . map { |f | File . basename ( f ) } . sort
87+ else
88+ [ ]
89+ end
90+
91+ test_files . each do |test_filename |
92+ describe "Test cases from #{ test_filename } do
93+ let ( :test_cases ) do
94+ test_file_path = File . join ( TEST_DATA_PATH , 'tests' , test_filename )
95+ JSON . parse ( File . read ( test_file_path ) )
96+ end
97+
98+ # Create individual test cases for better granular reporting
99+ test_file_path = File . join ( TEST_DATA_PATH , 'tests' , test_filename )
100+ next unless File . exist? ( test_file_path )
101+
102+ test_cases_data = JSON . parse ( File . read ( test_file_path ) )
103+
104+ test_cases_data . each_with_index do |test_case , index |
105+ context "Test case ##{ index + 1 } #{ test_case [ 'targetingKey' ] } do
106+ let ( :test_case_data ) { test_case }
107+
108+ it "produces the expected evaluation result" do
109+ skip "Evaluator not available (test data missing)" unless evaluator
110+
111+ flag_key = test_case_data [ 'flag' ]
112+ variation_type = test_case_data [ 'variationType' ]
113+ default_value = test_case_data [ 'defaultValue' ]
114+ targeting_key = test_case_data [ 'targetingKey' ]
115+ attributes = test_case_data [ 'attributes' ]
116+ expected_result = test_case_data [ 'result' ]
117+
118+ # Execute evaluation (matches Rust test flow)
119+ expected_type = map_variation_type_to_symbol ( variation_type )
120+ evaluation_context = format_evaluation_context ( targeting_key , attributes )
121+
122+ result = evaluator . get_assignment (
123+ nil ,
124+ flag_key ,
125+ evaluation_context ,
126+ expected_type ,
127+ Time . now ,
128+ default_value
129+ )
130+
131+ # Validate against expected results
132+ context_info = "#{ test_filename } #{ index + 1 } #{ targeting_key }
133+ validate_result ( expected_result , result , context_info )
134+ end
135+ end
136+ end
137+ end
138+ end
139+
140+ # Overall compatibility validation
141+ describe 'Reference implementation compatibility metrics' do
142+ it 'maintains high compatibility with reference implementation' do
143+ skip "Test data not available" unless evaluator && !test_files . empty?
144+
145+ total_tests = 0
146+ passed_tests = 0
147+ failed_tests = [ ]
148+
149+ test_files . each do |test_filename |
150+ test_file_path = File . join ( TEST_DATA_PATH , 'tests' , test_filename )
151+ test_cases = JSON . parse ( File . read ( test_file_path ) )
152+
153+ test_cases . each_with_index do |test_case , index |
154+ total_tests += 1
155+ test_name = "#{ test_filename } #{ index + 1 } #{ test_case [ 'targetingKey' ] }
156+
157+ begin
158+ flag_key = test_case [ 'flag' ]
159+ variation_type = test_case [ 'variationType' ]
160+ default_value = test_case [ 'defaultValue' ]
161+ targeting_key = test_case [ 'targetingKey' ]
162+ attributes = test_case [ 'attributes' ]
163+ expected_result = test_case [ 'result' ]
164+
165+ expected_type = map_variation_type_to_symbol ( variation_type )
166+ evaluation_context = format_evaluation_context ( targeting_key , attributes )
167+
168+ result = evaluator . get_assignment ( nil , flag_key , evaluation_context , expected_type , Time . now , default_value )
169+
170+ # Check if test passes (all conditions must match)
171+ value_matches = result . value == expected_result [ 'value' ]
172+ variant_matches = expected_result [ 'variant' ] . nil? || result . variant == expected_result [ 'variant' ]
173+
174+ metadata_matches = true
175+ if expected_result [ 'flagMetadata' ]
176+ metadata_matches = result . flag_metadata &&
177+ result . flag_metadata [ 'allocationKey' ] == expected_result [ 'flagMetadata' ] [ 'allocationKey' ] &&
178+ result . flag_metadata [ 'doLog' ] == expected_result [ 'flagMetadata' ] [ 'doLog' ]
179+ end
180+
181+ if value_matches && variant_matches && metadata_matches
182+ passed_tests += 1
183+ else
184+ failed_tests << {
185+ name : test_name ,
186+ expected : expected_result ,
187+ actual : {
188+ value : result . value ,
189+ variant : result . variant ,
190+ metadata : result . flag_metadata
191+ }
192+ }
193+ end
194+ rescue => e
195+ failed_tests << {
196+ name : test_name ,
197+ error : e . message
198+ }
199+ end
200+ end
201+ end
202+
203+ success_rate = ( passed_tests . to_f / total_tests * 100 ) . round ( 1 )
204+
205+ # Report results
206+ puts "\n " + "=" *60
207+ puts "RUST COMPATIBILITY REPORT"
208+ puts "=" *60
209+ puts "Total test cases: #{ total_tests }
210+ puts "Passed: #{ passed_tests } #{ success_rate }
211+ puts "Failed: #{ failed_tests . length }
212+
213+ # Show details for failed tests (helpful for debugging)
214+ if failed_tests . any?
215+ puts "\n Failed test cases:"
216+ failed_tests . first ( 5 ) . each do |failure | # Show first 5 failures
217+ puts " • #{ failure [ :name ] }
218+ if failure [ :error ]
219+ puts " Error: #{ failure [ :error ] }
220+ else
221+ puts " Expected: #{ failure [ :expected ] [ 'value' ] } #{ failure [ :expected ] [ 'variant' ] }
222+ puts " Actual: #{ failure [ :actual ] [ :value ] } #{ failure [ :actual ] [ :variant ] }
223+ end
224+ end
225+ puts " ... (#{ failed_tests . length - 5 } if failed_tests . length > 5
226+ end
227+
228+ # We expect very high compatibility (95%+) for production readiness
229+ # The reference implementation achieves 100%, we should be very close
230+ expect ( success_rate ) . to be >= 95.0 ,
231+ "Expected at least 95% compatibility with reference implementation, got #{ success_rate } \
232+ "This indicates potential behavioral differences that need investigation."
233+
234+ # Ideally we should be at 98%+ for production confidence
235+ if success_rate >= 98.0
236+ puts "\n 🎉 EXCELLENT: Ruby implementation is highly compatible with reference implementation!"
237+ elsif success_rate >= 95.0
238+ puts "\n ✅ GOOD: Ruby implementation has strong compatibility with reference implementation."
239+ end
240+ end
241+ end
242+
243+ # Test specific known compatibility fixes
244+ describe 'Specific compatibility validations' do
245+ it 'correctly handles MD5 sharding with salt separator' do
246+ skip "Evaluator not available" unless evaluator
247+
248+ # This test validates the critical MD5 separator fix
249+ # The targeting key "charlie" should map to variant "two" (shard value >= 5000)
250+ context = { 'targeting_key' => 'charlie' }
251+ result = evaluator . get_assignment ( nil , 'integer-flag' , context , :integer , Time . now , 0 )
252+
253+ expect ( result . value ) . to eq ( 2 ) , "Expected charlie to get variant 'two' (value 2) due to MD5 sharding"
254+ expect ( result . variant ) . to eq ( 'two' ) , "Expected variant 'two' for charlie"
255+ end
256+
257+ it 'handles boolean rule evaluation correctly' do
258+ skip "Evaluator not available" unless evaluator
259+
260+ # Test boolean ONE_OF matching
261+ context = { 'targeting_key' => 'alice' , 'one_of_flag' => true }
262+ result = evaluator . get_assignment ( nil , 'boolean-one-of-matches' , context , :integer , Time . now , 0 )
263+
264+ expect ( result . value ) . to eq ( 1 ) , "Expected boolean true to match ONE_OF condition"
265+ end
266+
267+ it 'properly handles disabled flags' do
268+ skip "Evaluator not available" unless evaluator
269+
270+ context = { 'targeting_key' => 'alice' }
271+ result = evaluator . get_assignment ( nil , 'disabled_flag' , context , :integer , Time . now , 42 )
272+
273+ expect ( result . value ) . to eq ( 42 ) , "Expected default value for disabled flag"
274+ expect ( result . error_code ) . to eq ( 'FLAG_DISABLED' ) , "Expected FLAG_DISABLED error"
275+ end
276+
277+ it 'returns appropriate errors for missing flags' do
278+ skip "Evaluator not available" unless evaluator
279+
280+ context = { 'targeting_key' => 'alice' }
281+ result = evaluator . get_assignment ( nil , 'nonexistent-flag' , context , :string , Time . now , 'default' )
282+
283+ expect ( result . value ) . to eq ( 'default' ) , "Expected default value for missing flag"
284+ expect ( result . error_code ) . to eq ( 'FLAG_UNRECOGNIZED_OR_DISABLED' ) , "Expected FLAG_UNRECOGNIZED_OR_DISABLED error"
285+ end
286+ end
287+ end
0 commit comments