Add new option: default_missing_path_to_null

README.md

@@ -100,7 +100,7 @@ JsonPath.new('$..color').first(json)
 # => "red"
 ```
 
-As well, we can directly create an `Enumerable` at any time using `#[]`. 
+As well, we can directly create an `Enumerable` at any time using `#[]`.
 
 ```ruby
 enum = JsonPath.new('$..color')[json]
@@ -157,7 +157,7 @@ JsonPath.new('$.title', allow_send: true).on(book)
 ### Other available options
 
 By default, JsonPath does not return null values on unexisting paths.
-This can be changed using the `:default_path_leaf_to_null` option
+This can be changed using the `:default_path_leaf_to_null` option to return nil for leaf nodes:
 
 ```ruby
 JsonPath.new('$..book[*].isbn').on(json)
@@ -167,6 +167,28 @@ JsonPath.new('$..book[*].isbn', default_path_leaf_to_null: true).on(json)
 # => [nil, nil, "0-553-21311-3", "0-395-19395-8"]
 ```
 
+Or it can be changed using the `:default_missing_path_to_null` option which will include both
+leaf nodes and missing intermediate paths.
+
+```ruby
+data = [
+  { "review" => nil },
+  { "review" => { "rating" => 5 } },
+  { "review" => { "rating" => nil } },
+  { "review" => { "comment" => "good" } },
+  { "review" => { "rating" => 3 } }
+]
+
+JsonPath.new('$[*].review.rating').on(data)
+# => [5, nil, 3]
+
+JsonPath.new('$[*].review.rating', default_path_leaf_to_null: true).on(data)
+# => [5, nil, nil, 3]
+
+JsonPath.new('$[*].review.rating', default_missing_path_to_null: true).on(data)
+# => [nil, 5, nil, nil, 3]
+```
+
 When JsonPath returns a Hash, you can ask to symbolize its keys
 using the `:symbolize_keys` option
 

lib/jsonpath.rb

@@ -16,6 +16,7 @@ class JsonPath
 
   DEFAULT_OPTIONS = {
     :default_path_leaf_to_null => false,
+    :default_missing_path_to_null => false,
     :symbolize_keys => false,
     :use_symbols => false,
     :allow_send => true,
@@ -52,6 +53,10 @@ def initialize(path, opts = {})
         raise ArgumentError, "character '#{scanner.peek(1)}' not supported in query"
       end
     end
+
+    # Disable default_missing_path_to_null option for recursive descent paths
+    # To avoid duplicating nils during recursive descent for every checked path
+    @opts[:default_missing_path_to_null] = false if @path.include?('..')
   end
 
   def find_matching_brackets(token, scanner)

lib/jsonpath/dig.rb

@@ -42,11 +42,11 @@ def yield_if_diggable(context, k, &blk)
         nil
       when Hash
         k = @options[:use_symbols] ? k.to_sym : k
-        return yield if context.key?(k) || @options[:default_path_leaf_to_null]
+        return yield if context.key?(k) || @options[:default_path_leaf_to_null] || @options[:default_missing_path_to_null]
       else
         if context.respond_to?(:dig)
           digged = dig_one(context, k)
-          yield if !digged.nil? || @options[:default_path_leaf_to_null]
+          yield if !digged.nil? || @options[:default_path_leaf_to_null] || @options[:default_missing_path_to_null]
         elsif @options[:allow_send] && context.respond_to?(k.to_s) && !Object.respond_to?(k.to_s)
           yield
         end

lib/jsonpath/enumerable.rb

@@ -17,6 +17,27 @@ def each(context = @object, key = nil, pos = 0, &blk)
       @_current_node = node
       return yield_value(blk, context, key) if pos == @path.size
 
+      # If node is nil and we have default_missing_path_to_null option,
+      # continue processing to potentially yield nil at the end
+      if node.nil? && @options[:default_missing_path_to_null] && pos < @path.size
+
+        expr = @path[pos]
+        case expr
+        when '*', '..', '@'
+          each(nil, nil, pos + 1, &blk)
+        when '$'
+          each(nil, nil, pos + 1, &blk)
+        when /^\[(.*)\]$/
+          expr[1, expr.size - 2].split(',').each do |sub_path|
+            case sub_path[0]
+            when '\'', '"'
+              each(nil, nil, pos + 1, &blk)
+            end
+          end
+        end
+        return
+      end
+
       case expr = @path[pos]
       when '*', '..', '@'
         each(context, key, pos + 1, &blk)
@@ -77,7 +98,10 @@ def handle_wildcard(node, expr, _context, _key, pos, &blk)
           elsif sub_path.count(':') == 0
             start_idx = end_idx = process_function_or_literal(array_args[0], 0)
             next unless start_idx
-            next if start_idx >= node.size
+            if start_idx >= node.size
+              each(nil, nil, pos + 1, &blk) if @options[:default_missing_path_to_null]
+              next
+            end
           else
             start_idx = process_function_or_literal(array_args[0], 0)
             next unless start_idx
@@ -129,9 +153,13 @@ def handle_question_mark(sub_path, node, pos, &blk)
     def yield_value(blk, context, key)
       case @mode
       when nil
-        blk.call(key ? dig_one(context, key) : context)
+        if context.nil? && @options[:default_missing_path_to_null]
+          blk.call(nil)
+        else
+          blk.call(key ? dig_one(context, key) : context)
+        end
       when :compact
-        if key && context[key].nil?
+        if key && context&.dig(key).nil?
           key.is_a?(Integer) ? context.delete_at(key) : context.delete(key)
         end
       when :delete

test/test_jsonpath.rb

@@ -895,7 +895,7 @@ def test_complex_nested_grouping
     path = "$..book[?((@['author'] == 'Evelyn Waugh' || @['author'] == 'Herman Melville') && (@['price'] == 33 || @['price'] == 9))]"
     assert_equal [@object['store']['book'][2]], JsonPath.new(path).on(@object)
   end
-  
+
   def test_nested_with_unknown_key
     path = "$..[?(@.price == 9 || @.price == 33)].title"
     assert_equal ["Sayings of the Century", "Moby Dick", "Sayings of the Century", "Moby Dick"], JsonPath.new(path).on(@object)
@@ -905,7 +905,7 @@ def test_nested_with_unknown_key_filtered_array
     path = "$..[?(@['price'] == 9 || @['price'] == 33)].title"
     assert_equal ["Sayings of the Century", "Moby Dick", "Sayings of the Century", "Moby Dick"], JsonPath.new(path).on(@object)
   end
-  
+
   def test_runtime_error_frozen_string
     skip('in ruby version below 2.2.0 this error is not raised') if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.0') || Gem::Version.new(RUBY_VERSION) > Gem::Version::new('2.6')
     json = '
@@ -1318,4 +1318,137 @@ def test_symbolize_key
     assert_equal [{"category": "reference"}],  JsonPath.new('$..book[0]', symbolize_keys: true).on(data)
     assert_equal [{"category": "reference"}],  JsonPath.new('$..book[0]').on(data, symbolize_keys: true)
   end
+
+  def test_recursive_descent_with_default_options
+    data = { 'store' => {
+      'book' => [
+        { 'category' => 'reference',
+          'author' => 'Nigel Rees',
+          'title' => 'Sayings of the Century',
+          'price' => 9,
+          'review' => nil },
+        { 'category' => 'fiction',
+          'author' => 'Evelyn Waugh',
+          'title' => 'Sword of Honour',
+          'price' => 13,
+          'review' => { 'rating' => 5, 'comment' => 'Excellent!' } },
+        { 'category' => 'fiction',
+          'author' => 'Herman Melville',
+          'title' => 'Moby Dick',
+          'price' => 9 }
+      ]
+    }}
+
+    leaf_path_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true).on(data)
+    assert_equal [5], leaf_path_result
+
+    missing_path_result = JsonPath.new('$..book[*].review.rating', default_missing_path_to_null: true).on(data)
+    assert_equal [5], missing_path_result
+
+    normal_result = JsonPath.new('$..book[*].review.rating').on(data)
+    assert_equal [5], normal_result
+  end
+
+  def test_default_missing_path_to_null_multiple_nesting_levels
+    data = { 'store' => {
+      'book' => [
+        { 'title' => 'Book One',
+          'metadata' => nil }, # Null at first level
+        { 'title' => 'Book Two',
+          'metadata' => { 'publication' => nil } }, # Null at second level
+        { 'title' => 'Book Three',
+          'metadata' => {
+            'publication' => {
+              'details' => nil
+            }
+          } }, # Null at third level
+        { 'title' => 'Book Four',
+          'metadata' => {
+            'publication' => {
+              'details' => {
+                'isbn' => '978-0123456789'
+              }
+            }
+          } }
+      ]
+    }}
+
+    result_without = JsonPath.new('$.store.book[*].metadata.publication.details.isbn').on(data)
+    assert_equal ['978-0123456789'], result_without
+
+    result_with = JsonPath.new('$.store.book[*].metadata.publication.details.isbn', default_missing_path_to_null: true).on(data)
+    assert_equal [nil, nil, nil, '978-0123456789'], result_with
+  end
+
+  def test_default_missing_path_to_null_finds_missing_keys
+    data = { 'store' => {
+      'book' => [
+        { 'title' => 'The Hobbit',
+          'category' => 'fantasy'
+        },
+        { 'title' => 'The Great Gatsby',
+          'category' => 'classic',
+          'metadata' => { 'isbn' => '978-0743273565' }
+        }
+      ]
+    }}
+
+    result_without = JsonPath.new('$.store.book[*].metadata.isbn').on(data)
+    assert_equal ['978-0743273565'], result_without
+
+    result_with = JsonPath.new('$.store.book[*].metadata.isbn', default_missing_path_to_null: true).on(data)
+    assert_equal [nil, '978-0743273565'], result_with
+  end
+
+  def test_default_missing_path_to_null_with_array_index_access
+    data = [
+      { 'title' => 'Sayings of the Century',
+        'reviews' => [{ 'rating' => 5 }] },
+      { 'title' => 'Sword of Honour',
+        'reviews' => [{ 'rating' => 4 }, { 'rating' => 3 }] },
+      { 'title' => 'Moby Dick',
+        'reviews' => [{ 'rating' => 5 }] }
+    ]
+
+    result_without = JsonPath.new('$[*].reviews[1].rating').on(data)
+    assert_equal [3], result_without
+
+    result_with = JsonPath.new('$[*].reviews[1].rating', default_missing_path_to_null: true).on(data)
+    assert_equal [nil, 3, nil], result_with
+  end
+
+  def test_default_missing_path_to_null_avoids_key_collision
+    data = [
+      { 'title' => 'Book One', 'author' => nil },
+      { 'title' => 'Book Two', 'author' => nil },
+      { 'title' => 'Book Three', 'author' => { 'name' => 'Jane Doe', 'title' => 'Dr.' } }
+    ]
+    result_without = JsonPath.new('$[*].author.name', default_missing_path_to_null: true).on(data)
+    assert_equal [nil, nil, 'Jane Doe'], result_without
+
+    result_title = JsonPath.new('$[*].author.title', default_missing_path_to_null: true).on(data)
+    assert_equal [nil, nil, 'Dr.'], result_title
+  end
+
+  def test_default_missing_path_to_null_with_recursive_descent
+    data = {
+      'library' => {
+        'name' => 'Central Library',
+        'books' => [
+          { 'title' => 'Book One', 'author' => nil },
+          { 'title' => 'Book Two', 'author' => { 'name' => 'Jane Smith' } },
+          { 'title' => 'Book Three' }
+        ],
+        'magazines' => [
+          { 'title' => 'Mag One', 'author' => { 'name' => 'Bob Jones' } }
+        ]
+      }
+    }
+
+    result_without = JsonPath.new('$..author.name').on(data)
+    assert_equal ['Jane Smith', 'Bob Jones'], result_without
+
+    result_with = JsonPath.new('$..author.name', default_missing_path_to_null: true).on(data)
+    assert_equal ['Jane Smith', 'Bob Jones'], result_with
+  end
 end