fix: do not split quoted key as path

Author: zmstone

Makefile

@@ -4,7 +4,7 @@ REBAR := $(CURDIR)/rebar3
 all: es
 
 $(REBAR):
-	@curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.14.3-emqx-7/rebar3" -o ./rebar3
+	@curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.19.0-emqx-6/rebar3" -o ./rebar3
 	@chmod +x ./rebar3
 
 .PHONY: compile

src/hocon.erl

@@ -192,7 +192,12 @@ do_expand([Other | More], Acc) ->
     do_expand(More, [Other | Acc]).
 
 create_nested(#{?HOCON_T := key} = Key, Value) ->
-    do_create_nested(paths(value_of(Key)), Value, Key).
+    case value_of(Key) of
+        {keypath, Path} ->
+            do_create_nested(Path, Value, Key);
+        Path ->
+            do_create_nested([Path], Value, Key)
+    end.
 
 do_create_nested([], Value, _OriginalKey) ->
     Value;
@@ -398,7 +403,7 @@ transform(#{?HOCON_T := object, ?HOCON_V := V}, Opts) ->
 do_transform([], Map, _Opts) ->
     Map;
 do_transform([{Key, Value} | More], Map, Opts) ->
-    [KeyReal] = paths(value_of(Key)),
+    KeyReal = unicode_bin(value_of(Key)),
     ValueReal = unpack(Value, Opts),
     do_transform(More, merge(KeyReal, ValueReal, Map), Opts).
 

src/hocon_parser.yrl

@@ -12,7 +12,7 @@ Terminals
   '{' '}' '[' ']' ','
   bool integer float null
   percent bytesize duration
-  string variable
+  unqstr string variable
   endstr endvar endarr endobj
   include key required.
 
@@ -29,6 +29,7 @@ partials -> '[' elements endarr : [make_array(line_of('$1'), '$2')].
 partials -> '{' endobj : [make_object(line_of('$1'), [])].
 partials -> '[' endarr : [make_array(line_of('$1'), [])].
 
+partial -> unqstr : str_to_bin(make_primitive_value('$1')).
 partial -> string : str_to_bin(make_primitive_value('$1')).
 partial -> variable : make_variable('$1').
 partial -> '{' fields '}' : make_object(line_of('$1'), '$2').
@@ -47,8 +48,10 @@ elements -> value ',' elements : ['$1' | '$3'].
 elements -> value elements : ['$1' | '$2'].
 elements -> value : ['$1'].
 
+directive -> include unqstr : make_include('$2', false).
 directive -> include string : make_include('$2', false).
 directive -> include endstr : make_include('$2', false).
+directive -> include required unqstr : make_include('$3', true).
 directive -> include required string : make_include('$3', true).
 directive -> include required endstr : make_include('$3', true).
 
@@ -88,7 +91,7 @@ make_include(String, false) ->  #{'$hcTyp' => include,
 
 make_concat(S) -> #{'$hcTyp' => concat, '$hcVal' => S}.
 
-str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string -> M#{'$hcVal' => bin(V)}.
+str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string orelse T =:= unqstr -> M#{'$hcTyp' := string, '$hcVal' => bin(V)}.
 
 line_of(Token) -> element(2, Token).
 value_of(Token) -> element(3, Token).

src/hocon_pp.erl

@@ -184,7 +184,7 @@ is_quote_key(K) ->
 
 is_quote_str(S) ->
     case hocon_scanner:string(S) of
-        {ok, [{string, 1, S}], 1} ->
+        {ok, [{Tag, 1, S}], 1} when Tag =:= string orelse Tag =:= unqstr ->
             %% contain $"{}[]:=,+#`^?!@*& \\ should be quote
             case re:run(S, "^[^$\"{}\\[\\]:=,+#`\\^?!@*&\\ \\\\]*$") of
                 nomatch -> true;

src/hocon_scanner.xrl

@@ -93,7 +93,7 @@ Rules.
 Erlang code.
 
 maybe_include("include", TokenLine)  -> {include, TokenLine};
-maybe_include(TokenChars, TokenLine) -> {string, TokenLine, TokenChars}.
+maybe_include(TokenChars, TokenLine) -> {unqstr, TokenLine, TokenChars}.
 
 get_filename_from_required("required(" ++ Filename) ->
     [$) | FilenameRev] = lists:reverse(Filename),

src/hocon_token.erl

@@ -109,6 +109,8 @@ trans_key([{'{', Line} | Tokens], Acc) ->
 trans_key([T | Tokens], Acc) ->
     trans_key(Tokens, [T | Acc]).
 
+trans_key_lb([{unqstr, Line, Value} | TokensRev]) ->
+    [{key, Line, {keypath, paths(Value)}} | TokensRev];
 trans_key_lb([{string, Line, Value} | TokensRev]) ->
     [{key, Line, Value} | TokensRev];
 trans_key_lb(Otherwise) ->
@@ -145,6 +147,7 @@ trans_splice_end([], Seq, Acc) ->
     lists:reverse(NewAcc).
 
 do_trans_splice_end([]) -> [];
+do_trans_splice_end([{unqstr, Line, Value} | T]) -> [{endstr, Line, Value} | T];
 do_trans_splice_end([{string, Line, Value} | T]) -> [{endstr, Line, Value} | T];
 do_trans_splice_end([{variable, Line, Value} | T]) -> [{endvar, Line, Value} | T];
 do_trans_splice_end([{'}', Line} | T]) -> [{endobj, Line} | T];
@@ -225,7 +228,12 @@ abspath(Var, PathStack) ->
 do_abspath(Var, ['$root']) ->
     Var;
 do_abspath(Var, [#{?HOCON_T := key} = K | More]) ->
-    do_abspath(unicode_bin([value_of(K), <<".">>, Var]), More).
+    do_abspath(unicode_bin([maybe_join(value_of(K)), <<".">>, Var]), More).
+
+maybe_join({keypath, Path}) ->
+    infix(Path, ".");
+maybe_join(Path) ->
+    Path.
 
 -spec load_include(boxed(), hocon:ctx()) -> boxed() | nothing.
 
@@ -308,3 +316,12 @@ format_error(Line, ErrorInfo, Ctx) ->
 
 unicode_bin(L) -> unicode:characters_to_binary(L, utf8).
 unicode_list(B) -> unicode:characters_to_list(B, utf8).
+
+paths(Key) when is_binary(Key) ->
+    paths(unicode:characters_to_list(Key, utf8));
+paths(Key) when is_list(Key) ->
+    lists:map(fun unicode_bin/1, string:tokens(Key, ".")).
+
+infix([], _) -> [];
+infix([X], _) -> [X];
+infix([H | T], I) -> [H, I | infix(T, I)].

test/hocon_tconf_tests.erl

@@ -82,7 +82,7 @@ union_with_default(_) ->
     undefined.
 
 default_value_test() ->
-    Conf = "{\"bar.field1\": \"foo\"}",
+    Conf = "{bar.field1: \"foo\"}",
     Res = check(Conf, #{format => richmap}),
     ?assertEqual(Res, check_plain(Conf)),
     ?assertEqual(
@@ -107,7 +107,7 @@ default_value_test() ->
     ).
 
 obfuscate_sensitive_values_test() ->
-    Conf = "{\"bar.field1\": \"foo\"}",
+    Conf = "{bar.field1: \"foo\"}",
     Res = check(Conf, #{format => richmap}),
     Res1 = check_plain(Conf, #{obfuscate_sensitive_values => true}),
     ?assertNotEqual(Res, Res1),
@@ -195,7 +195,7 @@ nest_ref_fill_default_test() ->
 env_override_test() ->
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\", bar.host: \"127.0.0.1\"}",
+            Conf = "{bar.field1: \"foo\", bar.host: \"127.0.0.1\"}",
             Opts = #{format => richmap},
             Res = check(Conf, Opts#{apply_override_envs => true}),
             ?assertEqual(
@@ -244,7 +244,7 @@ env_override_test() ->
 no_env_override_test() ->
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\"}",
+            Conf = "{bar.field1: \"foo\"}",
             Res = check(Conf, #{format => richmap}),
             PlainRes = check_plain(Conf, #{logger => fun(_, _) -> ok end}),
             ?assertEqual(Res, PlainRes),
@@ -270,7 +270,7 @@ unknown_env_test() ->
     Ref = make_ref(),
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\"}",
+            Conf = "{bar.field1: \"foo\"}",
             Opts = #{
                 logger => fun(Level, Msg) ->
                     Tester ! {Ref, Level, Msg},

test/hocon_tests.erl

@@ -820,7 +820,7 @@ files_unicode_path_test() ->
             "a=1\n"
             "b=2\n"
             "unicode = \"测试unicode文件路径\"\n"
-            "\"语言.英文\" = english\n"/utf8
+            "\"语言英文\" = english\n"/utf8
         >>,
     Filename =
         case file:native_name_encoding() of
@@ -834,7 +834,7 @@ files_unicode_path_test() ->
             #{format => richmap}
         ),
         ?assertEqual(<<"测试unicode文件路径"/utf8>>, deep_get("unicode", Conf, ?HOCON_V)),
-        ?assertEqual(<<"english">>, deep_get("语言.英文", Conf, ?HOCON_V))
+        ?assertEqual(<<"english">>, deep_get("语言英文", Conf, ?HOCON_V))
     after
         file:delete(Filename)
     end.
@@ -1002,3 +1002,24 @@ adjacent_maps_test_() ->
                 hocon:binary(<<"x = [{a = 1}\n {b = 2}]">>)
             )}
     ].
+
+map_with_placeholders_test() ->
+    RawConf =
+        #{
+            <<"headers">> =>
+                #{
+                    <<"fixed_key">> => <<"fixed_value">>,
+                    <<"${.payload.key}">> => <<"fixed_value">>,
+                    <<"${.payload.key}2">> => <<"${.payload.value}">>,
+                    <<"fixed_key2">> => <<"${.payload.value}">>
+                }
+        },
+    TmpFile = "/tmp/" ++ atom_to_list(?FUNCTION_NAME) ++ ".conf",
+    try
+        ok = file:write_file(TmpFile, hocon_pp:do(RawConf, #{})),
+        {ok, LoadedConf} = hocon:load(TmpFile, #{format => map}),
+        ?assertEqual(RawConf, LoadedConf),
+        ok
+    after
+        file:delete(TmpFile)
+    end.