Merge pull request #270 from emqx/do-not-split-dots-in-quoted-keys

fix: do not split quoted key as path

Author: zmstone

.github/workflows/run_elvis.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: erlang:23.2
+      image: erlang:25.3
 
     steps:
       - uses: actions/checkout@v1

.github/workflows/run_test_case.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: erlang:23.2
+      image: erlang:25.3
 
     steps:
       - uses: actions/checkout@v1

Makefile

@@ -4,7 +4,7 @@ REBAR := $(CURDIR)/rebar3
 all: es
 
 $(REBAR):
-	@curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.14.3-emqx-7/rebar3" -o ./rebar3
+	@curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.19.0-emqx-6/rebar3" -o ./rebar3
 	@chmod +x ./rebar3
 
 .PHONY: compile

rebar.config

@@ -33,14 +33,14 @@
     {test, [
         {deps, [
             {proper, "1.4.0"},
-            {cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.5"}}},
+            {cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.8"}}},
             {erlymatch, {git, "https://github.com/zmstone/erlymatch.git", {tag, "1.1.0"}}}
         ]},
         {extra_src_dirs, ["sample-configs", "sample-schemas", "_build/test/lib/cuttlefish/test"]},
         {erl_opts, [{i, "_build/test/lib/cuttlefish/include"}]}
     ]},
     {cuttlefish, [
-        {deps, [{cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.4"}}}]},
+        {deps, [{cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.8"}}}]},
         {erl_opts, [{d, 'CUTTLEFISH_CONVERTER', true}]}
     ]},
     {es, [

scripts/elvis-check.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-ELVIS_VERSION='1.0.0-emqx-2'
+ELVIS_VERSION='1.1.0-emqx-2'
 
 elvis_version="${2:-$ELVIS_VERSION}"
 

src/hocon.erl

@@ -192,7 +192,12 @@ do_expand([Other | More], Acc) ->
     do_expand(More, [Other | Acc]).
 
 create_nested(#{?HOCON_T := key} = Key, Value) ->
-    do_create_nested(paths(value_of(Key)), Value, Key).
+    case value_of(Key) of
+        {keypath, Path} ->
+            do_create_nested(Path, Value, Key);
+        Path ->
+            do_create_nested([Path], Value, Key)
+    end.
 
 do_create_nested([], Value, _OriginalKey) ->
     Value;
@@ -398,7 +403,7 @@ transform(#{?HOCON_T := object, ?HOCON_V := V}, Opts) ->
 do_transform([], Map, _Opts) ->
     Map;
 do_transform([{Key, Value} | More], Map, Opts) ->
-    [KeyReal] = paths(value_of(Key)),
+    KeyReal = unicode_bin(value_of(Key)),
     ValueReal = unpack(Value, Opts),
     do_transform(More, merge(KeyReal, ValueReal, Map), Opts).
 

src/hocon_maps.erl

@@ -320,9 +320,8 @@ flatten_l([H | T], Opts, Meta, Stack, Acc, [Tag | Tags]) ->
 bin(B) when is_binary(B) -> B;
 bin(I) when is_integer(I) -> integer_to_binary(I).
 
-infix([], _) -> [];
-infix([X], _) -> [X];
-infix([H | T], I) -> [H, I | infix(T, I)].
+infix(List, Sep) ->
+    lists:join(Sep, List).
 
 ensure_plain(M) ->
     case is_richmap(M) of

src/hocon_md.erl

@@ -17,7 +17,7 @@
 -module(hocon_md).
 
 -export([h/2, link/2, local_link/2, th/1, td/1, ul/1, code/1]).
--export([join/1, indent/2]).
+-export([indent/2]).
 
 h(1, Text) -> format("# ~s~n", [Text]);
 h(2, Text) -> format("## ~s~n", [Text]);
@@ -49,9 +49,6 @@ escape_bar(Str) ->
 
 code(Text) -> ["<code>", Text, "</code>"].
 
-join(Mds) ->
-    lists:join("\n", [Mds]).
-
 indent(N, Lines) when is_list(Lines) ->
     indent(N, unicode:characters_to_binary(infix(Lines, "\n"), utf8));
 indent(N, Lines0) ->
@@ -62,9 +59,8 @@ indent(N, Lines0) ->
 pad(_Pad, <<>>) -> <<>>;
 pad(Pad, Line) -> [Pad, Line].
 
-infix([], _) -> [];
-infix([X], _) -> [X];
-infix([H | T], In) -> [H, In | infix(T, In)].
+infix(List, Sep) ->
+    lists:join(Sep, List).
 
 %% ref: https://gist.github.com/asabaylus/3071099
 %% GitHub flavored markdown likes ':' being removed

src/hocon_parser.yrl

@@ -12,7 +12,7 @@ Terminals
   '{' '}' '[' ']' ','
   bool integer float null
   percent bytesize duration
-  string variable
+  unqstr string variable
   endstr endvar endarr endobj
   include key required.
 
@@ -29,6 +29,7 @@ partials -> '[' elements endarr : [make_array(line_of('$1'), '$2')].
 partials -> '{' endobj : [make_object(line_of('$1'), [])].
 partials -> '[' endarr : [make_array(line_of('$1'), [])].
 
+partial -> unqstr : str_to_bin(make_primitive_value('$1')).
 partial -> string : str_to_bin(make_primitive_value('$1')).
 partial -> variable : make_variable('$1').
 partial -> '{' fields '}' : make_object(line_of('$1'), '$2').
@@ -47,8 +48,10 @@ elements -> value ',' elements : ['$1' | '$3'].
 elements -> value elements : ['$1' | '$2'].
 elements -> value : ['$1'].
 
+directive -> include unqstr : make_include('$2', false).
 directive -> include string : make_include('$2', false).
 directive -> include endstr : make_include('$2', false).
+directive -> include required unqstr : make_include('$3', true).
 directive -> include required string : make_include('$3', true).
 directive -> include required endstr : make_include('$3', true).
 
@@ -88,7 +91,7 @@ make_include(String, false) ->  #{'$hcTyp' => include,
 
 make_concat(S) -> #{'$hcTyp' => concat, '$hcVal' => S}.
 
-str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string -> M#{'$hcVal' => bin(V)}.
+str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string orelse T =:= unqstr -> M#{'$hcTyp' := string, '$hcVal' => bin(V)}.
 
 line_of(Token) -> element(2, Token).
 value_of(Token) -> element(3, Token).

src/hocon_pp.erl

@@ -176,16 +176,16 @@ is_quote_key(K) ->
             true
     end.
 
+%% Return 'true' if a string is to be quoted when formatted as HOCON.
 %% A sequence of characters outside of a quoted string is a string value if:
 %% it does not contain "forbidden characters":
 %% '$', '"', '{', '}', '[', ']', ':', '=', ',', '+', '#', '`', '^', '?', '!', '@', '*',
 %% '&', '' (backslash), or whitespace.
 %% '$"{}[]:=,+#`^?!@*& \\'
-
-is_quote_str(S) ->
+is_to_quote_str(S) ->
     case hocon_scanner:string(S) of
-        {ok, [{string, 1, S}], 1} ->
-            %% contain $"{}[]:=,+#`^?!@*& \\ should be quote
+        {ok, [{Tag, 1, S}], 1} when Tag =:= string orelse Tag =:= unqstr ->
+            %% contain $"{}[]:=,+#`^?!@*& \\ should be quoted
             case re:run(S, "^[^$\"{}\\[\\]:=,+#`\\^?!@*&\\ \\\\]*$") of
                 nomatch -> true;
                 _ -> false
@@ -195,7 +195,7 @@ is_quote_str(S) ->
     end.
 
 maybe_quote_latin1_str(S) ->
-    case is_quote_str(S) of
+    case is_to_quote_str(S) of
         true -> bin(io_lib:format("~0p", [S]));
         false -> S
     end.
@@ -219,9 +219,8 @@ fmt({indent, Block}) ->
 split(Bin) ->
     [Line || Line <- binary:split(Bin, ?NL, [global]), Line =/= <<>>].
 
-infix([], _) -> [];
-infix([One], _) -> [One];
-infix([H | T], Infix) -> [[H, Infix] | infix(T, Infix)].
+infix(List, Sep) ->
+    lists:join(Sep, List).
 
 format_escape_sequences(Str) ->
     bin(lists:map(fun esc/1, Str)).

src/hocon_scanner.xrl

@@ -93,7 +93,7 @@ Rules.
 Erlang code.
 
 maybe_include("include", TokenLine)  -> {include, TokenLine};
-maybe_include(TokenChars, TokenLine) -> {string, TokenLine, TokenChars}.
+maybe_include(TokenChars, TokenLine) -> {unqstr, TokenLine, TokenChars}.
 
 get_filename_from_required("required(" ++ Filename) ->
     [$) | FilenameRev] = lists:reverse(Filename),

src/hocon_token.erl

@@ -109,6 +109,8 @@ trans_key([{'{', Line} | Tokens], Acc) ->
 trans_key([T | Tokens], Acc) ->
     trans_key(Tokens, [T | Acc]).
 
+trans_key_lb([{unqstr, Line, Value} | TokensRev]) ->
+    [{key, Line, {keypath, paths(Value)}} | TokensRev];
 trans_key_lb([{string, Line, Value} | TokensRev]) ->
     [{key, Line, Value} | TokensRev];
 trans_key_lb(Otherwise) ->
@@ -145,6 +147,7 @@ trans_splice_end([], Seq, Acc) ->
     lists:reverse(NewAcc).
 
 do_trans_splice_end([]) -> [];
+do_trans_splice_end([{unqstr, Line, Value} | T]) -> [{endstr, Line, Value} | T];
 do_trans_splice_end([{string, Line, Value} | T]) -> [{endstr, Line, Value} | T];
 do_trans_splice_end([{variable, Line, Value} | T]) -> [{endvar, Line, Value} | T];
 do_trans_splice_end([{'}', Line} | T]) -> [{endobj, Line} | T];
@@ -225,7 +228,12 @@ abspath(Var, PathStack) ->
 do_abspath(Var, ['$root']) ->
     Var;
 do_abspath(Var, [#{?HOCON_T := key} = K | More]) ->
-    do_abspath(unicode_bin([value_of(K), <<".">>, Var]), More).
+    do_abspath(unicode_bin([maybe_join(value_of(K)), <<".">>, Var]), More).
+
+maybe_join({keypath, Path}) ->
+    infix(Path, ".");
+maybe_join(Path) ->
+    Path.
 
 -spec load_include(boxed(), hocon:ctx()) -> boxed() | nothing.
 
@@ -308,3 +316,11 @@ format_error(Line, ErrorInfo, Ctx) ->
 
 unicode_bin(L) -> unicode:characters_to_binary(L, utf8).
 unicode_list(B) -> unicode:characters_to_list(B, utf8).
+
+paths(Key) when is_binary(Key) ->
+    paths(unicode:characters_to_list(Key, utf8));
+paths(Key) when is_list(Key) ->
+    lists:map(fun unicode_bin/1, string:tokens(Key, ".")).
+
+infix(List, Sep) ->
+    lists:join(Sep, List).

test/hocon_tconf_tests.erl

@@ -82,7 +82,7 @@ union_with_default(_) ->
     undefined.
 
 default_value_test() ->
-    Conf = "{\"bar.field1\": \"foo\"}",
+    Conf = "{bar.field1: \"foo\"}",
     Res = check(Conf, #{format => richmap}),
     ?assertEqual(Res, check_plain(Conf)),
     ?assertEqual(
@@ -107,7 +107,7 @@ default_value_test() ->
     ).
 
 obfuscate_sensitive_values_test() ->
-    Conf = "{\"bar.field1\": \"foo\"}",
+    Conf = "{bar.field1: \"foo\"}",
     Res = check(Conf, #{format => richmap}),
     Res1 = check_plain(Conf, #{obfuscate_sensitive_values => true}),
     ?assertNotEqual(Res, Res1),
@@ -195,7 +195,7 @@ nest_ref_fill_default_test() ->
 env_override_test() ->
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\", bar.host: \"127.0.0.1\"}",
+            Conf = "{bar.field1: \"foo\", bar.host: \"127.0.0.1\"}",
             Opts = #{format => richmap},
             Res = check(Conf, Opts#{apply_override_envs => true}),
             ?assertEqual(
@@ -244,7 +244,7 @@ env_override_test() ->
 no_env_override_test() ->
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\"}",
+            Conf = "{bar.field1: \"foo\"}",
             Res = check(Conf, #{format => richmap}),
             PlainRes = check_plain(Conf, #{logger => fun(_, _) -> ok end}),
             ?assertEqual(Res, PlainRes),
@@ -270,7 +270,7 @@ unknown_env_test() ->
     Ref = make_ref(),
     with_envs(
         fun() ->
-            Conf = "{\"bar.field1\": \"foo\"}",
+            Conf = "{bar.field1: \"foo\"}",
             Opts = #{
                 logger => fun(Level, Msg) ->
                     Tester ! {Ref, Level, Msg},

test/hocon_tests.erl

@@ -820,7 +820,7 @@ files_unicode_path_test() ->
             "a=1\n"
             "b=2\n"
             "unicode = \"测试unicode文件路径\"\n"
-            "\"语言.英文\" = english\n"/utf8
+            "\"语言英文\" = english\n"/utf8
         >>,
     Filename =
         case file:native_name_encoding() of
@@ -834,7 +834,7 @@ files_unicode_path_test() ->
             #{format => richmap}
         ),
         ?assertEqual(<<"测试unicode文件路径"/utf8>>, deep_get("unicode", Conf, ?HOCON_V)),
-        ?assertEqual(<<"english">>, deep_get("语言.英文", Conf, ?HOCON_V))
+        ?assertEqual(<<"english">>, deep_get("语言英文", Conf, ?HOCON_V))
     after
         file:delete(Filename)
     end.
@@ -1002,3 +1002,24 @@ adjacent_maps_test_() ->
                 hocon:binary(<<"x = [{a = 1}\n {b = 2}]">>)
             )}
     ].
+
+map_with_placeholders_test() ->
+    RawConf =
+        #{
+            <<"headers">> =>
+                #{
+                    <<"fixed_key">> => <<"fixed_value">>,
+                    <<"${.payload.key}">> => <<"fixed_value">>,
+                    <<"${.payload.key}2">> => <<"${.payload.value}">>,
+                    <<"fixed_key2">> => <<"${.payload.value}">>
+                }
+        },
+    TmpFile = "/tmp/" ++ atom_to_list(?FUNCTION_NAME) ++ ".conf",
+    try
+        ok = file:write_file(TmpFile, hocon_pp:do(RawConf, #{})),
+        {ok, LoadedConf} = hocon:load(TmpFile, #{format => map}),
+        ?assertEqual(RawConf, LoadedConf),
+        ok
+    after
+        file:delete(TmpFile)
+    end.