diff --git a/.github/workflows/run_elvis.yaml b/.github/workflows/run_elvis.yaml index 22d46d6..9a56d60 100644 --- a/.github/workflows/run_elvis.yaml +++ b/.github/workflows/run_elvis.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23.2 + image: erlang:25.3 steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/run_test_case.yaml b/.github/workflows/run_test_case.yaml index 1ef5492..e841c1e 100644 --- a/.github/workflows/run_test_case.yaml +++ b/.github/workflows/run_test_case.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest container: - image: erlang:23.2 + image: erlang:25.3 steps: - uses: actions/checkout@v1 diff --git a/Makefile b/Makefile index e4bbb15..e8b1ba9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ REBAR := $(CURDIR)/rebar3 all: es $(REBAR): - @curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.14.3-emqx-7/rebar3" -o ./rebar3 + @curl -k -f -L "https://github.com/emqx/rebar3/releases/download/3.19.0-emqx-6/rebar3" -o ./rebar3 @chmod +x ./rebar3 .PHONY: compile diff --git a/rebar.config b/rebar.config index 2c4d27a..9f6880c 100644 --- a/rebar.config +++ b/rebar.config @@ -33,14 +33,14 @@ {test, [ {deps, [ {proper, "1.4.0"}, - {cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.5"}}}, + {cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.8"}}}, {erlymatch, {git, "https://github.com/zmstone/erlymatch.git", {tag, "1.1.0"}}} ]}, {extra_src_dirs, ["sample-configs", "sample-schemas", "_build/test/lib/cuttlefish/test"]}, {erl_opts, [{i, "_build/test/lib/cuttlefish/include"}]} ]}, {cuttlefish, [ - {deps, [{cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.4"}}}]}, + {deps, [{cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {tag, "v3.3.8"}}}]}, {erl_opts, [{d, 'CUTTLEFISH_CONVERTER', true}]} ]}, {es, [ diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index 5554bdc..252c979 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -2,7 +2,7 @@ set -euo pipefail -ELVIS_VERSION='1.0.0-emqx-2' +ELVIS_VERSION='1.1.0-emqx-2' elvis_version="${2:-$ELVIS_VERSION}" diff --git a/src/hocon.erl b/src/hocon.erl index 3de8b4b..7e0f42f 100644 --- a/src/hocon.erl +++ b/src/hocon.erl @@ -192,7 +192,12 @@ do_expand([Other | More], Acc) -> do_expand(More, [Other | Acc]). create_nested(#{?HOCON_T := key} = Key, Value) -> - do_create_nested(paths(value_of(Key)), Value, Key). + case value_of(Key) of + {keypath, Path} -> + do_create_nested(Path, Value, Key); + Path -> + do_create_nested([Path], Value, Key) + end. do_create_nested([], Value, _OriginalKey) -> Value; @@ -398,7 +403,7 @@ transform(#{?HOCON_T := object, ?HOCON_V := V}, Opts) -> do_transform([], Map, _Opts) -> Map; do_transform([{Key, Value} | More], Map, Opts) -> - [KeyReal] = paths(value_of(Key)), + KeyReal = unicode_bin(value_of(Key)), ValueReal = unpack(Value, Opts), do_transform(More, merge(KeyReal, ValueReal, Map), Opts). diff --git a/src/hocon_maps.erl b/src/hocon_maps.erl index 3f4e8f4..bcd6779 100644 --- a/src/hocon_maps.erl +++ b/src/hocon_maps.erl @@ -320,9 +320,8 @@ flatten_l([H | T], Opts, Meta, Stack, Acc, [Tag | Tags]) -> bin(B) when is_binary(B) -> B; bin(I) when is_integer(I) -> integer_to_binary(I). -infix([], _) -> []; -infix([X], _) -> [X]; -infix([H | T], I) -> [H, I | infix(T, I)]. +infix(List, Sep) -> + lists:join(Sep, List). ensure_plain(M) -> case is_richmap(M) of diff --git a/src/hocon_md.erl b/src/hocon_md.erl index 85ecf54..44f99bc 100644 --- a/src/hocon_md.erl +++ b/src/hocon_md.erl @@ -17,7 +17,7 @@ -module(hocon_md). -export([h/2, link/2, local_link/2, th/1, td/1, ul/1, code/1]). --export([join/1, indent/2]). +-export([indent/2]). h(1, Text) -> format("# ~s~n", [Text]); h(2, Text) -> format("## ~s~n", [Text]); @@ -49,9 +49,6 @@ escape_bar(Str) -> code(Text) -> ["", Text, ""]. -join(Mds) -> - lists:join("\n", [Mds]). - indent(N, Lines) when is_list(Lines) -> indent(N, unicode:characters_to_binary(infix(Lines, "\n"), utf8)); indent(N, Lines0) -> @@ -62,9 +59,8 @@ indent(N, Lines0) -> pad(_Pad, <<>>) -> <<>>; pad(Pad, Line) -> [Pad, Line]. -infix([], _) -> []; -infix([X], _) -> [X]; -infix([H | T], In) -> [H, In | infix(T, In)]. +infix(List, Sep) -> + lists:join(Sep, List). %% ref: https://gist.github.com/asabaylus/3071099 %% GitHub flavored markdown likes ':' being removed diff --git a/src/hocon_parser.yrl b/src/hocon_parser.yrl index 5c238b8..e399f00 100644 --- a/src/hocon_parser.yrl +++ b/src/hocon_parser.yrl @@ -12,7 +12,7 @@ Terminals '{' '}' '[' ']' ',' bool integer float null percent bytesize duration - string variable + unqstr string variable endstr endvar endarr endobj include key required. @@ -29,6 +29,7 @@ partials -> '[' elements endarr : [make_array(line_of('$1'), '$2')]. partials -> '{' endobj : [make_object(line_of('$1'), [])]. partials -> '[' endarr : [make_array(line_of('$1'), [])]. +partial -> unqstr : str_to_bin(make_primitive_value('$1')). partial -> string : str_to_bin(make_primitive_value('$1')). partial -> variable : make_variable('$1'). partial -> '{' fields '}' : make_object(line_of('$1'), '$2'). @@ -47,8 +48,10 @@ elements -> value ',' elements : ['$1' | '$3']. elements -> value elements : ['$1' | '$2']. elements -> value : ['$1']. +directive -> include unqstr : make_include('$2', false). directive -> include string : make_include('$2', false). directive -> include endstr : make_include('$2', false). +directive -> include required unqstr : make_include('$3', true). directive -> include required string : make_include('$3', true). directive -> include required endstr : make_include('$3', true). @@ -88,7 +91,7 @@ make_include(String, false) -> #{'$hcTyp' => include, make_concat(S) -> #{'$hcTyp' => concat, '$hcVal' => S}. -str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string -> M#{'$hcVal' => bin(V)}. +str_to_bin(#{'$hcTyp' := T, '$hcVal' := V} = M) when T =:= string orelse T =:= unqstr -> M#{'$hcTyp' := string, '$hcVal' => bin(V)}. line_of(Token) -> element(2, Token). value_of(Token) -> element(3, Token). diff --git a/src/hocon_pp.erl b/src/hocon_pp.erl index ca1393a..290ee3e 100644 --- a/src/hocon_pp.erl +++ b/src/hocon_pp.erl @@ -176,16 +176,16 @@ is_quote_key(K) -> true end. +%% Return 'true' if a string is to be quoted when formatted as HOCON. %% A sequence of characters outside of a quoted string is a string value if: %% it does not contain "forbidden characters": %% '$', '"', '{', '}', '[', ']', ':', '=', ',', '+', '#', '`', '^', '?', '!', '@', '*', %% '&', '' (backslash), or whitespace. %% '$"{}[]:=,+#`^?!@*& \\' - -is_quote_str(S) -> +is_to_quote_str(S) -> case hocon_scanner:string(S) of - {ok, [{string, 1, S}], 1} -> - %% contain $"{}[]:=,+#`^?!@*& \\ should be quote + {ok, [{Tag, 1, S}], 1} when Tag =:= string orelse Tag =:= unqstr -> + %% contain $"{}[]:=,+#`^?!@*& \\ should be quoted case re:run(S, "^[^$\"{}\\[\\]:=,+#`\\^?!@*&\\ \\\\]*$") of nomatch -> true; _ -> false @@ -195,7 +195,7 @@ is_quote_str(S) -> end. maybe_quote_latin1_str(S) -> - case is_quote_str(S) of + case is_to_quote_str(S) of true -> bin(io_lib:format("~0p", [S])); false -> S end. @@ -219,9 +219,8 @@ fmt({indent, Block}) -> split(Bin) -> [Line || Line <- binary:split(Bin, ?NL, [global]), Line =/= <<>>]. -infix([], _) -> []; -infix([One], _) -> [One]; -infix([H | T], Infix) -> [[H, Infix] | infix(T, Infix)]. +infix(List, Sep) -> + lists:join(Sep, List). format_escape_sequences(Str) -> bin(lists:map(fun esc/1, Str)). diff --git a/src/hocon_scanner.xrl b/src/hocon_scanner.xrl index 72e8d9a..8f15752 100644 --- a/src/hocon_scanner.xrl +++ b/src/hocon_scanner.xrl @@ -93,7 +93,7 @@ Rules. Erlang code. maybe_include("include", TokenLine) -> {include, TokenLine}; -maybe_include(TokenChars, TokenLine) -> {string, TokenLine, TokenChars}. +maybe_include(TokenChars, TokenLine) -> {unqstr, TokenLine, TokenChars}. get_filename_from_required("required(" ++ Filename) -> [$) | FilenameRev] = lists:reverse(Filename), diff --git a/src/hocon_token.erl b/src/hocon_token.erl index b95dccb..a370238 100644 --- a/src/hocon_token.erl +++ b/src/hocon_token.erl @@ -109,6 +109,8 @@ trans_key([{'{', Line} | Tokens], Acc) -> trans_key([T | Tokens], Acc) -> trans_key(Tokens, [T | Acc]). +trans_key_lb([{unqstr, Line, Value} | TokensRev]) -> + [{key, Line, {keypath, paths(Value)}} | TokensRev]; trans_key_lb([{string, Line, Value} | TokensRev]) -> [{key, Line, Value} | TokensRev]; trans_key_lb(Otherwise) -> @@ -145,6 +147,7 @@ trans_splice_end([], Seq, Acc) -> lists:reverse(NewAcc). do_trans_splice_end([]) -> []; +do_trans_splice_end([{unqstr, Line, Value} | T]) -> [{endstr, Line, Value} | T]; do_trans_splice_end([{string, Line, Value} | T]) -> [{endstr, Line, Value} | T]; do_trans_splice_end([{variable, Line, Value} | T]) -> [{endvar, Line, Value} | T]; do_trans_splice_end([{'}', Line} | T]) -> [{endobj, Line} | T]; @@ -225,7 +228,12 @@ abspath(Var, PathStack) -> do_abspath(Var, ['$root']) -> Var; do_abspath(Var, [#{?HOCON_T := key} = K | More]) -> - do_abspath(unicode_bin([value_of(K), <<".">>, Var]), More). + do_abspath(unicode_bin([maybe_join(value_of(K)), <<".">>, Var]), More). + +maybe_join({keypath, Path}) -> + infix(Path, "."); +maybe_join(Path) -> + Path. -spec load_include(boxed(), hocon:ctx()) -> boxed() | nothing. @@ -308,3 +316,11 @@ format_error(Line, ErrorInfo, Ctx) -> unicode_bin(L) -> unicode:characters_to_binary(L, utf8). unicode_list(B) -> unicode:characters_to_list(B, utf8). + +paths(Key) when is_binary(Key) -> + paths(unicode:characters_to_list(Key, utf8)); +paths(Key) when is_list(Key) -> + lists:map(fun unicode_bin/1, string:tokens(Key, ".")). + +infix(List, Sep) -> + lists:join(Sep, List). diff --git a/test/hocon_tconf_tests.erl b/test/hocon_tconf_tests.erl index 54099d0..fb45e52 100644 --- a/test/hocon_tconf_tests.erl +++ b/test/hocon_tconf_tests.erl @@ -82,7 +82,7 @@ union_with_default(_) -> undefined. default_value_test() -> - Conf = "{\"bar.field1\": \"foo\"}", + Conf = "{bar.field1: \"foo\"}", Res = check(Conf, #{format => richmap}), ?assertEqual(Res, check_plain(Conf)), ?assertEqual( @@ -107,7 +107,7 @@ default_value_test() -> ). obfuscate_sensitive_values_test() -> - Conf = "{\"bar.field1\": \"foo\"}", + Conf = "{bar.field1: \"foo\"}", Res = check(Conf, #{format => richmap}), Res1 = check_plain(Conf, #{obfuscate_sensitive_values => true}), ?assertNotEqual(Res, Res1), @@ -195,7 +195,7 @@ nest_ref_fill_default_test() -> env_override_test() -> with_envs( fun() -> - Conf = "{\"bar.field1\": \"foo\", bar.host: \"127.0.0.1\"}", + Conf = "{bar.field1: \"foo\", bar.host: \"127.0.0.1\"}", Opts = #{format => richmap}, Res = check(Conf, Opts#{apply_override_envs => true}), ?assertEqual( @@ -244,7 +244,7 @@ env_override_test() -> no_env_override_test() -> with_envs( fun() -> - Conf = "{\"bar.field1\": \"foo\"}", + Conf = "{bar.field1: \"foo\"}", Res = check(Conf, #{format => richmap}), PlainRes = check_plain(Conf, #{logger => fun(_, _) -> ok end}), ?assertEqual(Res, PlainRes), @@ -270,7 +270,7 @@ unknown_env_test() -> Ref = make_ref(), with_envs( fun() -> - Conf = "{\"bar.field1\": \"foo\"}", + Conf = "{bar.field1: \"foo\"}", Opts = #{ logger => fun(Level, Msg) -> Tester ! {Ref, Level, Msg}, diff --git a/test/hocon_tests.erl b/test/hocon_tests.erl index 1c3c5a5..d5d97e2 100644 --- a/test/hocon_tests.erl +++ b/test/hocon_tests.erl @@ -820,7 +820,7 @@ files_unicode_path_test() -> "a=1\n" "b=2\n" "unicode = \"测试unicode文件路径\"\n" - "\"语言.英文\" = english\n"/utf8 + "\"语言英文\" = english\n"/utf8 >>, Filename = case file:native_name_encoding() of @@ -834,7 +834,7 @@ files_unicode_path_test() -> #{format => richmap} ), ?assertEqual(<<"测试unicode文件路径"/utf8>>, deep_get("unicode", Conf, ?HOCON_V)), - ?assertEqual(<<"english">>, deep_get("语言.英文", Conf, ?HOCON_V)) + ?assertEqual(<<"english">>, deep_get("语言英文", Conf, ?HOCON_V)) after file:delete(Filename) end. @@ -1002,3 +1002,24 @@ adjacent_maps_test_() -> hocon:binary(<<"x = [{a = 1}\n {b = 2}]">>) )} ]. + +map_with_placeholders_test() -> + RawConf = + #{ + <<"headers">> => + #{ + <<"fixed_key">> => <<"fixed_value">>, + <<"${.payload.key}">> => <<"fixed_value">>, + <<"${.payload.key}2">> => <<"${.payload.value}">>, + <<"fixed_key2">> => <<"${.payload.value}">> + } + }, + TmpFile = "/tmp/" ++ atom_to_list(?FUNCTION_NAME) ++ ".conf", + try + ok = file:write_file(TmpFile, hocon_pp:do(RawConf, #{})), + {ok, LoadedConf} = hocon:load(TmpFile, #{format => map}), + ?assertEqual(RawConf, LoadedConf), + ok + after + file:delete(TmpFile) + end.