Skip to content

Commit cdda9d1

Browse files
LostRuinsLostRuins
committed
Merge branch 'upstream' into concedo_experimental
# Conflicts: # .devops/tools.sh # build-xcframework.sh # ci/run.sh # examples/Miku.sh # examples/chat-13B.sh # examples/chat-persistent.sh # examples/chat-vicuna.sh # examples/chat.sh # examples/jeopardy/jeopardy.sh # examples/reason-act.sh # examples/server-llama2-13B.sh # examples/sycl/build.sh # examples/sycl/run-llama2.sh # examples/sycl/run-llama3.sh # examples/ts-type-to-grammar.sh # ggml/src/ggml-cpu/CMakeLists.txt # ggml/src/ggml-sycl/element_wise.cpp # ggml/src/ggml-sycl/element_wise.hpp # ggml/src/ggml-sycl/ggml-sycl.cpp # scripts/apple/validate-apps.sh # scripts/apple/validate-ios.sh # scripts/apple/validate-macos.sh # scripts/apple/validate-tvos.sh # scripts/apple/validate-visionos.sh # scripts/check-requirements.sh # scripts/ci-run.sh # scripts/compare-commits.sh # scripts/debug-test.sh # scripts/gen-authors.sh # scripts/get-hellaswag.sh # scripts/get-pg.sh # scripts/get-wikitext-103.sh # scripts/get-wikitext-2.sh # scripts/get-winogrande.sh # scripts/hf.sh # scripts/qnt-all.sh # scripts/run-all-perf.sh # scripts/run-all-ppl.sh # scripts/sync-ggml-am.sh # scripts/sync-ggml.sh # scripts/tool_bench.sh # tests/test-backend-ops.cpp # tests/test-lora-conversion-inference.sh # tests/test-tokenizer-0.sh # tools/server/README.md
2 parents 989f9e6 + eb3fa29 commit cdda9d1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1517
-116
lines changed

‎common/arg.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2796,6 +2796,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
27962796
params.ssl_file_cert = value;
27972797
}
27982798
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SSL_CERT_FILE"));
2799+
add_opt(common_arg(
2800+
{"--chat-template-kwargs"}, "STRING",
2801+
string_format("sets additional params for the json template parser"),
2802+
[](common_params & params, const std::string & value) {
2803+
auto parsed = json::parse(value);
2804+
for (const auto & item : parsed.items()) {
2805+
params.default_template_kwargs[item.key()] = item.value().dump();
2806+
}
2807+
}
2808+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_CHAT_TEMPLATE_KWARGS"));
27992809
add_opt(common_arg(
28002810
{"-to", "--timeout"}, "N",
28012811
string_format("server read/write timeout in seconds (default: %d)", params.timeout_read),

‎common/chat.cpp‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include <string>
1818
#include <vector>
1919

20+
using json = nlohmann::ordered_json;
21+
2022
static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {
2123
auto time = std::chrono::system_clock::to_time_t(now);
2224
auto local_time = *std::localtime(&time);
@@ -140,6 +142,7 @@ struct templates_params {
140142
bool add_generation_prompt = true;
141143
bool enable_thinking = true;
142144
std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
145+
json extra_context;
143146
};
144147

145148
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -720,16 +723,23 @@ static void foreach_function(const json & tools, const std::function<void(const
720723

721724
static std::string apply(
722725
const common_chat_template & tmpl,
723-
const nlohmann::ordered_json & messages,
724-
const nlohmann::ordered_json & tools,
725-
bool add_generation_prompt,
726-
const nlohmann::ordered_json & extra_context = nlohmann::ordered_json())
726+
const struct templates_params & inputs,
727+
const std::optional<json> & messages_override = std::nullopt,
728+
const std::optional<json> & tools_override = std::nullopt,
729+
const std::optional<json> & additional_context = std::nullopt)
727730
{
728731
minja::chat_template_inputs tmpl_inputs;
729-
tmpl_inputs.messages = messages;
730-
tmpl_inputs.tools = tools;
731-
tmpl_inputs.add_generation_prompt = add_generation_prompt;
732-
tmpl_inputs.extra_context = extra_context;
732+
tmpl_inputs.messages = messages_override ? *messages_override : inputs.messages;
733+
if (tools_override) {
734+
tmpl_inputs.tools = *tools_override;
735+
} else {
736+
tmpl_inputs.tools = inputs.tools.empty() ? json() : inputs.tools;
737+
}
738+
tmpl_inputs.add_generation_prompt = inputs.add_generation_prompt;
739+
tmpl_inputs.extra_context = inputs.extra_context;
740+
if (additional_context) {
741+
tmpl_inputs.extra_context.merge_patch(*additional_context);
742+
}
733743
// TODO: add flag to control date/time, if only for testing purposes.
734744
// tmpl_inputs.now = std::chrono::system_clock::now();
735745

@@ -828,7 +838,7 @@ static common_chat_params common_chat_params_init_generic(const common_chat_temp
828838
inputs.messages,
829839
"Respond in JSON format, either with `tool_call` (a request to call tools) or with `response` reply to the user's request");
830840

831-
data.prompt = apply(tmpl, tweaked_messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
841+
data.prompt = apply(tmpl, inputs, /* messages_override= */ tweaked_messages);
832842
data.format = COMMON_CHAT_FORMAT_GENERIC;
833843
return data;
834844
}
@@ -904,7 +914,7 @@ static common_chat_params common_chat_params_init_mistral_nemo(const common_chat
904914
data.preserved_tokens = {
905915
"[TOOL_CALLS]",
906916
};
907-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
917+
data.prompt = apply(tmpl, inputs);
908918
data.format = COMMON_CHAT_FORMAT_MISTRAL_NEMO;
909919
return data;
910920
}
@@ -934,7 +944,7 @@ static common_chat_params common_chat_params_init_command_r7b(const common_chat_
934944
adjusted_messages.push_back(msg);
935945
}
936946
}
937-
data.prompt = apply(tmpl, adjusted_messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, {});
947+
data.prompt = apply(tmpl, inputs, /* messages_override= */ adjusted_messages);
938948
data.format = COMMON_CHAT_FORMAT_COMMAND_R7B;
939949
if (string_ends_with(data.prompt, "<|START_THINKING|>")) {
940950
if (!inputs.enable_thinking) {
@@ -1122,7 +1132,7 @@ static common_chat_params common_chat_params_init_llama_3_x(const common_chat_te
11221132
} else {
11231133
data.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
11241134
}
1125-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, {
1135+
data.prompt = apply(tmpl, inputs, /* messages_override =*/ std::nullopt, /* tools_override= */ std::nullopt, json {
11261136
{"date_string", format_time(inputs.now, "%d %b %Y")},
11271137
{"tools_in_user_message", false},
11281138
{"builtin_tools", builtin_tools.empty() ? json() : builtin_tools},
@@ -1187,7 +1197,7 @@ static void common_chat_parse_llama_3_1(common_chat_msg_parser & builder, bool w
11871197

11881198
static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_template & tmpl, const struct templates_params & inputs) {
11891199
common_chat_params data;
1190-
auto prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
1200+
auto prompt = apply(tmpl, inputs);
11911201

11921202
// Hacks to fix the official (broken) prompt.
11931203
// It is advisable to use --chat-template-file models/templates/llama-cpp-deepseek-r1.jinja instead,
@@ -1282,7 +1292,7 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) {
12821292
static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) {
12831293
LOG_DBG("%s\n", __func__);
12841294
common_chat_params data;
1285-
data.prompt = apply(tmpl, inputs.messages, /* tools= */ nullptr, inputs.add_generation_prompt, {
1295+
data.prompt = apply(tmpl, inputs, /* messages_override =*/ std::nullopt, /* tools_override= */ json(), json {
12861296
{"datetime", format_time(inputs.now, "%b %d %Y %H:%M:%S GMT")},
12871297
{"functions", json(inputs.tools.empty() ? "" : inputs.tools.dump(2))},
12881298
});
@@ -1338,7 +1348,7 @@ static common_chat_params common_chat_params_init_functionary_v3_2(const common_
13381348
// Using ">>>f1\n", ">>>f2\n"... as trigger words for the grammar
13391349
// If the function is python, we also allow raw python code (if the line after `python\n` doesn't start w/ opening `{`), which the model seems to prefer for multiline code.
13401350
common_chat_params data;
1341-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
1351+
data.prompt = apply(tmpl, inputs);
13421352
data.format = COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2;
13431353
if (inputs.tools.is_array() && !inputs.tools.empty()) {
13441354
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
@@ -1465,7 +1475,7 @@ static common_chat_params common_chat_params_init_functionary_v3_1_llama_3_1(con
14651475
data.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
14661476
}
14671477

1468-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
1478+
data.prompt = apply(tmpl, inputs);
14691479
// TODO: if (has_raw_python)
14701480
return data;
14711481
}
@@ -1498,14 +1508,15 @@ static void common_chat_parse_functionary_v3_1_llama_3_1(common_chat_msg_parser
14981508
static common_chat_params common_chat_params_init_hermes_2_pro(const common_chat_template & tmpl, const struct templates_params & inputs) {
14991509
common_chat_params data;
15001510

1501-
json additional_context = {
1511+
json extra_context = json {
15021512
{"enable_thinking", inputs.enable_thinking},
15031513
};
1514+
extra_context.update(inputs.extra_context);
15041515

1505-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, additional_context);
1516+
data.prompt = apply(tmpl, inputs, /* messages_override =*/ std::nullopt, /* tools_override= */ std::nullopt, extra_context);
15061517
data.format = COMMON_CHAT_FORMAT_HERMES_2_PRO;
15071518
if (string_ends_with(data.prompt, "<think>\n")) {
1508-
if (!inputs.enable_thinking) {
1519+
if (!extra_context["enable_thinking"]) {
15091520
data.prompt += "</think>";
15101521
} else {
15111522
data.thinking_forced_open = true;
@@ -1691,7 +1702,7 @@ static void common_chat_parse_hermes_2_pro(common_chat_msg_parser & builder) {
16911702

16921703
static common_chat_params common_chat_params_init_without_tools(const common_chat_template & tmpl, const struct templates_params & inputs) {
16931704
common_chat_params data;
1694-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
1705+
data.prompt = apply(tmpl, inputs);
16951706
data.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
16961707
data.grammar_lazy = false;
16971708
if (!inputs.json_schema.is_null()) {
@@ -1722,6 +1733,12 @@ static common_chat_params common_chat_templates_apply_jinja(
17221733
params.enable_thinking = inputs.enable_thinking;
17231734
params.grammar = inputs.grammar;
17241735
params.now = inputs.now;
1736+
1737+
params.extra_context = json::object();
1738+
for (auto el : inputs.chat_template_kwargs) {
1739+
params.extra_context[el.first] = json::parse(el.second);
1740+
}
1741+
17251742
if (!inputs.json_schema.empty()) {
17261743
params.json_schema = json::parse(inputs.json_schema);
17271744
}

‎common/chat.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <chrono>
88
#include <string>
99
#include <vector>
10+
#include <map>
1011

1112
struct common_chat_templates;
1213

@@ -125,6 +126,7 @@ struct common_chat_templates_inputs {
125126
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE;
126127
bool enable_thinking = true;
127128
std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
129+
std::map<std::string, std::string> chat_template_kwargs;
128130
};
129131

130132
struct common_chat_params {

‎common/common.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <string>
99
#include <string_view>
1010
#include <vector>
11+
#include <map>
1112
#include <sstream>
1213

1314
#ifdef _WIN32
@@ -377,6 +378,8 @@ struct common_params {
377378
std::string ssl_file_key = ""; // NOLINT
378379
std::string ssl_file_cert = ""; // NOLINT
379380

381+
std::map<std::string, std::string> default_template_kwargs;
382+
380383
// "advanced" endpoints are disabled by default for better security
381384
bool webui = true;
382385
bool endpoint_slots = false;

‎ggml/include/ggml-backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ extern "C" {
339339
typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
340340

341341
// Compare the output of two backends
342-
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
342+
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor * test_node);
343343

344344
// Tensor initialization
345345
GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);

‎ggml/include/ggml.h‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,8 @@ extern "C" {
526526
GGML_OP_CROSS_ENTROPY_LOSS_BACK,
527527
GGML_OP_OPT_STEP_ADAMW,
528528

529+
GGML_OP_GLU,
530+
529531
GGML_OP_COUNT,
530532
};
531533

@@ -549,6 +551,14 @@ extern "C" {
549551
GGML_UNARY_OP_COUNT,
550552
};
551553

554+
enum ggml_glu_op {
555+
GGML_GLU_OP_REGLU,
556+
GGML_GLU_OP_GEGLU,
557+
GGML_GLU_OP_SWIGLU,
558+
559+
GGML_GLU_OP_COUNT,
560+
};
561+
552562
enum ggml_object_type {
553563
GGML_OBJECT_TYPE_TENSOR,
554564
GGML_OBJECT_TYPE_GRAPH,
@@ -671,6 +681,7 @@ extern "C" {
671681
GGML_API const char * ggml_op_symbol(enum ggml_op op);
672682

673683
GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
684+
GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);
674685
GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
675686

676687
GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);
@@ -775,6 +786,7 @@ extern "C" {
775786
GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
776787

777788
GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
789+
GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);
778790

779791
GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
780792
GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
@@ -1103,6 +1115,63 @@ extern "C" {
11031115
struct ggml_context * ctx,
11041116
struct ggml_tensor * a);
11051117

1118+
// gated linear unit ops
1119+
// A: n columns, r rows,
1120+
// result is n / 2 columns, r rows,
1121+
// expects gate in second half of row, unless swapped is true
1122+
GGML_API struct ggml_tensor * ggml_glu(
1123+
struct ggml_context * ctx,
1124+
struct ggml_tensor * a,
1125+
enum ggml_glu_op op,
1126+
bool swapped);
1127+
1128+
GGML_API struct ggml_tensor * ggml_reglu(
1129+
struct ggml_context * ctx,
1130+
struct ggml_tensor * a);
1131+
1132+
GGML_API struct ggml_tensor * ggml_reglu_swapped(
1133+
struct ggml_context * ctx,
1134+
struct ggml_tensor * a);
1135+
1136+
GGML_API struct ggml_tensor * ggml_geglu(
1137+
struct ggml_context * ctx,
1138+
struct ggml_tensor * a);
1139+
1140+
GGML_API struct ggml_tensor * ggml_geglu_swapped(
1141+
struct ggml_context * ctx,
1142+
struct ggml_tensor * a);
1143+
1144+
GGML_API struct ggml_tensor * ggml_swiglu(
1145+
struct ggml_context * ctx,
1146+
struct ggml_tensor * a);
1147+
1148+
GGML_API struct ggml_tensor * ggml_swiglu_swapped(
1149+
struct ggml_context * ctx,
1150+
struct ggml_tensor * a);
1151+
1152+
// A: n columns, r rows,
1153+
// B: n columns, r rows,
1154+
GGML_API struct ggml_tensor * ggml_glu_split(
1155+
struct ggml_context * ctx,
1156+
struct ggml_tensor * a,
1157+
struct ggml_tensor * b,
1158+
enum ggml_glu_op op);
1159+
1160+
GGML_API struct ggml_tensor * ggml_reglu_split(
1161+
struct ggml_context * ctx,
1162+
struct ggml_tensor * a,
1163+
struct ggml_tensor * b);
1164+
1165+
GGML_API struct ggml_tensor * ggml_geglu_split(
1166+
struct ggml_context * ctx,
1167+
struct ggml_tensor * a,
1168+
struct ggml_tensor * b);
1169+
1170+
GGML_API struct ggml_tensor * ggml_swiglu_split(
1171+
struct ggml_context * ctx,
1172+
struct ggml_tensor * a,
1173+
struct ggml_tensor * b);
1174+
11061175
// normalize along rows
11071176
GGML_API struct ggml_tensor * ggml_norm(
11081177
struct ggml_context * ctx,

0 commit comments

Comments
 (0)