Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify ConfigManager query specs #638

Merged
merged 1 commit into from
Feb 14, 2025
Merged

Simplify ConfigManager query specs #638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions include/caliper/ConfigManager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,22 +116,19 @@ class ConfigManager
/// according to the selected options
void update_channel_metadata(info_map_t& metadata) const;

/// \brief Returns a CalQL query based on the fields in \a input
/// and option list.
/// \brief Returns a CalQL query based on the query \a in
/// and the selected options.
///
/// Construct a CalQL query for the option list. Example:
/// Construct a CalQL query for the selected options. Example:
/// \code
/// auto query = opts.build_query("cross", {
/// { "select", "sum(inclusive#sum#time.duration) as Total unit sec" },
/// { "group by", "prop:nested" },
/// { "format", "tree" }
/// });
/// auto query = opts.build_query("cross",
/// "select sum(inclusive#sum#time.duration.ns) as Time unit sec group by path format tree");
/// \endcode
///
/// \param level The aggreatation level ("local" or "cross")
/// \param in Base CalQL clauses as needed by the controller
/// \return Complete CalQL query statement
std::string build_query(const char* level, const std::map<std::string, std::string>& in) const;
std::string build_query(const char* level, const std::string& in) const;

friend class ConfigManager;
};
Expand Down
119 changes: 50 additions & 69 deletions src/caliper/ConfigManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,19 +141,18 @@ std::string join_stringlist(const std::vector<std::string>& list)
return ret;
}

void join_stringlist(std::string& in, const std::vector<std::string>& list)
std::string join_stringlist(const std::vector<StringConverter>& list)
{
for (const auto& s : list) {
if (!in.empty())
in.append(",");
in.append(s);
std::string ret;
int c = 0;

for (const StringConverter& sc : list) {
if (c++ > 0)
ret.append(",");
ret.append(sc.to_string());
}
}

std::string find_or(const std::map<std::string, std::string>& m, const std::string& k, const std::string v = "")
{
auto it = m.find(k);
return it != m.end() ? it->second : v;
return ret;
}

std::string expand_variables(const std::string& in, const std::string& val)
Expand Down Expand Up @@ -340,15 +339,6 @@ void add_metadata(const std::string& args, info_map_t& info)

class ConfigManager::OptionSpec
{
struct query_arg_t {
std::vector<std::string> select;
std::vector<std::string> groupby;
std::vector<std::string> let;
std::vector<std::string> where;
std::vector<std::string> aggregate;
std::vector<std::string> orderby;
};

struct option_spec_t {
std::string type;
std::string description;
Expand All @@ -357,7 +347,7 @@ class ConfigManager::OptionSpec
std::vector<std::string> services;
std::vector<std::string> inherited_specs;

std::map<std::string, query_arg_t> query_args;
std::map<std::string, std::string> query;
std::map<std::string, std::string> config;
};

Expand All @@ -372,8 +362,10 @@ class ConfigManager::OptionSpec
m_error_msg = msg;
}

void parse_select(const std::vector<StringConverter>& list, query_arg_t& qarg)
std::string parse_select(const std::vector<StringConverter>& list)
{
std::string ret;

for (const StringConverter& sc : list) {
bool is_a_dict = false;
std::map<std::string, StringConverter> dict = sc.rec_dict(&is_a_dict);
Expand All @@ -397,50 +389,58 @@ class ConfigManager::OptionSpec
str.append(it->second.to_string());
str.append("\"");
}
qarg.select.push_back(str);
if (!ret.empty())
ret.append(",");
ret.append(str);
} else {
qarg.select.push_back(sc.to_string());
}
if (!ret.empty())
ret.append(",");
ret.append(sc.to_string());
}
}

return ret.empty() ? ret : std::string(" select ") + ret;
}

void parse_query_args(const std::vector<StringConverter>& list, option_spec_t& opt)
{
// parses the deprecated list-of-dicts form of the query args

for (const StringConverter& sc : list) {
std::map<std::string, StringConverter> dict = sc.rec_dict();
query_arg_t qarg;
std::string query;

auto it = dict.find("group by");
if (it != dict.end())
qarg.groupby = ::to_stringlist(it->second.rec_list());
query.append(" group by ").append(::join_stringlist(it->second.rec_list()));

it = dict.find("let");
if (it != dict.end())
qarg.let = ::to_stringlist(it->second.rec_list());
query.append(" let ").append(::join_stringlist(it->second.rec_list()));

it = dict.find("where");
if (it != dict.end())
qarg.where = ::to_stringlist(it->second.rec_list());
query.append(" where ").append(::join_stringlist(it->second.rec_list()));

it = dict.find("aggregate");
if (it != dict.end())
qarg.aggregate = ::to_stringlist(it->second.rec_list());
query.append(" aggregate ").append(::join_stringlist(it->second.rec_list()));

it = dict.find("order by");
if (it != dict.end())
qarg.orderby = ::to_stringlist(it->second.rec_list());
query.append(" order by ").append(::join_stringlist(it->second.rec_list()));

it = dict.find("select");
if (it != dict.end())
parse_select(it->second.rec_list(), qarg);
query.append(parse_select(it->second.rec_list()));

it = dict.find("level");
if (it == dict.end()) {
set_error(": query arg: missing \"level\"");
continue;
}

opt.query_args[it->second.to_string()] = qarg;
opt.query[it->second.to_string()] = std::move(query);
}
}

Expand Down Expand Up @@ -468,8 +468,17 @@ class ConfigManager::OptionSpec
if (ok && !m_error && it != dict.end())
parse_config(it->second.rec_dict(&ok), opt);
it = dict.find("query");
if (ok && !m_error && it != dict.end())
parse_query_args(it->second.rec_list(&ok), opt);
if (ok && !m_error && it != dict.end()) {
bool is_dict = false;
auto query_dict = it->second.rec_dict(&is_dict);
if (is_dict) {
for (const auto &query_entry : query_dict)
opt.query[query_entry.first] = query_entry.second.to_string();
} else {
// try parsing deprecated form of query args
parse_query_args(it->second.rec_list(&ok), opt);
}
}
it = dict.find("type");
if (ok && !m_error && it != dict.end())
opt.type = it->second.to_string();
Expand Down Expand Up @@ -691,50 +700,22 @@ struct ConfigManager::Options::OptionsImpl {
}
}

std::string build_query(const char* level, const std::map<std::string, std::string>& in) const
std::string build_query(const char* level, const std::string& in) const
{
std::string q_let = ::find_or(in, "let");
std::string q_select = ::find_or(in, "select");
std::string q_groupby = ::find_or(in, "group by");
std::string q_where = ::find_or(in, "where");
std::string q_aggregate = ::find_or(in, "aggregate");
std::string q_orderby = ::find_or(in, "order by");
std::string q_format = ::find_or(in, "format");
std::string ret = in;

for (const std::string& opt : enabled_options) {
auto s_it = spec.data.find(opt);
if (s_it == spec.data.end())
continue;

auto l_it = s_it->second.query_args.find(level);
if (l_it != s_it->second.query_args.end()) {
const auto& q = l_it->second;
::join_stringlist(q_let, q.let);
::join_stringlist(q_select, q.select);
::join_stringlist(q_groupby, q.groupby);
::join_stringlist(q_where, q.where);
::join_stringlist(q_aggregate, q.aggregate);
::join_stringlist(q_orderby, q.orderby);
auto l_it = s_it->second.query.find(level);
if (l_it != s_it->second.query.end()) {
ret.append(" ");
ret.append(l_it->second);
}
}

std::string ret;

if (!q_let.empty())
ret.append(" let ").append(q_let);
if (!q_select.empty())
ret.append(" select ").append(q_select);
if (!q_groupby.empty())
ret.append(" group by ").append(q_groupby);
if (!q_where.empty())
ret.append(" where ").append(q_where);
if (!q_aggregate.empty())
ret.append(" aggregate ").append(q_aggregate);
if (!q_orderby.empty())
ret.append(" order by ").append(q_orderby);
if (!q_format.empty())
ret.append(" format ").append(q_format);

return ret;
}

Expand Down Expand Up @@ -858,7 +839,7 @@ void ConfigManager::Options::update_channel_metadata(info_map_t& metadata) const
mP->update_channel_metadata(metadata);
}

std::string ConfigManager::Options::build_query(const char* level, const std::map<std::string, std::string>& in) const
std::string ConfigManager::Options::build_query(const char* level, const std::string& in) const
{
return mP->build_query(level, in);
}
Expand Down
30 changes: 12 additions & 18 deletions src/caliper/controllers/CudaActivityProfileController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ class CudaActivityProfileController : public cali::ChannelController
const char* name,
const config_map_t& initial_cfg,
const ConfigManager::Options& opts,
const std::string& format
const std::string& format_spec
)
: ChannelController(name, 0, initial_cfg)
{
std::string output(opts.get("output", "cuda_profile"));

if (output != "stdout" && output != "stderr") {
auto pos = output.find_last_of('.');
std::string ext = (format == "cali" ? ".cali" : ".json");
std::string ext = (format_spec == "cali" ? ".cali" : ".json");

if (pos == std::string::npos || output.substr(pos) != ext)
output.append(ext);
Expand All @@ -56,30 +56,24 @@ class CudaActivityProfileController : public cali::ChannelController
config()["CALI_ADIAK_IMPORT_CATEGORIES"] = opts.get("adiak.import_categories", "2,3");
}

std::string query =
" select *,scale(cupti.activity.duration,1e-9) as \"time (gpu)\" unit sec"
",scale(sum#cupti.host.duration,1e-9) as \"time\" unit sec"
" group by path,cupti.kernel.name,cupti.activity.kind,mpi.rank "
" format ";

query.append(format_spec);

if (use_mpi) {
config()["CALI_SERVICES_ENABLE"].append(",mpi,mpireport");
config()["CALI_AGGREGATE_KEY"] = "*,mpi.rank";
config()["CALI_MPIREPORT_FILENAME"] = output;
config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false";
config()["CALI_MPIREPORT_CONFIG"] = opts.build_query(
"local",
{ { "select",
"*,scale(cupti.activity.duration,1e-9) as \"time (gpu)\" unit sec"
" ,scale(sum#cupti.host.duration,1e-9) as \"time\" unit sec" },
{ "group by", "path,cupti.kernel.name,cupti.activity.kind,mpi.rank" },
{ "format", format } }
);
config()["CALI_MPIREPORT_CONFIG"] = opts.build_query("local", query);
} else {
config()["CALI_SERVICES_ENABLE"].append(",report");
config()["CALI_REPORT_FILENAME"] = output;
config()["CALI_REPORT_CONFIG"] = opts.build_query(
"local",
{ { "select",
"*,scale(cupti.activity.duration,1e-9) as \"time (gpu)\" unit sec"
" ,scale(sum#cupti.host.duration,1e-9) as \"time\" unit sec" },
{ "group by", "path,cupti.kernel.name,cupti.activity.kind" },
{ "format", format } }
);
config()["CALI_REPORT_CONFIG"] = opts.build_query("local", query);
}

opts.update_channel_config(config());
Expand Down
33 changes: 16 additions & 17 deletions src/caliper/controllers/CudaActivityReportController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,8 @@ class CudaActivityReportController : public cali::ChannelController
)
: ChannelController(name, 0, initial_cfg)
{
// Config for first aggregation step in MPI mode (process-local aggregation)
std::string local_select =
" inclusive_scale(sum#cupti.host.duration,1e-9)"
",inclusive_scale(cupti.activity.duration,1e-9)";
// Config for serial-mode aggregation
std::string serial_select =
std::string local_select =
" inclusive_scale(sum#cupti.host.duration,1e-9) as \"Host Time\""
",inclusive_scale(cupti.activity.duration,1e-9) as \"GPU Time\""
",inclusive_ratio(cupti.activity.duration,sum#cupti.host.duration,100.0) as \"GPU %\"";
Expand All @@ -54,31 +50,34 @@ class CudaActivityReportController : public cali::ChannelController

if (opts.is_enabled("show_kernels")) {
groupby += ",cupti.kernel.name";
serial_select = std::string("cupti.kernel.name as Kernel,") + serial_select;
cross_select = std::string("cupti.kernel.name as Kernel,") + cross_select;
local_select = std::string("cupti.kernel.name as Kernel,") + local_select;
cross_select = std::string("cupti.kernel.name as Kernel,") + cross_select;
}

std::string format = util::build_tree_format_spec(config(), opts);

std::string local_query = "select ";
local_query.append(local_select);
local_query.append(" group by ").append(groupby);
local_query.append(" format ").append(format);

std::string cross_query = "select ";
cross_query.append(cross_select);
cross_query.append(" group by ").append(groupby);
cross_query.append(" format ").append(format);

if (use_mpi) {
config()["CALI_SERVICES_ENABLE"].append(",mpi,mpireport");
config()["CALI_MPIREPORT_FILENAME"] = opts.get("output", "stderr");
config()["CALI_MPIREPORT_APPEND"] = opts.get("output.append");
config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false";
config()["CALI_MPIREPORT_LOCAL_CONFIG"] =
opts.build_query("local", { { "select", local_select }, { "group by", groupby } });
config()["CALI_MPIREPORT_CONFIG"] = opts.build_query(
"cross",
{ { "select", cross_select }, { "group by", groupby }, { "format", format } }
);
config()["CALI_MPIREPORT_LOCAL_CONFIG"] = opts.build_query("local", local_query);
config()["CALI_MPIREPORT_CONFIG"] = opts.build_query("cross", cross_query);
} else {
config()["CALI_SERVICES_ENABLE"].append(",report");
config()["CALI_REPORT_FILENAME"] = opts.get("output", "stderr");
config()["CALI_REPORT_APPEND"] = opts.get("output.append");
config()["CALI_REPORT_CONFIG"] = opts.build_query(
"local",
{ { "select", serial_select }, { "group by", groupby }, { "format", format } }
);
config()["CALI_REPORT_CONFIG"] = opts.build_query("local", local_query);
}

opts.update_channel_config(config());
Expand Down
Loading
Loading