Skip to content

Commit f996634

Browse files
daboehmedaboehme
authored
Handle newlines in strings (LLNL#506)
1 parent 909c5c8 commit f996634

File tree

8 files changed

+86
-44
lines changed

8 files changed

+86
-44
lines changed

‎python/caliper-reader/caliperreader/caliperstreamreader.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,10 @@ def _read_cali_record(line):
142142
for c in iterator:
143143
if c == '\\':
144144
c = next(iterator)
145-
string += c
145+
if c == 'n':
146+
string += '\n'
147+
else:
148+
string += c
146149
elif c == ',':
147150
entry.append(string)
148151
result[entry[0]] = entry[1:]

‎src/common/util/format_util.h

+41-11
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,55 @@ namespace util
1515
/// \brief Write string \a str to \a os,
1616
/// escaping all characters in \a mask_chars with \a esc.
1717
inline std::ostream&
18-
write_esc_string(std::ostream& os, const char* str, std::string::size_type size, const char* mask_chars = "\\\"", char esc = '\\')
18+
write_json_esc_string(std::ostream& os, const char* str, std::string::size_type size)
1919
{
2020
for (size_t i = 0; i < size; ++i) {
21-
for (const char* p = mask_chars; *p; ++p)
22-
if (str[i] == *p) {
23-
os << esc;
24-
break;
25-
}
26-
27-
os << str[i];
21+
const char c = str[i];
22+
23+
if (c == '\n') // handle newline in string
24+
os << "\\n";
25+
if (c < 0x20) // skip control characters
26+
continue;
27+
if (c == '\\' || c == '\"')
28+
os << '\\';
29+
30+
os << c;
2831
}
29-
32+
3033
return os;
3134
}
3235

36+
/// \brief Write string \a str to \a os,
37+
/// escaping all characters in \a mask_chars with \a esc.
38+
inline std::ostream&
39+
write_cali_esc_string(std::ostream& os, const char* str, std::string::size_type size)
40+
{
41+
for (size_t i = 0; i < size; ++i) {
42+
const char c = str[i];
43+
44+
if (c == '\n') // handle newline in string
45+
os << "\\n";
46+
if (c < 0x20) // skip control characters
47+
continue;
48+
if (c == '\\' || c == ',' || c == '=')
49+
os << '\\';
50+
51+
os << c;
52+
}
53+
54+
return os;
55+
}
56+
57+
inline std::ostream&
58+
write_json_esc_string(std::ostream& os, const std::string& str)
59+
{
60+
return write_json_esc_string(os, str.data(), str.size());
61+
}
62+
3363
inline std::ostream&
34-
write_esc_string(std::ostream& os, const std::string& str, const char* mask_chars = "\\\"", char esc = '\\')
64+
write_cali_esc_string(std::ostream& os, const std::string& str)
3565
{
36-
return write_esc_string(os, str.data(), str.size(), mask_chars, esc);
66+
return write_cali_esc_string(os, str.data(), str.size());
3767
}
3868

3969
std::ostream&

‎src/reader/CaliReader.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ vector<string> split(const string& line, char sep, bool keep_escape = false) {
3838
vec.emplace_back(std::move(str));
3939
str.clear();
4040
str.reserve(line.size());
41+
} else if (escaped && !keep_escape && *it == 'n') {
42+
str.push_back('\n');
43+
escaped = false;
4144
} else {
4245
str.push_back(*it);
4346
escaped = false;

‎src/reader/CaliWriter.cpp

+10-12
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,12 @@ using namespace cali;
1919
namespace
2020
{
2121

22-
const char* esc_chars { "\\,=\n" }; ///< characters that need to be escaped
23-
2422
void write_node_content(std::ostream& os, const cali::Node* node)
2523
{
2624
os << "__rec=node,id=" << node->id()
2725
<< ",attr=" << node->attribute();
2826

29-
util::write_esc_string(os << ",data=", node->data().to_string(), esc_chars);
27+
util::write_cali_esc_string(os << ",data=", node->data().to_string());
3028

3129
if (node->parent() && node->parent()->id() != CALI_INV_ID)
3230
os << ",parent=" << node->parent()->id();
@@ -36,12 +34,12 @@ void write_node_content(std::ostream& os, const cali::Node* node)
3634

3735
void write_record_content(std::ostream& os, const char* record_type, int nr, int ni, const std::vector<Entry>& rec) {
3836
os << "__rec=" << record_type;
39-
37+
4038
// write reference entries
4139

4240
if (nr > 0) {
4341
os << ",ref";
44-
42+
4543
for (const Entry& e : rec)
4644
if (e.is_reference())
4745
os << '=' << e.node()->id();
@@ -60,7 +58,7 @@ void write_record_content(std::ostream& os, const char* record_type, int nr, int
6058

6159
for (const Entry& e : rec)
6260
if (e.is_immediate())
63-
util::write_esc_string(os << '=', e.value().to_string(), esc_chars);
61+
util::write_cali_esc_string(os << '=', e.value().to_string());
6462
}
6563

6664
os << '\n';
@@ -83,7 +81,7 @@ struct CaliWriter::CaliWriterImpl
8381
: m_os(os),
8482
m_num_written(0)
8583
{ }
86-
84+
8785
void recursive_write_node(const CaliperMetadataAccessInterface& db, cali_id_t id)
8886
{
8987
if (id < 11) // don't write the hard-coded metadata nodes
@@ -114,7 +112,7 @@ struct CaliWriter::CaliWriterImpl
114112
g(m_os_lock);
115113

116114
std::ostream* real_os = m_os.stream();
117-
115+
118116
::write_node_content(*real_os, node);
119117
++m_num_written;
120118
}
@@ -125,7 +123,7 @@ struct CaliWriter::CaliWriterImpl
125123

126124
if (m_written_nodes.count(id) > 0)
127125
return;
128-
126+
129127
m_written_nodes.insert(id);
130128
}
131129
}
@@ -135,10 +133,10 @@ struct CaliWriter::CaliWriterImpl
135133
const std::vector<Entry>& rec)
136134
{
137135
// write node entries; count the number of ref and immediate entries
138-
136+
139137
int nr = 0;
140138
int ni = 0;
141-
139+
142140
for (const Entry& e : rec) {
143141
if (e.is_reference()) {
144142
recursive_write_node(db, e.node()->id());
@@ -150,7 +148,7 @@ struct CaliWriter::CaliWriterImpl
150148
}
151149

152150
// write the record
153-
151+
154152
{
155153
std::lock_guard<std::mutex>
156154
g(m_os_lock);

‎src/reader/JsonFormatter.cpp

+8-8
Original file line numberDiff line numberDiff line change
@@ -194,14 +194,14 @@ struct JsonFormatter::JsonFormatterImpl
194194
for (auto &p : quote_kvs) {
195195
*real_os << (count++ > 0 ? "," : "")
196196
<< (m_opt_pretty ? "\n\t" : "");
197-
util::write_esc_string(*real_os << "\"", p.first) << "\":";
198-
util::write_esc_string(*real_os << "\"", p.second) << "\"";
197+
util::write_json_esc_string(*real_os << "\"", p.first) << "\":";
198+
util::write_json_esc_string(*real_os << "\"", p.second) << "\"";
199199
}
200200
for (auto &p : noquote_kvs) {
201201
*real_os << (count++ > 0 ? "," : "")
202202
<< (m_opt_pretty ? "\n\t" : "");
203-
util::write_esc_string(*real_os << "\"", p.first) << "\":";
204-
util::write_esc_string(*real_os, p.second);
203+
util::write_json_esc_string(*real_os << "\"", p.first) << "\":";
204+
util::write_json_esc_string(*real_os, p.second);
205205
}
206206

207207
*real_os << (m_opt_pretty ? "\n" : "") << "}";
@@ -242,8 +242,8 @@ struct JsonFormatter::JsonFormatterImpl
242242

243243
// print meta-info
244244
for (const Node* node = a.node(); node && node->attribute() != CALI_INV_ID; node = node->parent()) {
245-
util::write_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
246-
util::write_esc_string(os << "\"", node->data().to_string()) << '\"';
245+
util::write_json_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
246+
util::write_json_esc_string(os << "\"", node->data().to_string()) << '\"';
247247
}
248248

249249
os << "}";
@@ -276,8 +276,8 @@ struct JsonFormatter::JsonFormatterImpl
276276
if (m_opt_pretty)
277277
os << '\t';
278278

279-
util::write_esc_string(os << '\"', db.get_attribute(p.first).name()) << "\": ";
280-
util::write_esc_string(os << '\"', p.second) << '\"';
279+
util::write_json_esc_string(os << '\"', db.get_attribute(p.first).name()) << "\": ";
280+
util::write_json_esc_string(os << '\"', p.second) << '\"';
281281
}
282282

283283
return os;

‎src/reader/JsonSplitFormatter.cpp

+10-10
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ class Hierarchy
5151
const std::string& label() const { return m_label; }
5252

5353
std::ostream& write_json(std::ostream& os) const {
54-
util::write_esc_string(os << "{ \"label\": \"", m_label ) << "\"";
55-
util::write_esc_string(os << ", \"column\": \"", m_column) << "\"";
54+
util::write_json_esc_string(os << "{ \"label\": \"", m_label ) << "\"";
55+
util::write_json_esc_string(os << ", \"column\": \"", m_column) << "\"";
5656

5757
if (parent() && parent()->id() != CALI_INV_ID)
5858
os << ", \"parent\": " << parent()->id();
@@ -250,7 +250,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
250250

251251
if (!path.attributes.empty())
252252
columns.push_back(path);
253-
253+
254254
return columns;
255255
}
256256

@@ -281,7 +281,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
281281
for (const Entry& e : list)
282282
if (e.attribute() == attr.id()) {
283283
if (quote)
284-
util::write_esc_string(os << "\"", e.value().to_string()) << "\"";
284+
util::write_json_esc_string(os << "\"", e.value().to_string()) << "\"";
285285
else
286286
os << e.value().to_string();
287287

@@ -294,7 +294,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
294294
void process_record(const CaliperMetadataAccessInterface& db, const EntryList& list) {
295295
std::lock_guard<std::mutex>
296296
g(m_records_lock);
297-
297+
298298
m_records.push_back(list);
299299
}
300300

@@ -316,8 +316,8 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
316316
global_vals[e.attribute()] = e.value().to_string();
317317

318318
for (auto &p : global_vals) {
319-
util::write_esc_string(os << ",\n \"", db.get_attribute(p.first).name()) << "\": ";
320-
util::write_esc_string(os << '"', p.second) << '\"';
319+
util::write_json_esc_string(os << ",\n \"", db.get_attribute(p.first).name()) << "\": ";
320+
util::write_json_esc_string(os << '"', p.second) << '\"';
321321
}
322322

323323
return os;
@@ -340,8 +340,8 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
340340
if (attr.id() < 12 || attr.is_hidden())
341341
continue;
342342

343-
util::write_esc_string(os << ", \"", attr.name_c_str()) << "\": ";
344-
util::write_esc_string(os << "\"", node->data().to_string()) << "\"";
343+
util::write_json_esc_string(os << ", \"", attr.name_c_str()) << "\": ";
344+
util::write_json_esc_string(os << "\"", node->data().to_string()) << "\"";
345345
}
346346
}
347347

@@ -355,7 +355,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
355355
{
356356
int count = 0;
357357
for (const Column& c : columns)
358-
util::write_esc_string(os << (count++ > 0 ? ", " : " ") << "\"", c.title) << "\"";
358+
util::write_json_esc_string(os << (count++ > 0 ? ", " : " ") << "\"", c.title) << "\"";
359359
}
360360

361361
// close "columns", start "column_metadata"

‎test/ci_app_tests/ci_test_basic.cpp

+6-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
#include "caliper/cali.h"
44

5-
int main()
5+
#include <cstring>
6+
7+
int main(int argc, char* argv[])
68
{
79
std::map<const char*, cali::Variant> metadata = {
810
{ "meta.int", cali::Variant(42) }
@@ -12,6 +14,9 @@ int main()
1214
cali_set_string_byname(" =\\weird \"\"attribute\"= ", " \\\\ weird,\" name\",");
1315
cali_set_global_string_byname(" =\\weird \"\" global attribute\"= ", " \\\\ weird,\" name\",");
1416

17+
if (argc > 1 && strcmp(argv[1], "newline") == 0)
18+
cali_set_string_byname("newline", "A newline:\n!");
19+
1520
cali::Annotation phase_ann("phase", metadata);
1621
std::size_t size = 8;
1722
cali::Annotation size_annot("dgs");

‎test/ci_app_tests/test_basictrace.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ def test_globals_selection(self):
152152
snapshots, { 'cali.caliper.version' } ) )
153153

154154
def test_esc(self):
155-
target_cmd = [ './ci_test_basic' ]
155+
target_cmd = [ './ci_test_basic', 'newline' ]
156156
query_cmd = [ '../../src/tools/cali-query/cali-query', '-j', '-s', 'cali.event.set' ]
157157

158158
caliper_config = {
@@ -163,7 +163,10 @@ def test_esc(self):
163163

164164
obj = json.loads( cat.run_test_with_query(target_cmd, query_cmd, caliper_config) )
165165

166+
self.assertEqual(len(obj), 2)
167+
166168
self.assertEqual(obj[0]['event.set# =\\weird ""attribute"= '], ' \\\\ weird," name",' )
169+
self.assertEqual(obj[1]['event.set#newline'], 'A newline:\n!')
167170

168171
def test_macros(self):
169172
# Use ConfigManager API here

0 commit comments

Comments
 (0)