Skip to content

Commit 6e02524

Browse files
firewavefirewave
committed
Tokenizer: provide TokenList in constructor
1 parent e4fbae2 commit 6e02524

9 files changed

+80
-58
lines changed

‎lib/cppcheck.cpp

+17-13
Original file line numberDiff line numberDiff line change
@@ -715,8 +715,9 @@ unsigned int CppCheck::checkClang(const FileWithDetails &file)
715715
}
716716

717717
try {
718-
Tokenizer tokenizer(mSettings, mErrorLogger);
719-
tokenizer.list.appendFileIfNew(file.spath());
718+
TokenList tokenlist{&mSettings};
719+
tokenlist.appendFileIfNew(file.spath());
720+
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
720721
std::istringstream ast(output2);
721722
clangimport::parseClangAstDump(tokenizer, ast);
722723
ValueFlow::setValues(tokenizer.list,
@@ -902,18 +903,17 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
902903

903904
if (mUnusedFunctionsCheck && (mSettings.useSingleJob() || analyzerInformation)) {
904905
std::size_t hash = 0;
905-
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
906-
Tokenizer tokenizer(mSettings, mErrorLogger);
906+
TokenList tokenlist{&mSettings};
907907
// enforce the language since markup files are special and do not adhere to the enforced language
908-
tokenizer.list.setLang(Standards::Language::C, true);
908+
tokenlist.setLang(Standards::Language::C, true);
909909
if (fileStream) {
910910
std::vector<std::string> files;
911911
simplecpp::TokenList tokens(*fileStream, files, file.spath());
912912
if (analyzerInformation) {
913913
const Preprocessor preprocessor(mSettings, mErrorLogger);
914914
hash = calculateHash(preprocessor, tokens, mSettings, mSuppressions);
915915
}
916-
tokenizer.list.createTokens(std::move(tokens));
916+
tokenlist.createTokens(std::move(tokens));
917917
}
918918
else {
919919
std::vector<std::string> files;
@@ -922,8 +922,10 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
922922
const Preprocessor preprocessor(mSettings, mErrorLogger);
923923
hash = calculateHash(preprocessor, tokens, mSettings, mSuppressions);
924924
}
925-
tokenizer.list.createTokens(std::move(tokens));
925+
tokenlist.createTokens(std::move(tokens));
926926
}
927+
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
928+
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
927929
mUnusedFunctionsCheck->parseTokens(tokenizer, mSettings);
928930

929931
if (analyzerInformation) {
@@ -1123,19 +1125,21 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
11231125
continue;
11241126
}
11251127

1126-
Tokenizer tokenizer(mSettings, mErrorLogger);
1127-
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
1128-
tokenizer.setTimerResults(&s_timerResults);
1129-
tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?
1128+
TokenList tokenlist{&mSettings};
11301129

11311130
try {
11321131
// Create tokens, skip rest of iteration if failed
11331132
Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() {
11341133
simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, mCurrentConfig, files, true);
1135-
tokenizer.list.createTokens(std::move(tokensP));
1134+
tokenlist.createTokens(std::move(tokensP));
11361135
});
11371136
hasValidConfig = true;
11381137

1138+
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
1139+
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
1140+
tokenizer.setTimerResults(&s_timerResults);
1141+
tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?
1142+
11391143
// locations macros
11401144
mLogger->setLocationMacros(tokenizer.tokens(), files);
11411145

@@ -1221,7 +1225,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
12211225
mLogger->setAnalyzerInfo(nullptr);
12221226
return mLogger->exitcode();
12231227
} catch (const InternalError &e) {
1224-
ErrorMessage errmsg = ErrorMessage::fromInternalError(e, &tokenizer.list, file.spath());
1228+
ErrorMessage errmsg = ErrorMessage::fromInternalError(e, &tokenlist, file.spath());
12251229
mErrorLogger.reportErr(errmsg);
12261230
}
12271231
}

‎lib/tokenize.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,8 @@ static bool isClassStructUnionEnumStart(const Token * tok)
115115

116116
//---------------------------------------------------------------------------
117117

118-
Tokenizer::Tokenizer(const Settings &settings, ErrorLogger &errorLogger) :
119-
list(&settings),
118+
Tokenizer::Tokenizer(TokenList tokenList, const Settings &settings, ErrorLogger &errorLogger) :
119+
list(std::move(tokenList)),
120120
mSettings(settings),
121121
mErrorLogger(errorLogger),
122122
mTemplateSimplifier(new TemplateSimplifier(*this))
@@ -10969,7 +10969,8 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
1096910969

1097010970
void Tokenizer::getErrorMessages(ErrorLogger& errorLogger, const Settings& settings)
1097110971
{
10972-
Tokenizer tokenizer(settings, errorLogger);
10972+
TokenList tokenlist{&settings};
10973+
Tokenizer tokenizer(std::move(tokenlist), settings, errorLogger);
1097310974
tokenizer.invalidConstFunctionTypeError(nullptr);
1097410975
// checkLibraryNoReturn
1097510976
tokenizer.unhandled_macro_class_x_y(nullptr, "", "", "", "");

‎lib/tokenize.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class CPPCHECKLIB Tokenizer {
5454
friend class TestTokenizer;
5555

5656
public:
57-
explicit Tokenizer(const Settings & settings, ErrorLogger &errorLogger);
57+
Tokenizer(TokenList tokenList, const Settings & settings, ErrorLogger &errorLogger);
5858
~Tokenizer();
5959

6060
void setTimerResults(TimerResults *tr) {

‎lib/tokenlist.h

+2
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ class CPPCHECKLIB TokenList {
5757
TokenList(const TokenList &) = delete;
5858
TokenList &operator=(const TokenList &) = delete;
5959

60+
TokenList(TokenList&& other) NOEXCEPT = default;
61+
6062
/** @return the source file path. e.g. "file.cpp" */
6163
const std::string& getSourceFilePath() const;
6264

‎test/helpers.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,19 @@ namespace tinyxml2 {
4545
class SimpleTokenizer : public Tokenizer {
4646
public:
4747
explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
48-
: Tokenizer{s_settings, errorlogger}
48+
: Tokenizer{TokenList{&s_settings}, s_settings, errorlogger}
4949
{
5050
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
5151
}
5252

5353
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
54-
: Tokenizer{settings, errorlogger}
54+
: Tokenizer{TokenList{&settings}, settings, errorlogger}
5555
{
5656
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
5757
}
5858

5959
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
60-
: Tokenizer{settings, errorlogger}
60+
: Tokenizer{TokenList{&settings}, settings, errorlogger}
6161
{
6262
list.setLang(Path::identify(filename, false));
6363
list.appendFileIfNew(filename);
@@ -238,14 +238,14 @@ class SimpleTokenizer2 : public Tokenizer {
238238
public:
239239
template<size_t size>
240240
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
241-
: Tokenizer{settings, errorlogger}
241+
: Tokenizer{TokenList{&settings}, settings, errorlogger}
242242
{
243243
preprocess(code, mFiles, file0, *this, errorlogger);
244244
}
245245

246246
// TODO: get rid of this
247247
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
248-
: Tokenizer{settings, errorlogger}
248+
: Tokenizer{TokenList{&settings}, settings, errorlogger}
249249
{
250250
preprocess(code, mFiles, file0, *this, errorlogger);
251251
}

‎test/testclangimport.cpp

+5-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <list>
2727
#include <sstream>
2828
#include <string>
29+
#include <utility>
2930
#include <vector>
3031

3132

@@ -139,7 +140,8 @@ class TestClangImport : public TestFixture {
139140

140141
std::string parse(const char clang[]) {
141142
const Settings settings = settingsBuilder().clang().build();
142-
Tokenizer tokenizer(settings, *this);
143+
TokenList tokenlist{&settings};
144+
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
143145
std::istringstream istr(clang);
144146
clangimport::parseClangAstDump(tokenizer, istr);
145147
if (!tokenizer.tokens()) {
@@ -1059,7 +1061,8 @@ class TestClangImport : public TestFixture {
10591061

10601062
#define GET_SYMBOL_DB(AST) \
10611063
const Settings settings = settingsBuilder().clang().platform(Platform::Type::Unix64).build(); \
1062-
Tokenizer tokenizer(settings, *this); \
1064+
TokenList tokenlist{&settings}; \
1065+
Tokenizer tokenizer(std::move(tokenlist), settings, *this); \
10631066
{ \
10641067
std::istringstream istr(AST); \
10651068
clangimport::parseClangAstDump(tokenizer, istr); \

‎test/testsimplifytemplate.cpp

+15-12
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include <cstring>
3030
#include <sstream>
3131
#include <string>
32+
#include <utility>
3233
#include <vector>
3334

3435
class TestSimplifyTemplate : public TestFixture {
@@ -5427,12 +5428,12 @@ class TestSimplifyTemplate : public TestFixture {
54275428
}
54285429

54295430
unsigned int templateParameters(const char code[]) {
5430-
Tokenizer tokenizer(settings, *this);
5431-
5431+
TokenList tokenlist{&settings};
54325432
std::istringstream istr(code);
5433-
tokenizer.list.appendFileIfNew("test.cpp");
5434-
if (!tokenizer.list.createTokens(istr, Path::identify("test.cpp", false)))
5433+
tokenlist.appendFileIfNew("test.cpp");
5434+
if (!tokenlist.createTokens(istr, Path::identify("test.cpp", false)))
54355435
return false;
5436+
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
54365437
tokenizer.createLinks();
54375438
tokenizer.splitTemplateRightAngleBrackets(false);
54385439

@@ -5496,12 +5497,13 @@ class TestSimplifyTemplate : public TestFixture {
54965497

54975498
// Helper function to unit test TemplateSimplifier::getTemplateNamePosition
54985499
int templateNamePositionHelper(const char code[], unsigned offset = 0) {
5499-
Tokenizer tokenizer(settings, *this);
5500+
TokenList tokenlist{&settings};
55005501

55015502
std::istringstream istr(code);
5502-
tokenizer.list.appendFileIfNew("test.cpp");
5503-
if (!tokenizer.list.createTokens(istr, Path::identify("test.cpp", false)))
5503+
tokenlist.appendFileIfNew("test.cpp");
5504+
if (!tokenlist.createTokens(istr, Path::identify("test.cpp", false)))
55045505
return false;
5506+
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
55055507
tokenizer.createLinks();
55065508
tokenizer.splitTemplateRightAngleBrackets(false);
55075509

@@ -5568,11 +5570,11 @@ class TestSimplifyTemplate : public TestFixture {
55685570

55695571
// Helper function to unit test TemplateSimplifier::findTemplateDeclarationEnd
55705572
bool findTemplateDeclarationEndHelper(const char code[], const char pattern[], unsigned offset = 0) {
5571-
Tokenizer tokenizer(settings, *this);
5572-
5573+
TokenList tokenlist{&settings};
55735574
std::istringstream istr(code);
5574-
if (!TokenListHelper::createTokens(tokenizer.list, istr, "test.cpp"))
5575+
if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp"))
55755576
return false;
5577+
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
55765578
tokenizer.createLinks();
55775579
tokenizer.splitTemplateRightAngleBrackets(false);
55785580

@@ -5598,11 +5600,12 @@ class TestSimplifyTemplate : public TestFixture {
55985600

55995601
// Helper function to unit test TemplateSimplifier::getTemplateParametersInDeclaration
56005602
bool getTemplateParametersInDeclarationHelper(const char code[], const std::vector<std::string> & params) {
5601-
Tokenizer tokenizer(settings, *this);
5603+
TokenList tokenlist{&settings};
56025604

56035605
std::istringstream istr(code);
5604-
if (!TokenListHelper::createTokens(tokenizer.list, istr, "test.cpp"))
5606+
if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp"))
56055607
return false;
5608+
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
56065609
tokenizer.createLinks();
56075610
tokenizer.splitTemplateRightAngleBrackets(false);
56085611

‎test/testsimplifytypedef.cpp

+19-14
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <cstddef>
2929
#include <sstream>
3030
#include <string>
31+
#include <utility>
3132

3233
class TestSimplifyTypedef : public TestFixture {
3334
public:
@@ -272,11 +273,11 @@ class TestSimplifyTypedef : public TestFixture {
272273
}
273274

274275
std::string simplifyTypedef(const char code[]) {
275-
Tokenizer tokenizer(settings1, *this);
276-
276+
TokenList tokenlist{&settings1};
277277
std::istringstream istr(code);
278-
if (!tokenizer.list.createTokens(istr, Standards::Language::CPP))
278+
if (!tokenlist.createTokens(istr, Standards::Language::CPP))
279279
return "";
280+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
280281
tokenizer.createLinks();
281282
tokenizer.simplifyTypedef();
282283

@@ -306,11 +307,12 @@ class TestSimplifyTypedef : public TestFixture {
306307

307308

308309
std::string simplifyTypedefC(const char code[]) {
309-
Tokenizer tokenizer(settings1, *this);
310+
TokenList tokenlist{&settings1};
310311

311312
std::istringstream istr(code);
312-
if (!TokenListHelper::createTokens(tokenizer.list, istr, "file.c"))
313+
if (!TokenListHelper::createTokens(tokenlist, istr, "file.c"))
313314
return "";
315+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
314316
tokenizer.createLinks();
315317
tokenizer.simplifyTypedef();
316318
try {
@@ -322,11 +324,11 @@ class TestSimplifyTypedef : public TestFixture {
322324
}
323325

324326
std::string dumpTypedefInfo(const char code[]) {
325-
Tokenizer tokenizer(settings1, *this);
326-
327+
TokenList tokenlist{&settings1};
327328
std::istringstream istr(code);
328-
if (!TokenListHelper::createTokens(tokenizer.list, istr, "file.c"))
329+
if (!TokenListHelper::createTokens(tokenlist, istr, "file.c"))
329330
return {};
331+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
330332
tokenizer.createLinks();
331333
tokenizer.simplifyTypedef();
332334
try {
@@ -4451,9 +4453,10 @@ class TestSimplifyTypedef : public TestFixture {
44514453
"uint8_t t;"
44524454
"void test(rFunctionPointer_fp functionPointer);";
44534455

4454-
Tokenizer tokenizer(settings1, *this);
4456+
TokenList tokenlist{&settings1};
44554457
std::istringstream istr(code);
4456-
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
4458+
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4459+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
44574460
tokenizer.createLinks();
44584461
tokenizer.simplifyTypedef();
44594462

@@ -4493,9 +4496,10 @@ class TestSimplifyTypedef : public TestFixture {
44934496
" MY_INT x = 0;\n"
44944497
"}";
44954498

4496-
Tokenizer tokenizer(settings1, *this);
4499+
TokenList tokenlist{&settings1};
44974500
std::istringstream istr(code);
4498-
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
4501+
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4502+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
44994503
tokenizer.createLinks();
45004504
tokenizer.simplifyTypedef();
45014505

@@ -4511,9 +4515,10 @@ class TestSimplifyTypedef : public TestFixture {
45114515
" F x = 0;\n"
45124516
"}";
45134517

4514-
Tokenizer tokenizer(settings1, *this);
4518+
TokenList tokenlist{&settings1};
45154519
std::istringstream istr(code);
4516-
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
4520+
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4521+
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
45174522
tokenizer.createLinks();
45184523
tokenizer.simplifyTypedef();
45194524

0 commit comments

Comments
 (0)