Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

testrunner: make sure that more redirects/outputs are actually being consumed #5721

Merged
merged 1 commit into from
Dec 6, 2023
Merged

testrunner: make sure that more redirects/outputs are actually being consumed #5721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -758,7 +758,7 @@ test/testfunctions.o: test/testfunctions.cpp lib/addoninfo.h lib/check.h lib/che
test/testgarbage.o: test/testgarbage.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testgarbage.cpp

test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h
test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h test/redirect.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testimportproject.cpp

test/testincompletestatement.o: test/testincompletestatement.cpp lib/addoninfo.h lib/check.h lib/checkother.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h test/fixture.h test/helpers.h
Expand Down
17 changes: 16 additions & 1 deletion test/fixture.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,20 @@ bool TestFixture::prepareTest(const char testname[])
void TestFixture::teardownTest()
{
teardownTestInternal();

// TODO: enable
/*
{
const std::string s = errout.str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger err: " + s);
}
*/
{
const std::string s = output_str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger out: " + s);
}
}

std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
Expand Down Expand Up @@ -381,14 +395,15 @@ std::size_t TestFixture::runTests(const options& args)

void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
{
output << outmsg << std::endl;
mOutput << outmsg << std::endl;
}

void TestFixture::reportErr(const ErrorMessage &msg)
{
if (msg.severity == Severity::none && msg.id == "logChecker")
return;
const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
// TODO: remove the unique error handling?
if (errout.str().find(errormessage) == std::string::npos)
errout << errormessage << std::endl;
}
Expand Down
10 changes: 8 additions & 2 deletions test/fixture.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,11 +236,17 @@ class TestFixture : public ErrorLogger {
return SettingsBuilder(*this, std::move(settings));
}

// TODO: make sure the output has been consumed in the test
std::string output_str() {
std::string s = mOutput.str();
mOutput.str("");
return s;
}

std::ostringstream errout;
std::ostringstream output;

private:
std::ostringstream mOutput;

void reportOut(const std::string &outmsg, Color c = Color::Reset) override;
void reportErr(const ErrorMessage &msg) override;
void run(const std::string &str);
Expand Down
16 changes: 16 additions & 0 deletions test/testimportproject.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "settings.h"
#include "filesettings.h"
#include "fixture.h"
#include "redirect.h"

#include <list>
#include <map>
Expand Down Expand Up @@ -112,6 +113,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands1() const {
REDIRECT;
constexpr char json[] = R"([{
"directory": "/tmp",
"command": "gcc -DTEST1 -DTEST2=2 -o /tmp/src.o -c /tmp/src.c",
Expand All @@ -125,6 +127,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands2() const {
REDIRECT;
// Absolute file path
#ifdef _WIN32
const char json[] = R"([{
Expand Down Expand Up @@ -152,6 +155,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands3() const {
REDIRECT;
const char json[] = R"([{
"directory": "/tmp/",
"command": "gcc -c src.c",
Expand All @@ -165,6 +169,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands4() const {
REDIRECT;
constexpr char json[] = R"([{
"directory": "/tmp/",
"command": "gcc -c src.mm",
Expand All @@ -177,6 +182,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands5() const {
REDIRECT;
constexpr char json[] =
R"([{
"directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
Expand All @@ -196,6 +202,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands6() const {
REDIRECT;
constexpr char json[] =
R"([{
"directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
Expand All @@ -217,6 +224,7 @@ class TestImportProject : public TestFixture {


void importCompileCommands7() const {
REDIRECT;
// cmake -DFILESDIR="/some/path" ..
constexpr char json[] =
R"([{
Expand All @@ -237,6 +245,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands8() const {
REDIRECT;
// cmake -DFILESDIR="C:\Program Files\Cppcheck" -G"NMake Makefiles" ..
constexpr char json[] =
R"([{
Expand All @@ -250,6 +259,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands9() const {
REDIRECT;
// IAR output (https://sourceforge.net/p/cppcheck/discussion/general/thread/608af51e0a/)
constexpr char json[] =
R"([{
Expand All @@ -266,6 +276,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands10() const { // #10887
REDIRECT;
constexpr char json[] =
R"([{
"file": "/home/danielm/cppcheck/1/test folder/1.c" ,
Expand All @@ -285,6 +296,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands11() const { // include path order
REDIRECT;
constexpr char json[] =
R"([{
"file": "1.c" ,
Expand All @@ -307,6 +319,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommandsArgumentsSection() const {
REDIRECT;
constexpr char json[] = "[ { \"directory\": \"/tmp/\","
"\"arguments\": [\"gcc\", \"-c\", \"src.c\"],"
"\"file\": \"src.c\" } ]";
Expand All @@ -318,15 +331,18 @@ class TestImportProject : public TestFixture {
}

void importCompileCommandsNoCommandSection() const {
REDIRECT;
constexpr char json[] = "[ { \"directory\": \"/tmp/\","
"\"file\": \"src.mm\" } ]";
std::istringstream istr(json);
TestImporter importer;
ASSERT_EQUALS(false, importer.importCompileCommands(istr));
ASSERT_EQUALS(0, importer.fileSettings.size());
ASSERT_EQUALS("cppcheck: error: no 'arguments' or 'command' field found in compilation database entry\n", GET_REDIRECT_OUTPUT);
}

void importCppcheckGuiProject() const {
REDIRECT;
constexpr char xml[] = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<project version=\"1\">\n"
" <root name=\".\"/>\n"
Expand Down
3 changes: 1 addition & 2 deletions test/testprocessexecutor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ class TestProcessExecutorBase : public TestFixture {
*/
void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -265,7 +264,7 @@ class TestProcessExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}*/));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down
5 changes: 2 additions & 3 deletions test/testsingleexecutor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ class TestSingleExecutorBase : public TestFixture {

void check(int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -169,7 +168,7 @@ class TestSingleExecutorBase : public TestFixture {
expected += "Checking " + fprefix() + "_" + zpad3(i) + ".cpp ...\n";
expected += std::to_string(i) + "/100 files checked " + std::to_string(i) + "% done\n";
}
ASSERT_EQUALS(expected, output.str());
ASSERT_EQUALS(expected, output_str());
}

void many_files_showtime() {
Expand Down Expand Up @@ -259,7 +258,7 @@ class TestSingleExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down
8 changes: 5 additions & 3 deletions test/testsuppressions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,6 @@ class TestSuppressions : public TestFixture {
unsigned int checkSuppression(std::map<std::string, std::string> &f, const std::string &suppression = emptyString) {
// Clear the error log
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
for (std::map<std::string, std::string>::const_iterator i = f.cbegin(); i != f.cend(); ++i) {
Expand All @@ -210,6 +209,7 @@ class TestSuppressions : public TestFixture {
CppCheck cppCheck(*this, true, nullptr);
Settings& settings = cppCheck.settings();
settings.jobs = 1;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (suppression == "unusedFunction")
Expand All @@ -234,13 +234,13 @@ class TestSuppressions : public TestFixture {

unsigned int checkSuppressionThreads(const char code[], const std::string &suppression = emptyString) {
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
files.emplace_back("test.cpp", strlen(code));

Settings settings;
settings.jobs = 2;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (!suppression.empty()) {
Expand All @@ -264,13 +264,13 @@ class TestSuppressions : public TestFixture {
#if !defined(WIN32) && !defined(__MINGW32__) && !defined(__CYGWIN__)
unsigned int checkSuppressionProcesses(const char code[], const std::string &suppression = emptyString) {
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
files.emplace_back("test.cpp", strlen(code));

Settings settings;
settings.jobs = 2;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (!suppression.empty()) {
Expand Down Expand Up @@ -1089,6 +1089,7 @@ class TestSuppressions : public TestFixture {

CppCheck cppCheck(*this, false, nullptr); // <- do not "use global suppressions". pretend this is a thread that just checks a file.
Settings& settings = cppCheck.settings();
settings.quiet = true;
settings.nomsg.addSuppressionLine("uninitvar");
settings.exitCode = 1;

Expand Down Expand Up @@ -1123,6 +1124,7 @@ class TestSuppressions : public TestFixture {

CppCheck cppCheck(*this, true, nullptr);
Settings& settings = cppCheck.settings();
settings.quiet = true;
settings.severity.enable(Severity::style);
settings.inlineSuppressions = true;
settings.relativePaths = true;
Expand Down
3 changes: 1 addition & 2 deletions test/testthreadexecutor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ class TestThreadExecutorBase : public TestFixture {
*/
void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -262,7 +261,7 @@ class TestThreadExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down
Loading