add smoketest:

- UTF-8 with BOM
- UTF-16 with BOM

cxx-squid/src/main/java/org/sonar/cxx/parser/CxxLexer.java

@@ -22,6 +22,7 @@
 import com.sonar.sslr.api.Preprocessor;
 import com.sonar.sslr.impl.Lexer;
 import com.sonar.sslr.impl.channel.BlackHoleChannel;
+import com.sonar.sslr.impl.channel.BomCharacterChannel;
 import com.sonar.sslr.impl.channel.IdentifierAndKeywordChannel;
 import com.sonar.sslr.impl.channel.PunctuatorChannel;
 import static com.sonar.sslr.impl.channel.RegexpChannelBuilder.ANY_CHAR;
@@ -109,6 +110,7 @@ public static Lexer create(Charset charset, Preprocessor... preprocessors) {
       // C++ Standard, Section 2.13 "Operators and punctuators"
       .withChannel(new RightAngleBracketsChannel())
       .withChannel(new PunctuatorChannel(CxxPunctuator.values()))
+      .withChannel(new BomCharacterChannel())
       .withChannel(new UnknownCharacterChannel());
 
     for (var preprocessor : preprocessors) {

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CppLexer.java

@@ -20,6 +20,7 @@
 package org.sonar.cxx.preprocessor;
 
 import com.sonar.sslr.impl.Lexer;
+import com.sonar.sslr.impl.channel.BomCharacterChannel;
 import com.sonar.sslr.impl.channel.IdentifierAndKeywordChannel;
 import com.sonar.sslr.impl.channel.PunctuatorChannel;
 import static com.sonar.sslr.impl.channel.RegexpChannelBuilder.ANY_CHAR;
@@ -32,10 +33,10 @@
 import static com.sonar.sslr.impl.channel.RegexpChannelBuilder.regexp;
 import com.sonar.sslr.impl.channel.UnknownCharacterChannel;
 import java.nio.charset.Charset;
-import org.sonar.cxx.parser.CxxTokenType;
 import org.sonar.cxx.channels.CharacterLiteralsChannel;
 import org.sonar.cxx.channels.KeywordChannel;
 import org.sonar.cxx.channels.StringLiteralsChannel;
+import org.sonar.cxx.parser.CxxTokenType;
 
 public final class CppLexer {
 
@@ -92,6 +93,7 @@ public static Lexer create(Charset charset) {
       //###.withChannel(new SpecialIdentifiers("import"))
       .withChannel(new IdentifierAndKeywordChannel(and("[a-zA-Z_]", o2n("\\w")), true))
       .withChannel(new PunctuatorChannel(CppPunctuator.values()))
+      .withChannel(new BomCharacterChannel())
       .withChannel(new UnknownCharacterChannel());
 
     return builder.build();

integration-tests/features/regex.feature

@@ -32,15 +32,15 @@ Feature: Regex
       """
     And the following metrics have following values:
       | metric     | value |
-      | ncloc      | 6     |
+      | ncloc      | 3     |
       | lines      | 18    |
       | files      | 3     |
       | violations | 0     |
 
   Scenario: Test rule 'cxx:FileHeader' with regex and UTF-16 files (with BOM) with different file endings.
     Given the project "regex_project"
     And rule "cxx:FileHeader" with params "isRegularExpression=true;headerFormat=//\s*<copyright>\s*//\s*Copyright \(c\) (AAA BBB|CCC DDD) GmbH. All rights reserved.\s*//\s*</copyright>\s*" is activated
-    When I run sonar-scanner with "-X -Dsonar.inclusions=**/utf16-BOM-*.cc"
+    When I run sonar-scanner with "-X -Dsonar.inclusions=**/utf16-BOM-*.cc -Dsonar.sourceEncoding=UTF-16LE"
     Then the analysis finishes successfully
     And the analysis in server has completed
     And the analysis log contains no error/warning messages except those matching:
@@ -49,7 +49,7 @@ Feature: Regex
       """
     And the following metrics have following values:
       | metric     | value |
-      | ncloc      | 23    |
+      | ncloc      | 3     |
       | lines      | 18    |
       | files      | 3     |
       | violations | 0     |

integration-tests/features/smoketest.feature

@@ -3,7 +3,7 @@ Feature: Smoketests
 
   This is for running smoketests using somewhat more complex test data.
 
-  Scenario: Smoketest
+  Scenario: Smoketest UTF-8 (without BOM)
     Given the project "smoketest_project"
     And rule "cppcheck:unusedVariable" is activated
     And rule "cppcheck:unreadVariable" is activated
@@ -68,6 +68,136 @@ Feature: Smoketests
       | security_hotspots              | 0     |
 
 
+  Scenario: Smoketest UTF-8 (with BOM)
+    Given the project "smoketest_project_utf8_bom"
+    And rule "cppcheck:unusedVariable" is activated
+    And rule "cppcheck:unreadVariable" is activated
+    And rule "cppcheck:deallocDealloc" is activated
+    And rule "cppcheck:doubleFree" is activated
+    And rule "cppcheck:uninitvar" is activated
+    And rule "cppcheck:unusedFunction" is activated
+    And rule "cppcheck:missingInclude" is activated
+    When I run sonar-scanner with "-X"
+    Then the analysis finishes successfully
+    And the analysis in server has completed
+    And the analysis log contains no error/warning messages except those matching:
+      """
+      .*WARN.*Unable to get a valid mac address, will use a dummy address
+      .*WARN.*cannot find the sources for '#include <gtest/gtest\.h>'
+      .*WARN.*cannot find the sources for '#include <iostream>'
+      .*WARN.*Cannot find the file '.*component_XXX.cc', skipping
+      .*WARN.*Cannot find a report for '.*'
+      """
+    And the following metrics have following values:
+      | metric                         | value |
+      | ncloc                          | 56    |
+      | lines                          | 151   |
+      | statements                     | 36    |
+      | classes                        | 1     |
+      | files                          | 8     |
+      | directories                    | None  |
+      | functions                      | 5     |
+      | comment_lines_density          | 30    |
+      | comment_lines                  | 24    |
+      | duplicated_lines_density       | 55.6  |
+      | duplicated_lines               | 84    |
+      | duplicated_blocks              | 2     |
+      | duplicated_files               | 2     |
+      | complexity                     | 7     |
+      | cognitive_complexity           | 2     |
+      | file_complexity                | 0.9   |
+      | violations                     | 12    |
+      | lines_to_cover                 | 31    |
+      | coverage                       | 53.8  |
+      | line_coverage                  | 54.8  |
+      | branch_coverage                | 50    |
+      | uncovered_conditions           | 4     |
+      | uncovered_lines                | 14    |
+      | tests                          | 5     |
+      | test_failures                  | 2     |
+      | test_errors                    | 0     |
+      | skipped_tests                  | 1     |
+      | test_execution_time            | 159   |
+      | test_success_density           | None  |
+      | false_positive_issues          | 0     |
+      | open_issues                    | 12    |
+      | confirmed_issues               | 0     |
+      | reopened_issues                | 0     |
+      | code_smells                    | 6     |
+      | sqale_index                    | 30    |
+      | sqale_debt_ratio               | 1.8   |
+      | bugs                           | 6     |
+      | reliability_remediation_effort | 30    |
+      | vulnerabilities                | 0     |
+      | security_remediation_effort    | 0     |
+      | security_hotspots              | 0     |
+
+
+  Scenario: Smoketest UTF-16LE (with BOM)
+    Given the project "smoketest_project_utf16"
+    And rule "cppcheck:unusedVariable" is activated
+    And rule "cppcheck:unreadVariable" is activated
+    And rule "cppcheck:deallocDealloc" is activated
+    And rule "cppcheck:doubleFree" is activated
+    And rule "cppcheck:uninitvar" is activated
+    And rule "cppcheck:unusedFunction" is activated
+    And rule "cppcheck:missingInclude" is activated
+    When I run sonar-scanner with "-X -Dsonar.sourceEncoding=UTF-16LE"
+    Then the analysis finishes successfully
+    And the analysis in server has completed
+    And the analysis log contains no error/warning messages except those matching:
+      """
+      .*WARN.*Unable to get a valid mac address, will use a dummy address
+      .*WARN.*cannot find the sources for '#include <gtest/gtest\.h>'
+      .*WARN.*cannot find the sources for '#include <iostream>'
+      .*WARN.*Cannot find the file '.*component_XXX.cc', skipping
+      .*WARN.*Cannot find a report for '.*'
+      """
+    And the following metrics have following values:
+      | metric                         | value |
+      | ncloc                          | 56    |
+      | lines                          | 151   |
+      | statements                     | 36    |
+      | classes                        | 1     |
+      | files                          | 8     |
+      | directories                    | None  |
+      | functions                      | 5     |
+      | comment_lines_density          | 30    |
+      | comment_lines                  | 24    |
+      | duplicated_lines_density       | 55.6  |
+      | duplicated_lines               | 84    |
+      | duplicated_blocks              | 2     |
+      | duplicated_files               | 2     |
+      | complexity                     | 7     |
+      | cognitive_complexity           | 2     |
+      | file_complexity                | 0.9   |
+      | violations                     | 12    |
+      | lines_to_cover                 | 31    |
+      | coverage                       | 53.8  |
+      | line_coverage                  | 54.8  |
+      | branch_coverage                | 50    |
+      | uncovered_conditions           | 4     |
+      | uncovered_lines                | 14    |
+      | tests                          | 5     |
+      | test_failures                  | 2     |
+      | test_errors                    | 0     |
+      | skipped_tests                  | 1     |
+      | test_execution_time            | 159   |
+      | test_success_density           | None  |
+      | false_positive_issues          | 0     |
+      | open_issues                    | 12    |
+      | confirmed_issues               | 0     |
+      | reopened_issues                | 0     |
+      | code_smells                    | 6     |
+      | sqale_index                    | 30    |
+      | sqale_debt_ratio               | 1.8   |
+      | bugs                           | 6     |
+      | reliability_remediation_effort | 30    |
+      | vulnerabilities                | 0     |
+      | security_remediation_effort    | 0     |
+      | security_hotspots              | 0     |
+
+
   Scenario: Project using Bullseye coverage, xUnit, Cppcheck, Rats, Vera++
     This test verifies that analysis is able to import Bullseye coverage reports and import custom rules reports.
     Custom rules are created using Rest API, after test ends rules are deleted.

integration-tests/testdata/smoketest_project/Readme.txt

@@ -1,4 +1,5 @@
 This project is used for integration testing. Tested features:
+- files in the folder are UTF-8 without BOM
 - parsing / ast construction / basic metrics like ncloc, classes etc.
 - feeding of issues using:
   - a cppcheck report

integration-tests/testdata/smoketest_project_utf16/sonar-project.properties

@@ -0,0 +1,28 @@
+# metadata
+sonar.projectKey=smoketest_project_utf16
+
+# disable SCM support
+sonar.scm.disabled=true
+
+# disable XML sensor
+sonar.xml.file.suffixes=.disable-xml
+
+# file extensions assigned to the cxx programming language
+sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
+
+# comma-separated paths to directories containing source files
+sonar.sources=src
+
+# comma-separated paths to directories containing test source files
+sonar.tests=tests/unittests
+
+# comma-separated list of directories where the plugin should search for include files
+sonar.cxx.includeDirectories=src,3rdparty
+
+# paths to the reports
+sonar.cxx.cppcheck.reportPaths=build/cppcheck-report.xml
+sonar.cxx.cobertura.reportPaths=build/gcovr-report*.xml
+sonar.cxx.cobertura.itReportPaths=build/gcovr-report*.xml
+sonar.cxx.cobertura.overallReportPaths=build/gcovr-report*.xml
+sonar.cxx.valgrind.reportPaths=build/valgrind-report.xml
+sonar.cxx.xunit.reportPaths=build/xunit-report.xml

integration-tests/testdata/smoketest_project_utf8_bom/3rdparty/extlib.cc

@@ -0,0 +1,3 @@
+int very_useful_function(int a, int b){
+    return a * b;
+}

integration-tests/testdata/smoketest_project_utf8_bom/3rdparty/extlib.hh

@@ -0,0 +1 @@
+int very_useful_function(int a, int b);

integration-tests/testdata/smoketest_project_utf8_bom/Readme.txt

@@ -0,0 +1,7 @@
+This project is used for integration testing. Tested features:
+- files in the folder are UTF-8 with BOM
+- parsing / ast construction / basic metrics like ncloc, classes etc.
+- feeding of issues using:
+  - a cppcheck report
+  - a valgrind report
+- unit test statistics, using the junitReport format

integration-tests/testdata/smoketest_project_utf8_bom/build/cppcheck-report.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<results version="2">
+  <cppcheck version="1.61"/>
+  <errors>
+    <error  id="unusedVariable" severity="style" msg="Unused variable: x">
+      <location file="src/lib/component1.cc" line="17" />
+    </error>
+    <error  id="unreadVariable" severity="style" msg="Variable 'i' is assigned a value that is never used.">
+      <location file="src/lib/component1.cc" line="37" />
+    </error>
+    <error  id="deallocDealloc" severity="error" msg="Deallocating a deallocated pointer: ip">
+      <location file="src/lib/component1.cc" line="47" />
+    </error>
+    <error  id="doubleFree" severity="error" msg="Memory pointed to by 'ip' is freed twice.">
+      <location file="src/lib/component1.cc" line="47" />
+    </error>
+    <error  id="uninitvar" severity="error" msg="Uninitialized variable: a">
+      <location file="src/lib/component1.cc" line="32" />
+    </error>
+    <error  id="unusedVariable" severity="style" msg="Unused variable: x">
+      <location file="src/lib/component2.cc" line="16" />
+    </error>
+    <error  id="unreadVariable" severity="style" msg="Variable 'i' is assigned a value that is never used.">
+      <location file="src/lib/component2.cc" line="36" />
+    </error>
+    <error  id="deallocDealloc" severity="error" msg="Deallocating a deallocated pointer: ip">
+      <location file="src/lib/component2.cc" line="46" />
+    </error>
+    <error   id="doubleFree" severity="error" msg="Memory pointed to by 'ip' is freed twice.">
+      <location file="src/lib/component2.cc" line="46"/>
+    </error>
+    <error  id="uninitvar" severity="error" msg="Uninitialized variable: a">
+      <location file="src/lib/component2.cc" line="31"/>
+    </error>
+    <!-- alternative notation for the file path. -->
+    <error  id="unusedFunction" severity="style" msg="The function 'do_valgrind_errors' is never used.">
+      <location file="./src/lib/component1.cc" line="24" />
+    </error>
+    <!-- Violation for a file which does not exist. Should not be saved. -->
+    <error  id="unusedFunction" severity="style" msg="The function 'do_valgrind_errors' is never used.">
+      <location file="src/lib/component_XXX.cc" line="24" />
+    </error>
+    <error id="missingInclude" severity="style" msg="Cppcheck cannot find all the include files. Cppcheck can check the code without the include files found. But the results will probably be more accurate if all the include files are found. Please check your project's include directories and add all of them as include directories for Cppcheck. To see what files Cppcheck cannot find use --check-config.">
+      <location file="*" />
+    </error>
+  </errors>
+</results>
+

integration-tests/testdata/smoketest_project_utf8_bom/build/gcovr-report.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" ?>
+<!DOCTYPE coverage
+  SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>
+<coverage branch-rate="0.5" line-rate="1.0" timestamp="1405885702" version="gcovr 2.4 (r2774)">
+	<sources>
+		<source>.</source>
+	</sources>
+	<packages>
+		<package branch-rate="0.5" complexity="0.0" line-rate="1.0" name="src.lib">
+			<classes>
+				<class branch-rate="0.5" complexity="0.0" filename="src/lib/component1.cc" line-rate="1.0" name="component1_cc">
+					<lines>
+						<line branch="true" condition-coverage="50% (1/2)" hits="2" number="32">
+							<conditions>
+								<condition coverage="50%" number="0" type="jump"/>
+							</conditions>
+						</line>
+						<line branch="false" hits="2" number="35"/>
+						<line branch="false" hits="2" number="36"/>
+						<line branch="false" hits="2" number="37"/>
+						<line branch="false" hits="2" number="40"/>
+						<line branch="false" hits="2" number="41"/>
+						<line branch="false" hits="2" number="42"/>
+						<line branch="false" hits="4" number="11"/>
+						<line branch="false" hits="2" number="45"/>
+						<line branch="false" hits="2" number="46"/>
+						<line branch="false" hits="2" number="47"/>
+						<line branch="false" hits="2" number="50"/>
+						<line branch="true" condition-coverage="50% (1/2)" hits="2" number="51">
+							<conditions>
+								<condition coverage="50%" number="0" type="jump"/>
+							</conditions>
+						</line>
+						<line branch="false" hits="4" number="20"/>
+						<line branch="false" hits="2" number="24"/>
+						<line branch="true" condition-coverage="50% (2/4)" hits="8" number="52">
+							<conditions>
+								<condition coverage="50%" number="0" type="jump"/>
+							</conditions>
+						</line>
+						<line branch="false" hits="2" number="28"/>
+					</lines>
+				</class>
+			</classes>
+		</package>
+	</packages>
+</coverage>
+

integration-tests/testdata/smoketest_project_utf8_bom/build/xunit-report.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="5" failures="2" disabled="1" errors="0" time="0.244" name="AllTests">
+  <testsuite name="Component1Test" tests="5" failures="2" disabled="1" errors="0" time="0.184">
+    <testcase name="foo_successfull" status="run" time="0.018" classname="Component1Test" />
+    <testcase name="foo_failing" status="run" time="0.066" classname="Component1Test">
+      <failure message="Value of: 112&#x0A;Expected: bar.foo()&#x0A;Which is: 111" type=""><![CDATA[test_component1.cc:18
+Value of: 112
+Expected: bar.foo()
+Which is: 111]]></failure>
+    </testcase>
+    <testcase name="foo_throwing" status="run" time="0.065" classname="Component1Test">
+      <failure message="Unknown C++ exception thrown in the test body." type=""><![CDATA[unknown file
+Unknown C++ exception thrown in the test body.]]></failure>
+    </testcase>
+    <testcase name="DISABLED_foo_skipped" status="notrun" time="0" classname="Component1Test" />
+    <testcase name="foo_leaking" status="run" time="0.01" classname="Component1Test" />
+  </testsuite>
+</testsuites>