合并多个构建时代码覆盖率错误
Wrong code coverage when merging several builds
在我们的项目中，我们遇到了 JaCoCo 使用 Jenkins 或 SonarQube 计算错误覆盖率的问题。
这是我们常用的 Jenkins 构建脚本示例：
<property name="demo" location="."/>
<property name="exec_name" value="build1.exec"/>
<import file= "${demo}/buildscripts/common/product_common_ant.xml"/>
<property name="coverage_data" location="\server\jenkins\jobs\Test-Coverage\coverage_data\JobName"/>
<target name="testrun" depends="compile" description="Execute tests with coverage">
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="UnitTests" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/unitTests.xml"/>
</testng>
</jacoco:coverage>
<copy file="${test-output}/testng-results.xml" tofile = "${test-output}/unit-test-results.xml"/>
<fail if="testng.failure"/>
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="IntegrationTests" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/integTests_Build.xml"/>
</testng>
</jacoco:coverage>
<copy file = "${test-output}/testng-results.xml" tofile = "${test-output}/integ-test-results.xml"/>
<fail if="testng.failure"/>
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="BUildGUITests1" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/funcTests_build_part01.xml"/>
</testng>
</jacoco:coverage>
<copy file = "${test-output}/testng-results.xml" tofile = "${test-output}/gui-test-results.xml"/>
<fail if="testng.failure"/>
</target>
<target name="mergeTestResults" depends="testrun, copyCoverageResults">
<copy file="${buildscripts}/empty-test-results.xml" tofile="${test-output}/testng-merge.xml"/>
<for param="xmlFile">
<path>
<fileset dir="${test-output}" >
<include name="unit-test-results.xml" />
<include name="integ-test-results.xml" />
<include name="gui-test-results.xml" />
</fileset>
</path>
<sequential>
<copy file="${test-output}/testng-merge.xml" toFile="${test-output}/start-merge.xml"/>
<xslt style="${buildscripts}/merge-test-results.xsl"
destdir="${test-output}"
in="${test-output}/start-merge.xml"
out="${test-output}/testng-merge.xml">
<param name="with" expression="@{xmlFile}" />
</xslt>
<delete file="${test-output}/start-merge.xml" />
</sequential>
</for>
</target>
所以有例如3 个套件，其中 运行 在此构建和覆盖数据中保存到
\server\jenkins\jobs\Test-Coverage\coverage_data\JobName\buildX.exec
覆盖构建具有以下脚本
<project name="Product Test Coverage Build" default="report" basedir="..\.." xmlns:jacoco="antlib:org.jacoco.ant">
<echo>Current username: ${user.name}</echo>
<defaultexcludes echo="true"/>
<property name="demo" location="."/>
<property name="src" location="${demo}/src"/>
<property name="bin" location="${demo}/bin"/>
<property name="lib1" location="${demo}/lib"/>
<property name="lib2" location="${demo}/lib64"/>
<property name="coverage" location="${demo}/coverage"/>
<property name="pr11_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part1-64\"/>
<property name="pr12_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part2-64\"/>
<property name="pr13_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part3-64\"/>
<property name="pr14_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part4-64\"/>
<property name="pr15_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part5-64\"/>
<property name="pr16_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part6-64\"/>
<property name="pr17_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part7-64\"/>
<property name="pr18_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part8-64\"/>
<property name="pr19_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part9-64\"/>
<property name="pr21_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part1-64\"/>
<property name="pr22_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part2-64\"/>
<property name="pr23_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part3-32\"/>
<property name="pr24_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part4-64\"/>
<property name="pr25_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part5-64\"/>
<property name="pr26_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part6-64\"/>
<property name="pr27_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part7-32\"/>
<property name="pr28_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part8-64\"/>
<property name="pr210_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part10-64\"/>
<property name="pr31_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-Unit-Integ-Tests\"/>
<property name="pr32_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-32\"/>
<property name="pr33_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-Integration-1\"/>
<taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml" classpathref="testrun_classpath"/>
<target name="merge" depends ="compile">
<jacoco:merge destfile="${coverage}/merged.exec">
<fileset dir="${pr11_coverage}" includes="*.exec"/>
<fileset dir="${pr12_coverage}" includes="*.exec"/>
<fileset dir="${pr13_coverage}" includes="*.exec"/>
<fileset dir="${pr14_coverage}" includes="*.exec"/>
<fileset dir="${pr15_coverage}" includes="*.exec"/>
<fileset dir="${pr16_coverage}" includes="*.exec"/>
<fileset dir="${pr17_coverage}" includes="*.exec"/>
<fileset dir="${pr18_coverage}" includes="*.exec"/>
<fileset dir="${pr19_coverage}" includes="*.exec"/>
<fileset dir="${pr21_coverage}" includes="*.exec"/>
<fileset dir="${pr22_coverage}" includes="*.exec"/>
<fileset dir="${pr23_coverage}" includes="*.exec"/>
<fileset dir="${pr24_coverage}" includes="*.exec"/>
<fileset dir="${pr25_coverage}" includes="*.exec"/>
<fileset dir="${pr26_coverage}" includes="*.exec"/>
<fileset dir="${pr27_coverage}" includes="*.exec"/>
<fileset dir="${pr28_coverage}" includes="*.exec"/>
<fileset dir="${pr210_coverage}" includes="*.exec"/>
<fileset dir="${pr31_coverage}" includes="*.exec"/>
<fileset dir="${pr32_coverage}" includes="*.exec"/>
<fileset dir="${pr33_coverage}" includes="*.exec"/>
</jacoco:merge>
</target>
<property environment="env"/>
<property name="coverage_rep" location="\server\jenkins\jobs\Product-Test-Coverage\builds${env.BUILD_NUMBER}\"/>
<target name="report" depends="merge">
<jacoco:report>
<executiondata>
<file file="${coverage}/merged.exec"/>
</executiondata>
<structure name="Product Overall Code Coverage">
<sourcefiles encoding="UTF-8">
<fileset dir="${src}"/>
</sourcefiles>
</structure>
<html destdir="${coverage}"/>
<csv destfile="${coverage}/report.csv"/>
<xml destfile="${coverage}/report.xml"/>
</jacoco:report>
<copy file="${coverage}/index.html" tofile="${coverage_rep}/index.html"/>
</target>
我把编译构建的部分去掉了
当 Jenkins 或 SonarQube 运行 这个脚本时，我们得到的代码覆盖率大约为 19%，而 运行 在 Eclipse 中本地化所有测试并使用 Eclemma（与 Jacoco）合并给我们大约 65 %（这是更可靠的结果）。
我们的构建在每次 SVN 提交时 运行 并在每次 运行 之前编译。所有测试服务器都使用相同的 Java 版本，尽管同时使用 32 位和 64 位架构。
我们做错了什么？
一年多之后，代码覆盖率突然又变得正确了。奇怪，不幸的是没有办法找出之前的错误。
在我们的项目中，我们遇到了 JaCoCo 使用 Jenkins 或 SonarQube 计算错误覆盖率的问题。
这是我们常用的 Jenkins 构建脚本示例：
<property name="demo" location="."/>
<property name="exec_name" value="build1.exec"/>
<import file= "${demo}/buildscripts/common/product_common_ant.xml"/>
<property name="coverage_data" location="\server\jenkins\jobs\Test-Coverage\coverage_data\JobName"/>
<target name="testrun" depends="compile" description="Execute tests with coverage">
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="UnitTests" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/unitTests.xml"/>
</testng>
</jacoco:coverage>
<copy file="${test-output}/testng-results.xml" tofile = "${test-output}/unit-test-results.xml"/>
<fail if="testng.failure"/>
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="IntegrationTests" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/integTests_Build.xml"/>
</testng>
</jacoco:coverage>
<copy file = "${test-output}/testng-results.xml" tofile = "${test-output}/integ-test-results.xml"/>
<fail if="testng.failure"/>
<jacoco:coverage destfile="${cov_res_file}" append="true">
<testng classpathref="testrun_classpath" suitename="BUildGUITests1" failureProperty="testng.failure" verbose="2" outputdir="${test-output}" workingDir="${demo}">
<xmlfileset dir="${test}" includes="builds/funcTests_build_part01.xml"/>
</testng>
</jacoco:coverage>
<copy file = "${test-output}/testng-results.xml" tofile = "${test-output}/gui-test-results.xml"/>
<fail if="testng.failure"/>
</target>
<target name="mergeTestResults" depends="testrun, copyCoverageResults">
<copy file="${buildscripts}/empty-test-results.xml" tofile="${test-output}/testng-merge.xml"/>
<for param="xmlFile">
<path>
<fileset dir="${test-output}" >
<include name="unit-test-results.xml" />
<include name="integ-test-results.xml" />
<include name="gui-test-results.xml" />
</fileset>
</path>
<sequential>
<copy file="${test-output}/testng-merge.xml" toFile="${test-output}/start-merge.xml"/>
<xslt style="${buildscripts}/merge-test-results.xsl"
destdir="${test-output}"
in="${test-output}/start-merge.xml"
out="${test-output}/testng-merge.xml">
<param name="with" expression="@{xmlFile}" />
</xslt>
<delete file="${test-output}/start-merge.xml" />
</sequential>
</for>
</target>
所以有例如3 个套件，其中 运行 在此构建和覆盖数据中保存到
\server\jenkins\jobs\Test-Coverage\coverage_data\JobName\buildX.exec
覆盖构建具有以下脚本
<project name="Product Test Coverage Build" default="report" basedir="..\.." xmlns:jacoco="antlib:org.jacoco.ant">
<echo>Current username: ${user.name}</echo>
<defaultexcludes echo="true"/>
<property name="demo" location="."/>
<property name="src" location="${demo}/src"/>
<property name="bin" location="${demo}/bin"/>
<property name="lib1" location="${demo}/lib"/>
<property name="lib2" location="${demo}/lib64"/>
<property name="coverage" location="${demo}/coverage"/>
<property name="pr11_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part1-64\"/>
<property name="pr12_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part2-64\"/>
<property name="pr13_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part3-64\"/>
<property name="pr14_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part4-64\"/>
<property name="pr15_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part5-64\"/>
<property name="pr16_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part6-64\"/>
<property name="pr17_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part7-64\"/>
<property name="pr18_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part8-64\"/>
<property name="pr19_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product-Part9-64\"/>
<property name="pr21_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part1-64\"/>
<property name="pr22_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part2-64\"/>
<property name="pr23_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part3-32\"/>
<property name="pr24_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part4-64\"/>
<property name="pr25_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part5-64\"/>
<property name="pr26_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part6-64\"/>
<property name="pr27_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part7-32\"/>
<property name="pr28_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part8-64\"/>
<property name="pr210_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product2-Part10-64\"/>
<property name="pr31_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-Unit-Integ-Tests\"/>
<property name="pr32_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-32\"/>
<property name="pr33_coverage" location="\server\jenkins\jobs\Product-Test-Coverage\coverage_data\Product3-Integration-1\"/>
<taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml" classpathref="testrun_classpath"/>
<target name="merge" depends ="compile">
<jacoco:merge destfile="${coverage}/merged.exec">
<fileset dir="${pr11_coverage}" includes="*.exec"/>
<fileset dir="${pr12_coverage}" includes="*.exec"/>
<fileset dir="${pr13_coverage}" includes="*.exec"/>
<fileset dir="${pr14_coverage}" includes="*.exec"/>
<fileset dir="${pr15_coverage}" includes="*.exec"/>
<fileset dir="${pr16_coverage}" includes="*.exec"/>
<fileset dir="${pr17_coverage}" includes="*.exec"/>
<fileset dir="${pr18_coverage}" includes="*.exec"/>
<fileset dir="${pr19_coverage}" includes="*.exec"/>
<fileset dir="${pr21_coverage}" includes="*.exec"/>
<fileset dir="${pr22_coverage}" includes="*.exec"/>
<fileset dir="${pr23_coverage}" includes="*.exec"/>
<fileset dir="${pr24_coverage}" includes="*.exec"/>
<fileset dir="${pr25_coverage}" includes="*.exec"/>
<fileset dir="${pr26_coverage}" includes="*.exec"/>
<fileset dir="${pr27_coverage}" includes="*.exec"/>
<fileset dir="${pr28_coverage}" includes="*.exec"/>
<fileset dir="${pr210_coverage}" includes="*.exec"/>
<fileset dir="${pr31_coverage}" includes="*.exec"/>
<fileset dir="${pr32_coverage}" includes="*.exec"/>
<fileset dir="${pr33_coverage}" includes="*.exec"/>
</jacoco:merge>
</target>
<property environment="env"/>
<property name="coverage_rep" location="\server\jenkins\jobs\Product-Test-Coverage\builds${env.BUILD_NUMBER}\"/>
<target name="report" depends="merge">
<jacoco:report>
<executiondata>
<file file="${coverage}/merged.exec"/>
</executiondata>
<structure name="Product Overall Code Coverage">
<sourcefiles encoding="UTF-8">
<fileset dir="${src}"/>
</sourcefiles>
</structure>
<html destdir="${coverage}"/>
<csv destfile="${coverage}/report.csv"/>
<xml destfile="${coverage}/report.xml"/>
</jacoco:report>
<copy file="${coverage}/index.html" tofile="${coverage_rep}/index.html"/>
</target>
我把编译构建的部分去掉了
当 Jenkins 或 SonarQube 运行 这个脚本时，我们得到的代码覆盖率大约为 19%，而 运行 在 Eclipse 中本地化所有测试并使用 Eclemma（与 Jacoco）合并给我们大约 65 %（这是更可靠的结果）。
我们的构建在每次 SVN 提交时 运行 并在每次 运行 之前编译。所有测试服务器都使用相同的 Java 版本，尽管同时使用 32 位和 64 位架构。
我们做错了什么？
一年多之后，代码覆盖率突然又变得正确了。奇怪，不幸的是没有办法找出之前的错误。