Clean up code coverage reporting.

CTest is now run directly and `make coverage` (like `make test`) now
expects you to run `make` beforehand, which is more flexible for the
user. This patch also reduces clutter by properly excluding unwanted
files and reduces the number of explicit exlusion regexes that are
required.

Gcov reports are still confusing and report very low branch coverage
(which is picked up by Codecov, unfortunately), but the llvm-cov reports
are nice and clean now.

CMakeLists.txt

@@ -385,20 +385,13 @@ endif(WITH_TESTS)
 if(WITH_COVERAGE)
     # Include code coverage, use with -DCMAKE_BUILD_TYPE=Debug
     include(CodeCoverage)
-    set(COVERAGE_EXCLUDES
-            "\\(.+/\\)?tests/.\\*"
-            "\\(.+/\\)?build/.\\*"
-            "\\(.+/\\)?thirdparty/.\\*"
-            "\\(.+/\\)?CMakeFiles/.\\*"
-            "src/main.cpp"
-            ".\\*/moc_\\[^/\\]+\\.cpp"
-            ".\\*/ui_\\[^/\\]+\\.h"
-            ".\\*/\\[^/\\]+_autogen/.\\*"
-            "\\(.+/\\)?zxcvbn/.\\*"
-            "/Applications/.\\*"
-            "/opt/.\\*")
     append_coverage_compiler_flags()
 
+    set(COVERAGE_EXCLUDES
+            "'^(.+/)?(thirdparty|zxcvbn)/.*'"
+            "'^(.+/)?main\\.cpp$$'"
+            "'^(.+/)?cli/keepassxc-cli\\.cpp$$'"
+            "'^(.+/)?proxy/keepassxc-proxy\\.cpp$$'")
     if(WITH_COVERAGE AND CMAKE_COMPILER_IS_CLANGXX)
         set(MAIN_BINARIES
                 "$<TARGET_FILE:${PROGNAME}>"
@@ -406,14 +399,13 @@ if(WITH_COVERAGE)
                 "$<TARGET_FILE:keepassxc-proxy>")
         setup_target_for_coverage_llvm(
                 NAME coverage
-                EXECUTABLE $(MAKE) test
                 BINARY ${MAIN_BINARIES}
-                SOURCES ${CMAKE_SOURCE_DIR}/src
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
         )
     else()
         setup_target_for_coverage_gcovr(
                 NAME coverage
-                EXECUTABLE $(MAKE) test
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
         )
     endif()
 endif()

cmake/CodeCoverage.cmake

@@ -45,7 +45,7 @@ elseif(CMAKE_COMPILER_IS_CLANGXX)
 endif()
 
 set(CMAKE_COVERAGE_FORMAT
-    "html" "txt"
+    "html" "xml"
     CACHE STRING "Coverage report output format.")
 set_property(CACHE CMAKE_COVERAGE_FORMAT PROPERTY STRINGS "html" "txt")
 
@@ -93,7 +93,7 @@ endif()
 function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
 
     set(options NONE)
-    set(oneValueArgs NAME)
+    set(oneValueArgs NAME SOURCES_ROOT)
     set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
     cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
@@ -110,33 +110,48 @@ function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
 
     add_custom_target(${Coverage_NAME}
         # Run tests
-        COMMAND $(MAKE)
+        COMMAND ctest -C $<CONFIG> $ENV{ARGS} $$ARGS
-        COMMAND ${Coverage_EXECUTABLE}
 
-        # Create folder
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}
         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
         DEPENDS ${Coverage_DEPENDENCIES}
     )
 
     if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Create folder
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}-html
+
             # Running gcovr HTML
             COMMAND ${GCOVR_PATH} --html --html-details
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                 --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                 -o ${Coverage_NAME}-html/index.html
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running gcovr to produce HTML code coverage report ${Coverage_NAME}-html."
         )
     endif()
 
+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Running gcovr TXT
+            COMMAND ${GCOVR_PATH} --xml
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
+                --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
+                -o ${Coverage_NAME}.xml
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+            COMMENT "Running gcovr to produce XML code coverage report ${Coverage_NAME}.xml."
+        )
+    endif()
+
     if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
-            # Running gcovr XML
+            # Running gcovr TXT
             COMMAND ${GCOVR_PATH}
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                 --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                 -o ${Coverage_NAME}.txt
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running gcovr to produce TXT code coverage report ${Coverage_NAME}.txt."
@@ -158,8 +173,8 @@ endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR
 function(SETUP_TARGET_FOR_COVERAGE_LLVM)
 
     set(options NONE)
-    set(oneValueArgs NAME PROF_FILE)
+    set(oneValueArgs NAME SOURCES_ROOT PROF_FILE)
-    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS SOURCES DEPENDENCIES)
+    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS DEPENDENCIES)
     cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
     if(XCRUN_PATH)
@@ -191,8 +206,7 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
     endif()
 
     add_custom_target(${Coverage_NAME}
-        COMMAND $(MAKE)
+        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ctest -C $<CONFIG> $$ARGS
-        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ${Coverage_EXECUTABLE}
 
         COMMAND ${LLVM_PROFDATA_PATH} merge -sparse ${LLVM_PROFILE_DIR}/* -o coverage.profdata
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
@@ -201,15 +215,19 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
     if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
             COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES}
+                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT}
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running llvm-cov to produce HTML code coverage report ${Coverage_NAME}-html")
     endif()
 
+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        message(WARNING "XML coverage report format not supported for llvm-cov")
+    endif()
+
     if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
             COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES} > ${Coverage_NAME}.txt
+                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT} > ${Coverage_NAME}.txt
 
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running llvm-cov to produce TXT code coverage report ${Coverage_NAME}.txt.")

codecov.yaml

@@ -2,5 +2,7 @@ coverage:
   range: 60..80
   round: nearest
   precision: 2
+fixes:
+  - "*/src/::"
 comment:
   require_changes: true