Clean up code coverage reporting.

CTest is now run directly and `make coverage` (like `make test`) now expects you to run `make` beforehand, which is more flexible for the user. This patch also reduces clutter by properly excluding unwanted files and reduces the number of explicit exlusion regexes that are required. Gcov reports are still confusing and report very low branch coverage (which is picked up by Codecov, unfortunately), but the llvm-cov reports are nice and clean now.
2024-09-19 20:02:18 +02:00 · 2021-09-27 14:13:30 +02:00 · 2021-09-27 14:13:30 +02:00 · 7ff8720a39
commit 7ff8720a39
parent e6bf8463d9
3 changed files with 42 additions and 30 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -385,20 +385,13 @@ endif(WITH_TESTS)
 if(WITH_COVERAGE)
    # Include code coverage, use with -DCMAKE_BUILD_TYPE=Debug
    include(CodeCoverage)
-    set(COVERAGE_EXCLUDES
-            "\\(.+/\\)?tests/.\\*"
-            "\\(.+/\\)?build/.\\*"
-            "\\(.+/\\)?thirdparty/.\\*"
-            "\\(.+/\\)?CMakeFiles/.\\*"
-            "src/main.cpp"
-            ".\\*/moc_\\[^/\\]+\\.cpp"
-            ".\\*/ui_\\[^/\\]+\\.h"
-            ".\\*/\\[^/\\]+_autogen/.\\*"
-            "\\(.+/\\)?zxcvbn/.\\*"
-            "/Applications/.\\*"
-            "/opt/.\\*")
    append_coverage_compiler_flags()

+    set(COVERAGE_EXCLUDES
+            "'^(.+/)?(thirdparty|zxcvbn)/.*'"
+            "'^(.+/)?main\\.cpp$$'"
+            "'^(.+/)?cli/keepassxc-cli\\.cpp$$'"
+            "'^(.+/)?proxy/keepassxc-proxy\\.cpp$$'")
    if(WITH_COVERAGE AND CMAKE_COMPILER_IS_CLANGXX)
        set(MAIN_BINARIES
                "$<TARGET_FILE:${PROGNAME}>"
@ -406,14 +399,13 @@ if(WITH_COVERAGE)
                "$<TARGET_FILE:keepassxc-proxy>")
        setup_target_for_coverage_llvm(
                NAME coverage
-                EXECUTABLE $(MAKE) test
                BINARY ${MAIN_BINARIES}
-                SOURCES ${CMAKE_SOURCE_DIR}/src
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
        )
    else()
        setup_target_for_coverage_gcovr(
                NAME coverage
-                EXECUTABLE $(MAKE) test
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
        )
    endif()
 endif()
--- a/cmake/CodeCoverage.cmake
+++ b/cmake/CodeCoverage.cmake
@ -45,7 +45,7 @@ elseif(CMAKE_COMPILER_IS_CLANGXX)
 endif()

 set(CMAKE_COVERAGE_FORMAT
-    "html" "txt"
+    "html" "xml"
    CACHE STRING "Coverage report output format.")
 set_property(CACHE CMAKE_COVERAGE_FORMAT PROPERTY STRINGS "html" "txt")

@ -93,7 +93,7 @@ endif()
 function(SETUP_TARGET_FOR_COVERAGE_GCOVR)

    set(options NONE)
-    set(oneValueArgs NAME)
+    set(oneValueArgs NAME SOURCES_ROOT)
    set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
    cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})

@ -110,33 +110,48 @@ function(SETUP_TARGET_FOR_COVERAGE_GCOVR)

    add_custom_target(${Coverage_NAME}
        # Run tests
-        COMMAND $(MAKE)
-        COMMAND ${Coverage_EXECUTABLE}
+        COMMAND ctest -C $<CONFIG> $ENV{ARGS} $$ARGS

-        # Create folder
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}
        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
        DEPENDS ${Coverage_DEPENDENCIES}
    )

    if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Create folder
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}-html
+
            # Running gcovr HTML
            COMMAND ${GCOVR_PATH} --html --html-details
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                -o ${Coverage_NAME}-html/index.html
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
            COMMENT "Running gcovr to produce HTML code coverage report ${Coverage_NAME}-html."
        )
    endif()

+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Running gcovr TXT
+            COMMAND ${GCOVR_PATH} --xml
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
+                --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
+                -o ${Coverage_NAME}.xml
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+            COMMENT "Running gcovr to produce XML code coverage report ${Coverage_NAME}.xml."
+        )
+    endif()
+
    if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
-            # Running gcovr XML
+            # Running gcovr TXT
            COMMAND ${GCOVR_PATH}
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                -o ${Coverage_NAME}.txt
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
            COMMENT "Running gcovr to produce TXT code coverage report ${Coverage_NAME}.txt."
@ -158,8 +173,8 @@ endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR
 function(SETUP_TARGET_FOR_COVERAGE_LLVM)

    set(options NONE)
-    set(oneValueArgs NAME PROF_FILE)
-    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS SOURCES DEPENDENCIES)
+    set(oneValueArgs NAME SOURCES_ROOT PROF_FILE)
+    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS DEPENDENCIES)
    cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})

    if(XCRUN_PATH)
@ -191,8 +206,7 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
    endif()

    add_custom_target(${Coverage_NAME}
-        COMMAND $(MAKE)
-        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ${Coverage_EXECUTABLE}
+        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ctest -C $<CONFIG> $$ARGS

        COMMAND ${LLVM_PROFDATA_PATH} merge -sparse ${LLVM_PROFILE_DIR}/* -o coverage.profdata
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
@ -201,15 +215,19 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
    if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
            COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES}
+                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT}
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
            COMMENT "Running llvm-cov to produce HTML code coverage report ${Coverage_NAME}-html")
    endif()

+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        message(WARNING "XML coverage report format not supported for llvm-cov")
+    endif()
+
    if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
            COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES} > ${Coverage_NAME}.txt
+                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT} > ${Coverage_NAME}.txt

            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
            COMMENT "Running llvm-cov to produce TXT code coverage report ${Coverage_NAME}.txt.")
--- a/codecov.yaml
+++ b/codecov.yaml
@ -2,5 +2,7 @@ coverage:
  range: 60..80
  round: nearest
  precision: 2
+fixes:
+  - "*/src/::"
 comment:
  require_changes: true