Support #1512: Measure coverage for tests and examples 44/2844/11
authorAnirudh Manoj <anirudh@hpccharm.com>
Mon, 24 Jul 2017 18:47:48 +0000 (13:47 -0500)
committerEric Bohm <ericbohm@gmail.com>
Fri, 13 Jul 2018 19:18:00 +0000 (14:18 -0500)
This includes a script (coverage/coverage.sh) to build charm with
--coverage, run make test on tests and examples, then assemble the
results using lcov and provide browsable analysis using gen-html.

By default it checks netlrts-linux-x86_64-smp, but any number of
targets can be run by putting them on the command line.

The total coverage for ck.C and convcore.c is summarized in summary.txt

The verbose file by file result can be browsed in
coverage/netlrts-linux-x86_64-smp-cov-html/tmp/index.html

or by platform name from the command line arguments
coverage/[platform-name]-cov-html/tmp/index.html

Change-Id: I4bf9ae1dd767ac066585afa30933eb820af79095

coverage/ck-convcore-summary.C [new file with mode: 0644]
coverage/coverage.sh [new file with mode: 0755]
coverage/tests.desc [new file with mode: 0644]

diff --git a/coverage/ck-convcore-summary.C b/coverage/ck-convcore-summary.C
new file mode 100644 (file)
index 0000000..f02a494
--- /dev/null
@@ -0,0 +1,74 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+    if(argc != 2) {
+        fprintf(stderr, "Usage:\n\tck-convcore [lcov trace file]\n");
+        exit(1);
+    }
+
+    FILE *tracefile = fopen(argv[1], "r");
+    if(tracefile == NULL) {
+        fprintf(stderr, "Error: couldn't open tracefile %s\n", argv[1]);
+        exit(1);
+    }
+
+    int inCK = 0, inCONV = 0, finishedCK = 0, finishedCONV = 0;
+    int FNFck = 0, FNHck = 0, LHck = 0, LFck = 0;
+    int FNFconv = 0, FNHconv = 0, LHconv = 0, LFconv = 0;
+    char *lineptr = NULL;
+    size_t n = 0;
+    while(getline(&lineptr, &n, tracefile) != -1) {
+        if(inCK) {
+            if(lineptr[0] == 'F' && lineptr[1] == 'N' && lineptr[2] == 'F')
+                FNFck = atoi(lineptr + 4);
+            else if(lineptr[0] == 'F' && lineptr[1] == 'N' && lineptr[2] == 'H')
+                FNHck = atoi(lineptr + 4);
+            else if(lineptr[0] == 'L' && lineptr[1] == 'F')
+                LFck = atoi(lineptr + 3);
+            else if(lineptr[0] == 'L' && lineptr[1] == 'H') {
+                LHck = atoi(lineptr + 3);
+                inCK = 0;
+                finishedCK = 1;
+                if(finishedCK && finishedCONV)
+                    break;
+            }
+        } else if(inCONV) {
+            if(lineptr[0] == 'F' && lineptr[1] == 'N' && lineptr[2] == 'F')
+                FNFconv = atoi(lineptr + 4);
+            else if(lineptr[0] == 'F' && lineptr[1] == 'N' && lineptr[2] == 'H')
+                FNHconv = atoi(lineptr + 4);
+            else if(lineptr[0] == 'L' && lineptr[1] == 'F')
+                LFconv = atoi(lineptr + 3);
+            else if(lineptr[0] == 'L' && lineptr[1] == 'H') {
+                LHconv = atoi(lineptr + 3);
+                inCONV = 0;
+                finishedCONV = 1;
+                if(finishedCK && finishedCONV)
+                    break;
+            }
+        } else if(lineptr[0] == 'S' && lineptr[1] == 'F') {
+            if(strstr(lineptr, "tmp/ck.C") != NULL)
+                inCK = 1;
+            else if(strstr(lineptr, "tmp/convcore.c") != NULL)
+                inCONV = 1;
+        }
+
+        free(lineptr);
+        lineptr = NULL;
+        n = 0;
+    }
+
+    free(lineptr);
+    lineptr = NULL;
+    n = 0;
+
+    printf("Summary ck.C in %s:\n", argv[1]);
+    printf("\tlines......: %.1f%% (%d of %d lines)\n", ((double)(100 * LHck) / LFck), LHck, LFck);
+    printf("\tfunctions..: %.1f%% (%d of %d functions)\n", ((double)(100 * FNHck) / FNFck), FNHck, FNFck);
+
+    printf("Summary convcore.c in %s:\n", argv[1]);
+    printf("\tlines......: %.1f%% (%d of %d lines)\n", ((double)(100 * LHconv) / LFconv), LHconv, LFconv);
+    printf("\tfunctions..: %.1f%% (%d of %d functions)\n", ((double)(100 * FNHconv) / FNFconv), FNHconv, FNFconv);
+}
diff --git a/coverage/coverage.sh b/coverage/coverage.sh
new file mode 100755 (executable)
index 0000000..41adda7
--- /dev/null
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# it is possible to run the coverage across multiple builds, but the
+# coverage process is designed to report that SomeBinary has coverage
+# percentage Y.  Not, source file X has sum(across all compilations
+# and tests) coverage.  So what you end up with is a discrete coverage
+# calculation for each platform for each source file.  Which makes it
+# pretty pointless to run this on more than one platform at a time.
+
+# However, if you want to build a bunch at one, here is a representative set
+#declare -a BUILDS=("multicore-linux-x86_64" "netlrts-linux-x86_64" "netlrts-linux-x86_64-smp")
+
+if [[ "$#" -gt 0 ]]; then
+    declare -a BUILDS
+    while [[ "$#" > 0 ]]
+    do
+        BUILDS+=("$1");
+        shift;
+        echo "BUILD now $BUILDS";
+        echo "ARGC now $#";
+    done
+else
+    declare -a BUILDS=("netlrts-linux-x86_64-smp")
+fi
+
+echo "testing builds $BUILDS"
+
+/bin/rm -f summary.txt
+
+covdir="$PWD"
+
+for b in "${BUILDS[@]}" ; do
+    covbuild="$b-covbuild"
+    echo "coverage base directory $covdir buildir $covbuild"
+    /bin/rm -f "$b"-cov.info
+    cd ../
+    if [ -d "$covbuild" ] ; then
+        /bin/rm -rf "$covbuild"
+    fi
+    ./build LIBS "$b" -j8 --suffix "covbuild" --coverage -O0 -save #-fprofile-update=atomic
+    cd "$covbuild/tests/"
+    make -j8 OPTS+="--coverage -save"
+    cd ../examples
+    make -j8 OPTS+="--coverage -save"
+    cd ../tmp/
+    make -j8 test
+    testname="$(cut -d'-' -f1 <<<"$b")"
+    testsuff="$(cut -d'-' -f4 <<<"$b")"
+    if [ "$testsuff" = "smp" ] ; then
+       testname=$testsuff
+    fi
+    echo "test $testname"
+    cd ../..
+    lcov --capture --directory "$covbuild"/tmp --rc lcov_branch_coverage=1 --output-file "$covdir"/"$b"-cov.info --test-name "$testname"
+    # lcov --add-tracefile "$b"-cov.info --output-file coverage/"$b"-total.info >> summary.txt
+    cd "$covdir"
+    genhtml "$covdir/$b-cov.info" --description-file tests.desc -o "$covdir/$b"-cov-html >> gensummary.txt
+done
+
+
+/bin/rm -f ckconv
+
+gcc ck-convcore-summary.C -o ckconv
+for b in "${BUILDS[@]}" ; do
+    ./ckconv "$b"-cov.info >> summary.txt
+done
diff --git a/coverage/tests.desc b/coverage/tests.desc
new file mode 100644 (file)
index 0000000..c6fddd2
--- /dev/null
@@ -0,0 +1,6 @@
+TN: smp
+TD: netlrts-linux-x86_64-smp SMP
+TN: multicore
+TD: multicore-linux-x86_64 multicore
+TN: netlrts
+TD: netlrts-linux-x86_64 non-smp