From a56f50ac462eebd8c839053d13f77bfc8c1d7076 Mon Sep 17 00:00:00 2001
From: Alexander Vasilyev <mizabrik@gmail.com>
Date: Mon, 23 Mar 2020 23:49:51 +0300
Subject: [PATCH] Calculate coverage of private code.

Create empty coverage profile from problem code to calculate coverage
correctly. For some wierd reasons it will also account for not tested packages.
This change prevents students from adding new code and testing it,
instead of getting good coverage for initial codebase :)
---
 tools/testtool/commands/coverage.go        | 45 +++++++++++++++-------
 tools/testtool/commands/test_submission.go | 31 ++++++++++++---
 2 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/tools/testtool/commands/coverage.go b/tools/testtool/commands/coverage.go
index 4a824c8..9f4710e 100644
--- a/tools/testtool/commands/coverage.go
+++ b/tools/testtool/commands/coverage.go
@@ -83,15 +83,34 @@ func searchCoverageComment(fname string) (*CoverageRequirements, error) {
 	return &CoverageRequirements{}, nil
 }
 
-// calCoverage calculates coverage percent for given coverage profile.
-func calCoverage(fileNames []string) (float64, error) {
-	type block struct {
+// calCoverage calculates coverage percent of code blocks recorded in targetProfile
+// by given coverage profiles.
+func calCoverage(targetProfile string, fileNames []string) (float64, error) {
+	type key struct {
 		fileName            string
 		startLine, startCol int
 		endLine, endCol     int
 		numStmt             int
 	}
-	counts := map[block]int{}
+	newKey := func(p *cover.Profile, b cover.ProfileBlock) key {
+		return key{
+			p.FileName,
+			b.StartLine, b.StartCol,
+			b.EndLine, b.EndCol,
+			b.NumStmt,
+		}
+	}
+
+	executed := map[key]bool{}
+	targetProfiles, err := cover.ParseProfiles(targetProfile)
+	if err != nil {
+		return 0.0, fmt.Errorf("cannot parse target profile %s: %w", targetProfile, err)
+	}
+	for _, p := range targetProfiles {
+		for _, b := range p.Blocks {
+			executed[newKey(p, b)] = false
+		}
+	}
 
 	for _, f := range fileNames {
 		profiles, err := cover.ParseProfiles(f)
@@ -101,21 +120,19 @@ func calCoverage(fileNames []string) (float64, error) {
 
 		for _, p := range profiles {
 			for _, b := range p.Blocks {
-				counts[block{
-					p.FileName,
-					b.StartLine, b.StartCol,
-					b.EndLine, b.EndCol,
-					b.NumStmt,
-				}] += b.Count
+				k := newKey(p, b)
+				if _, ok := executed[k]; ok && b.Count > 0 {
+					executed[k] = true
+				}
 			}
 		}
 	}
 
 	var total, covered int
-	for b, count := range counts {
-		total += b.numStmt
-		if count > 0 {
-			covered += b.numStmt
+	for k, e := range executed {
+		total += k.numStmt
+		if e {
+			covered += k.numStmt
 		}
 	}
 
diff --git a/tools/testtool/commands/test_submission.go b/tools/testtool/commands/test_submission.go
index 1e744ef..d23b22e 100644
--- a/tools/testtool/commands/test_submission.go
+++ b/tools/testtool/commands/test_submission.go
@@ -243,8 +243,12 @@ func runTests(testDir, privateRepo, problem string) error {
 	}
 
 	coverageReq := getCoverageRequirements(path.Join(privateRepo, problem))
+	coveragePackages := []string{}
 	if coverageReq.Enabled {
 		log.Printf("required coverage: %.2f%%", coverageReq.Percent)
+		for _, pkg := range coverageReq.Packages {
+			coveragePackages = append(coveragePackages, path.Join(moduleImportPath, problem, pkg))
+		}
 	}
 
 	binariesJSON, _ := json.Marshal(binaries)
@@ -254,11 +258,7 @@ func runTests(testDir, privateRepo, problem string) error {
 		testBinaries[testPkg] = binPath
 		cmd := []string{"test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg}
 		if coverageReq.Enabled {
-			pkgs := make([]string, len(coverageReq.Packages))
-			for i, pkg := range coverageReq.Packages {
-				pkgs[i] = path.Join(moduleImportPath, problem, pkg)
-			}
-			cmd = append(cmd, "-cover", "-coverpkg", strings.Join(pkgs, ","))
+			cmd = append(cmd, "-cover", "-coverpkg", strings.Join(coveragePackages, ","))
 		}
 		if err := runGo(cmd...); err != nil {
 			return fmt.Errorf("error building test in %s: %w", testPkg, err)
@@ -324,7 +324,26 @@ func runTests(testDir, privateRepo, problem string) error {
 	if coverageReq.Enabled {
 		log.Printf("checking coverage is at least %.2f%%...", coverageReq.Percent)
 
-		percent, err := calCoverage(coverProfiles)
+		// For some reason, this command will record all coverage blocks in coverpkg,
+		// even if no test binaries depend on given package.
+		// Hacky way to record all the code present in problem definition.
+		targetProfile := path.Join(os.TempDir(), randomName())
+		coverCmd := exec.Command("go",
+			"test",
+			"-coverpkg", strings.Join(coveragePackages, ","),
+			"-coverprofile", targetProfile,
+			"-run", "^$",
+			"./...",
+		)
+		coverCmd.Env = append(os.Environ(), "GOFLAGS=")
+		coverCmd.Dir = path.Join(privateRepo, problem)
+		coverCmd.Stderr = os.Stderr
+		log.Printf("> %s", strings.Join(coverCmd.Args, " "))
+		if err := coverCmd.Run(); err != nil {
+			return fmt.Errorf("error getting target coverage profile: %w", err)
+		}
+
+		percent, err := calCoverage(targetProfile, coverProfiles)
 		if err != nil {
 			return err
 		}