Calculate coverage of private code.

Create empty coverage profile from problem code to calculate coverage
correctly. For some wierd reasons it will also account for not tested packages.
This change prevents students from adding new code and testing it,
instead of getting good coverage for initial codebase :)

tools/testtool/commands/coverage.go

@@ -83,15 +83,34 @@ func searchCoverageComment(fname string) (*CoverageRequirements, error) {
 	return &CoverageRequirements{}, nil
 }
 
-// calCoverage calculates coverage percent for given coverage profile.
+// calCoverage calculates coverage percent of code blocks recorded in targetProfile
-func calCoverage(fileNames []string) (float64, error) {
+// by given coverage profiles.
-	type block struct {
+func calCoverage(targetProfile string, fileNames []string) (float64, error) {
+	type key struct {
 		fileName            string
 		startLine, startCol int
 		endLine, endCol     int
 		numStmt             int
 	}
-	counts := map[block]int{}
+	newKey := func(p *cover.Profile, b cover.ProfileBlock) key {
+		return key{
+			p.FileName,
+			b.StartLine, b.StartCol,
+			b.EndLine, b.EndCol,
+			b.NumStmt,
+		}
+	}
+
+	executed := map[key]bool{}
+	targetProfiles, err := cover.ParseProfiles(targetProfile)
+	if err != nil {
+		return 0.0, fmt.Errorf("cannot parse target profile %s: %w", targetProfile, err)
+	}
+	for _, p := range targetProfiles {
+		for _, b := range p.Blocks {
+			executed[newKey(p, b)] = false
+		}
+	}
 
 	for _, f := range fileNames {
 		profiles, err := cover.ParseProfiles(f)
@@ -101,21 +120,19 @@ func calCoverage(fileNames []string) (float64, error) {
 
 		for _, p := range profiles {
 			for _, b := range p.Blocks {
-				counts[block{
+				k := newKey(p, b)
-					p.FileName,
+				if _, ok := executed[k]; ok && b.Count > 0 {
-					b.StartLine, b.StartCol,
+					executed[k] = true
-					b.EndLine, b.EndCol,
+				}
-					b.NumStmt,
-				}] += b.Count
 			}
 		}
 	}
 
 	var total, covered int
-	for b, count := range counts {
+	for k, e := range executed {
-		total += b.numStmt
+		total += k.numStmt
-		if count > 0 {
+		if e {
-			covered += b.numStmt
+			covered += k.numStmt
 		}
 	}
 

tools/testtool/commands/test_submission.go

@@ -243,8 +243,12 @@ func runTests(testDir, privateRepo, problem string) error {
 	}
 
 	coverageReq := getCoverageRequirements(path.Join(privateRepo, problem))
+	coveragePackages := []string{}
 	if coverageReq.Enabled {
 		log.Printf("required coverage: %.2f%%", coverageReq.Percent)
+		for _, pkg := range coverageReq.Packages {
+			coveragePackages = append(coveragePackages, path.Join(moduleImportPath, problem, pkg))
+		}
 	}
 
 	binariesJSON, _ := json.Marshal(binaries)
@@ -254,11 +258,7 @@ func runTests(testDir, privateRepo, problem string) error {
 		testBinaries[testPkg] = binPath
 		cmd := []string{"test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg}
 		if coverageReq.Enabled {
-			pkgs := make([]string, len(coverageReq.Packages))
+			cmd = append(cmd, "-cover", "-coverpkg", strings.Join(coveragePackages, ","))
-			for i, pkg := range coverageReq.Packages {
-				pkgs[i] = path.Join(moduleImportPath, problem, pkg)
-			}
-			cmd = append(cmd, "-cover", "-coverpkg", strings.Join(pkgs, ","))
 		}
 		if err := runGo(cmd...); err != nil {
 			return fmt.Errorf("error building test in %s: %w", testPkg, err)
@@ -324,7 +324,26 @@ func runTests(testDir, privateRepo, problem string) error {
 	if coverageReq.Enabled {
 		log.Printf("checking coverage is at least %.2f%%...", coverageReq.Percent)
 
-		percent, err := calCoverage(coverProfiles)
+		// For some reason, this command will record all coverage blocks in coverpkg,
+		// even if no test binaries depend on given package.
+		// Hacky way to record all the code present in problem definition.
+		targetProfile := path.Join(os.TempDir(), randomName())
+		coverCmd := exec.Command("go",
+			"test",
+			"-coverpkg", strings.Join(coveragePackages, ","),
+			"-coverprofile", targetProfile,
+			"-run", "^$",
+			"./...",
+		)
+		coverCmd.Env = append(os.Environ(), "GOFLAGS=")
+		coverCmd.Dir = path.Join(privateRepo, problem)
+		coverCmd.Stderr = os.Stderr
+		log.Printf("> %s", strings.Join(coverCmd.Args, " "))
+		if err := coverCmd.Run(); err != nil {
+			return fmt.Errorf("error getting target coverage profile: %w", err)
+		}
+
+		percent, err := calCoverage(targetProfile, coverProfiles)
 		if err != nil {
 			return err
 		}