Calculate coverage problem-wise, not per package.

This seems to be fair, especially since coverage target is
defined problem-wise. Practical reason: coverme/app can not
be covered sufficiently without testing Start/initRoutes/run;
out of these, only initRoutes can be tested, and testing it
separately is a wierd solution.

tools/testtool/commands/coverage.go

@@ -72,19 +72,38 @@ func searchCoverageComment(fname string) (*CoverageRequirements, error) {
 }
 
 // calCoverage calculates coverage percent for given coverage profile.
-func calCoverage(profile string) (float64, error) {
-	profiles, err := cover.ParseProfiles(profile)
+func calCoverage(fileNames []string) (float64, error) {
+	type block struct {
+		fileName            string
+		startLine, startCol int
+		endLine, endCol     int
+		numStmt             int
+	}
+	counts := map[block]int{}
+
+	for _, f := range fileNames {
+		profiles, err := cover.ParseProfiles(f)
 		if err != nil {
-		return 0.0, fmt.Errorf("cannot parse coverage profile file %s: %w", profile, err)
+			return 0.0, fmt.Errorf("cannot parse coverage profile file %s: %w", f, err)
+		}
+
+		for _, p := range profiles {
+			for _, b := range p.Blocks {
+				counts[block{
+					p.FileName,
+					b.StartLine, b.StartCol,
+					b.EndLine, b.EndCol,
+					b.NumStmt,
+				}] += b.Count
+			}
+		}
 	}
 
 	var total, covered int
-	for _, p := range profiles {
-		for _, block := range p.Blocks {
-			total += block.NumStmt
-			if block.Count > 0 {
-				covered += block.NumStmt
-			}
+	for b, count := range counts {
+		total += b.numStmt
+		if count > 0 {
+			covered += b.numStmt
 		}
 	}
 

tools/testtool/commands/test_submission.go

@@ -261,6 +261,7 @@ func runTests(testDir, privateRepo, problem string) error {
 		}
 	}
 
+	coverProfiles := []string{}
 	for testPkg, testBinary := range testBinaries {
 		relPath := strings.TrimPrefix(testPkg, moduleImportPath)
 		coverProfile := path.Join(os.TempDir(), randomName())
@@ -269,6 +270,7 @@ func runTests(testDir, privateRepo, problem string) error {
 			cmd := exec.Command(testBinary)
 			if coverageReq.Enabled {
 				cmd = exec.Command(testBinary, "-test.coverprofile", coverProfile)
+				coverProfiles = append(coverProfiles, coverProfile)
 			}
 			if currentUserIsRoot() {
 				if err := sandbox(cmd); err != nil {
@@ -286,20 +288,6 @@ func runTests(testDir, privateRepo, problem string) error {
 			}
 		}
 
-		if coverageReq.Enabled {
-			log.Printf("checking coverage is at least %.2f%% for %s", coverageReq.Percent, testPkg)
-
-			percent, err := calCoverage(coverProfile)
-			if err != nil {
-				return err
-			}
-
-			if percent < coverageReq.Percent {
-				return fmt.Errorf("poor coverage %.2f%%; expected at least %.2f%%",
-					percent, coverageReq.Percent)
-			}
-		}
-
 		{
 			benchCmd := exec.Command(testBinary, "-test.bench=.", "-test.run=^$")
 			if currentUserIsRoot() {
@@ -329,6 +317,21 @@ func runTests(testDir, privateRepo, problem string) error {
 		}
 	}
 
+	if coverageReq.Enabled {
+		log.Printf("checking coverage is at least %.2f%%...", coverageReq.Percent)
+
+		percent, err := calCoverage(coverProfiles)
+		if err != nil {
+			return err
+		}
+		log.Printf("coverage is %.2f%%", percent)
+
+		if percent < coverageReq.Percent {
+			return fmt.Errorf("poor coverage %.2f%%; expected at least %.2f%%",
+				percent, coverageReq.Percent)
+		}
+	}
+
 	return nil
 }