Support test coverage requirements via comment.

tools/testtool/commands/test_submission.go

@@ -242,12 +242,21 @@ func runTests(testDir, privateRepo, problem string) error {
 		}
 	}
 
+	coverageReq := getCoverageRequirements(privateRepo)
+	if coverageReq.Enabled {
+		log.Printf("required coverage: %.2f%%", coverageReq.Percent)
+	}
+
 	binariesJSON, _ := json.Marshal(binaries)
 
 	for testPkg := range testPkgs {
 		binPath := filepath.Join(binCache, randomName())
 		testBinaries[testPkg] = binPath
-		if err := runGo("test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg); err != nil {
+		cmd := []string{"test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg}
+		if coverageReq.Enabled {
+			cmd = append(cmd, "-cover")
+		}
+		if err := runGo(cmd...); err != nil {
 			return fmt.Errorf("error building test in %s: %w", testPkg, err)
 		}
 	}
@@ -257,6 +266,9 @@ func runTests(testDir, privateRepo, problem string) error {
 
 		{
 			cmd := exec.Command(testBinary)
+			if coverageReq.Enabled {
+				cmd = exec.Command(testBinary, "-test.coverprofile", "c.out")
+			}
 			if currentUserIsRoot() {
 				if err := sandbox(cmd); err != nil {
 					log.Fatal(err)
@@ -273,6 +285,20 @@ func runTests(testDir, privateRepo, problem string) error {
 			}
 		}
 
+		if coverageReq.Enabled {
+			log.Printf("checking coverage is at least %.2f%% for %s", coverageReq.Percent, testPkg)
+
+			percent, err := calCoverage(filepath.Join(testDir, relPath, "c.out"))
+			if err != nil {
+				return err
+			}
+
+			if percent < coverageReq.Percent {
+				return fmt.Errorf("poor coverage %.2f%%; expected at least %.2f%%",
+					percent, coverageReq.Percent)
+			}
+		}
+
 		{
 			benchCmd := exec.Command(testBinary, "-test.bench=.", "-test.run=^$")
 			if currentUserIsRoot() {

tools/testtool/testdata/coverage/sum/subpkg/empty_test.go

@@ -0,0 +1,22 @@
+// This is subpkg package comment.
+package subpkg
+
+// Incorrect coverage comments:
+
+// min coverage: -1%
+
+// min coverage: 100.001%
+
+// min coverage: 100 %
+
+// min coverage:10%
+
+//  min coverage: 19%
+
+// Correct coverage comment:
+
+// min coverage: 90%
+
+// Testtool uses first matching comment.
+
+// min coverage: 91%

tools/testtool/testdata/coverage/sum/sum.go

@@ -0,0 +1,2 @@
+// This is package comment.
+package sum

tools/testtool/testdata/coverage/sum/sum_test.go

@@ -0,0 +1,8 @@
+package sum
+
+// This is a single line comment.
+
+/*
+This is multiline
+comment!
+*/

tools/testtool/testdata/submissions/correct/coverme/private/.golangci.yml

@@ -0,0 +1,77 @@
+# options for analysis running
+run:
+  # default concurrency is a available CPU number
+  concurrency: 8
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 5m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters
+linters-settings:
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: gitlab.com
+
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - gofmt
+    - golint
+    - gosimple
+    - govet
+    - ineffassign
+    - scopelint
+    - staticcheck
+    - typecheck
+    - unconvert
+
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+    - Using the variable on range scope .* in function literal
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: true
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0

tools/testtool/testdata/submissions/correct/coverme/private/coverme/coverage_test.go

@@ -0,0 +1,16 @@
+// +build !change
+
+package coverme_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/slon/shad-go/coverme"
+)
+
+// min coverage: 60%
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), coverme.Sum(1, 1))
+}

tools/testtool/testdata/submissions/correct/coverme/private/coverme/service.go

@@ -0,0 +1,10 @@
+// +build !change
+
+package coverme
+
+func Sum(a, b int64) int64 {
+	if a == 0 {
+		return b
+	}
+	return a + b
+}

tools/testtool/testdata/submissions/correct/coverme/private/coverme/solution_test.go

@@ -0,0 +1,14 @@
+// +build solution
+
+package coverme
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), Sum(1, 1))
+	require.Equal(t, int64(1), Sum(0, 1))
+}

tools/testtool/testdata/submissions/correct/coverme/private/coverme/subpkg/f.go

@@ -0,0 +1,12 @@
+// +build !change
+
+package subpkg
+
+func AddOne(n int) int {
+	if n == 0 {
+		return 1
+	} else if n == 1 {
+		return 2
+	}
+	return n + 1
+}

tools/testtool/testdata/submissions/correct/coverme/private/coverme/subpkg/f_test.go

@@ -0,0 +1,13 @@
+package subpkg
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// min coverage: 100%
+
+func TestAddOne(t *testing.T) {
+	require.Equal(t, 1, AddOne(0))
+}

tools/testtool/testdata/submissions/correct/coverme/private/go.mod

@@ -0,0 +1,5 @@
+module gitlab.com/slon/shad-go
+
+go 1.13
+
+require github.com/stretchr/testify v1.5.1

tools/testtool/testdata/submissions/correct/coverme/private/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/testtool/testdata/submissions/correct/coverme/student/coverme/coverage_test.go

@@ -0,0 +1,16 @@
+// +build !change
+
+package coverme_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/slon/shad-go/coverme"
+)
+
+// min coverage: 60%
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), coverme.Sum(1, 1))
+}

tools/testtool/testdata/submissions/correct/coverme/student/coverme/service.go

@@ -0,0 +1,10 @@
+// +build !change
+
+package coverme
+
+func Sum(a, b int64) int64 {
+	if a == 0 {
+		return b
+	}
+	return a + b
+}

tools/testtool/testdata/submissions/correct/coverme/student/coverme/subpkg/additional_test.go

@@ -0,0 +1,12 @@
+package subpkg
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddOne_2(t *testing.T) {
+	require.Equal(t, 2, AddOne(1))
+	require.Equal(t, 30, AddOne(29))
+}

tools/testtool/testdata/submissions/correct/coverme/student/coverme/subpkg/f.go

@@ -0,0 +1,12 @@
+// +build !change
+
+package subpkg
+
+func AddOne(n int) int {
+	if n == 0 {
+		return 1
+	} else if n == 1 {
+		return 2
+	}
+	return n + 1
+}

tools/testtool/testdata/submissions/correct/coverme/student/coverme/subpkg/f_test.go

@@ -0,0 +1,13 @@
+package subpkg
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// min coverage: 100%
+
+func TestAddOne(t *testing.T) {
+	require.Equal(t, 1, AddOne(0))
+}

tools/testtool/testdata/submissions/correct/coverme/student/go.mod

@@ -0,0 +1,5 @@
+module gitlab.com/slon/shad-go
+
+go 1.13
+
+require github.com/stretchr/testify v1.5.1

tools/testtool/testdata/submissions/correct/coverme/student/go.sum

@@ -0,0 +1,6 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/.golangci.yml

@@ -0,0 +1,77 @@
+# options for analysis running
+run:
+  # default concurrency is a available CPU number
+  concurrency: 8
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 5m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters
+linters-settings:
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: gitlab.com
+
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - gofmt
+    - golint
+    - gosimple
+    - govet
+    - ineffassign
+    - scopelint
+    - staticcheck
+    - typecheck
+    - unconvert
+
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+    - Using the variable on range scope .* in function literal
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: true
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/go.mod

@@ -0,0 +1,5 @@
+module gitlab.com/slon/shad-go
+
+go 1.13
+
+require github.com/stretchr/testify v1.5.1

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/poorcoverage/coverage_test.go

@@ -0,0 +1,16 @@
+// +build !change
+
+package poorcoverage_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/slon/shad-go/poorcoverage"
+)
+
+// min coverage: 100%
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), poorcoverage.Sum(1, 1))
+}

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/poorcoverage/service.go

@@ -0,0 +1,10 @@
+// +build !change
+
+package poorcoverage
+
+func Sum(a, b int64) int64 {
+	if a == 0 {
+		return b
+	}
+	return a + b
+}

tools/testtool/testdata/submissions/incorrect/poorcoverage/private/poorcoverage/solution_test.go

@@ -0,0 +1,14 @@
+// +build solution
+
+package poorcoverage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), Sum(1, 1))
+	require.Equal(t, int64(1), Sum(0, 1))
+}

tools/testtool/testdata/submissions/incorrect/poorcoverage/student/go.mod

@@ -0,0 +1,5 @@
+module gitlab.com/slon/shad-go
+
+go 1.13
+
+require github.com/stretchr/testify v1.5.1

tools/testtool/testdata/submissions/incorrect/poorcoverage/student/go.sum

@@ -0,0 +1,6 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/testtool/testdata/submissions/incorrect/poorcoverage/student/poorcoverage/coverage_test.go

@@ -0,0 +1,16 @@
+// +build !change
+
+package poorcoverage_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/slon/shad-go/poorcoverage"
+)
+
+// min coverage: 100%
+
+func TestSum(t *testing.T) {
+	require.Equal(t, int64(2), poorcoverage.Sum(1, 1))
+}

tools/testtool/testdata/submissions/incorrect/poorcoverage/student/poorcoverage/service.go

@@ -0,0 +1,10 @@
+// +build !change
+
+package poorcoverage
+
+func Sum(a, b int64) int64 {
+	if a == 0 {
+		return b
+	}
+	return a + b
+}