Task grading

.grader-ci.yml

@@ -1,6 +1,4 @@
 grade:
   image: eu.gcr.io/shad-ts/grader/go
-  only:
-    - /^submits/.*$/
   script:
-    - echo "Not implemented"
+    - testtool grade

go.mod

@@ -6,4 +6,5 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/tools v0.0.0-20200125223703-d33eef8e6825
+	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -53,3 +53,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/testtool/commands/deadlines.go

@@ -0,0 +1,108 @@
+package commands
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"gopkg.in/yaml.v2"
+)
+
+const timeFormat = "02-01-2006 15:04"
+
+type (
+	Task struct {
+		Name  string `yaml:"task"`
+		Score int    `yaml:"score"`
+	}
+
+	Group struct {
+		Name     string `yaml:"group"`
+		Start    string `yaml:"start"`
+		Deadline string `yaml:"deadline"`
+		Tasks    []Task `yaml:"tasks"`
+	}
+
+	Deadlines []Group
+)
+
+func (g Group) IsOpen() bool {
+	t, _ := time.Parse(timeFormat, g.Start)
+	return time.Until(t) < 0
+}
+
+func (d Deadlines) FindTask(name string) (*Group, *Task) {
+	for _, g := range d {
+		for _, t := range g.Tasks {
+			if t.Name == name {
+				return &g, &t
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func loadDeadlines(filename string) (Deadlines, error) {
+	b, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	var d Deadlines
+	if err := yaml.Unmarshal(b, &d); err != nil {
+		return nil, fmt.Errorf("error reading deadlines: %w", err)
+	}
+
+	for _, g := range d {
+		if _, err := time.Parse(timeFormat, g.Start); err != nil {
+			return nil, fmt.Errorf("invalid time format in task %q: %w", g.Name, err)
+		}
+
+		if _, err := time.Parse(timeFormat, g.Deadline); err != nil {
+			return nil, fmt.Errorf("invalid time format in task %q: %w", g.Name, err)
+		}
+	}
+
+	return d, nil
+}
+
+func findChangedTasks(d Deadlines, files []string) []string {
+	tasks := map[string]struct{}{}
+
+	for _, f := range files {
+		for {
+			dir, _ := filepath.Split(f)
+			if dir == "" {
+				break
+			}
+
+			f = dir
+		}
+
+		if f == "" {
+			continue
+		}
+
+		group, task := d.FindTask(f)
+		if task == nil {
+			continue
+		}
+
+		if !group.IsOpen() {
+			continue
+		}
+
+		tasks[f] = struct{}{}
+	}
+
+	var l []string
+	for t := range tasks {
+		l = append(l, t)
+	}
+
+	sort.Strings(l)
+	return l
+}

tools/testtool/commands/deadlines_test.go

@@ -0,0 +1,17 @@
+package commands
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeadlines(t *testing.T) {
+	d, err := loadDeadlines("../../../.deadlines.yml")
+	require.NoError(t, err)
+	require.NotEmpty(t, d)
+
+	_, sum := d.FindTask("sum")
+	require.NotNil(t, sum)
+	require.Equal(t, "sum", sum.Name)
+}

tools/testtool/commands/git.go

@@ -0,0 +1,22 @@
+package commands
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func listChangedFiles(gitPath string) ([]string, error) {
+	var gitOutput bytes.Buffer
+
+	cmd := exec.Command("git", "diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD")
+	cmd.Dir = gitPath
+	cmd.Stdout = &gitOutput
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+
+	return strings.Split(gitOutput.String(), "\n"), nil
+}

tools/testtool/commands/git_test.go

@@ -0,0 +1,13 @@
+package commands
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGit(t *testing.T) {
+	files, err := listChangedFiles(".")
+	require.NoError(t, err)
+	require.NotEmpty(t, files)
+}

tools/testtool/commands/grade.go

@@ -0,0 +1,82 @@
+package commands
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+const (
+	privateRepoRoot = "/opt/shad"
+	deadlinesYML    = ".deadlines.yml"
+)
+
+func grade() error {
+	userID := os.Getenv("GITLAB_USER_ID")
+	testerToken := os.Getenv("TESTER_TOKEN")
+	submitRoot := os.Getenv("CI_PROJECT_DIR")
+
+	changedFiles, err := listChangedFiles(submitRoot)
+	if err != nil {
+		return err
+	}
+
+	deadlines, err := loadDeadlines(filepath.Join(privateRepoRoot, deadlinesYML))
+	if err != nil {
+		return err
+	}
+
+	changedTasks := findChangedTasks(deadlines, changedFiles)
+	log.Printf("detected change in tasks %v", changedTasks)
+
+	var failed bool
+	for _, task := range changedTasks {
+		log.Printf("testing task %s", task)
+
+		var testFailed bool
+
+		err := testSubmission(submitRoot, privateRepoRoot, task)
+		if err != nil {
+			log.Printf("task %s failed: %s", task, err)
+			failed = true
+
+			var testFailedErr *TestFailedError
+			testFailed = errors.As(err, &testFailedErr)
+
+			if !testFailed {
+				continue
+			}
+		} else {
+			log.Printf("task %s passed", task)
+		}
+
+		if err := reportTestResults(testerToken, task, userID, testFailed); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if failed {
+		return fmt.Errorf("some tasks failed")
+	}
+
+	return nil
+}
+
+var gradeCmd = &cobra.Command{
+	Use:   "grade",
+	Short: "test all tasks in the last commit",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := grade(); err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(gradeCmd)
+}

tools/testtool/commands/report.go

@@ -0,0 +1,44 @@
+package commands
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+)
+
+var testingToken = ""
+
+const reportEndpoint = "https://go.manytask.org/api/report"
+
+func reportTestResults(token string, task string, userID string, failed bool) error {
+	form := url.Values{}
+	form.Set("token", token)
+	form.Set("task", task)
+	form.Set("user_id", userID)
+
+	if failed {
+		form.Set("failed", "1")
+	}
+
+	var rsp *http.Response
+	var err error
+
+	for i := 0; i < 3; i++ {
+		rsp, err = http.PostForm(reportEndpoint, form)
+		if err != nil {
+			log.Printf("retrying report: %v", err)
+			continue
+		}
+
+		if rsp.StatusCode != 200 {
+			err = fmt.Errorf("server returned status %d", rsp.StatusCode)
+			log.Printf("retrying report: %v", err)
+			continue
+		}
+
+		return nil
+	}
+
+	return err
+}

tools/testtool/commands/report_test.go

@@ -0,0 +1,15 @@
+package commands
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReport(t *testing.T) {
+	if testingToken == "" {
+		t.Skip("token is missing")
+	}
+
+	require.NoError(t, reportTestResults(testingToken, "sum", "1", false))
+}

tools/testtool/commands/test_submission.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -27,10 +28,8 @@ const (
 )
 
 var testSubmissionCmd = &cobra.Command{
-	Use:     "test",
-	Aliases: []string{"check", "test-submission", "check-submission"},
-	Short:   "test submission",
-	Long:    `run solution on private and private tests`,
+	Use:   "check-task",
+	Short: "test single task",
 	Run: func(cmd *cobra.Command, args []string) {
 		problem, err := cmd.Flags().GetString(problemFlag)
 		if err != nil {
@@ -47,7 +46,9 @@ var testSubmissionCmd = &cobra.Command{
 			log.Fatalf("%s does not have %s directory", privateRepo, problem)
 		}
 
-		testSubmission(studentRepo, privateRepo, problem)
+		if err := testSubmission(studentRepo, privateRepo, problem); err != nil {
+			log.Fatal(err)
+		}
 	},
 }
 
@@ -85,7 +86,7 @@ func problemDirExists(repo, problem string) bool {
 	return info.IsDir()
 }
 
-func testSubmission(studentRepo, privateRepo, problem string) {
+func testSubmission(studentRepo, privateRepo, problem string) error {
 	// Create temp directory to store all files required to test the solution.
 	tmpRepo, err := ioutil.TempDir("/tmp", problem+"-")
 	if err != nil {
@@ -125,7 +126,7 @@ func testSubmission(studentRepo, privateRepo, problem string) {
 
 	// Run tests.
 	log.Printf("running tests")
-	runTests(tmpRepo, problem)
+	return runTests(tmpRepo, problem)
 }
 
 // copyDir recursively copies src directory to dst.
@@ -172,8 +173,20 @@ func randomName() string {
 	return hex.EncodeToString(raw[:])
 }
 
+type TestFailedError struct {
+	E error
+}
+
+func (e *TestFailedError) Error() string {
+	return fmt.Sprintf("test failed: %v", e)
+}
+
+func (e *TestFailedError) Unwrap() error {
+	return e.E
+}
+
 // runTests runs all tests in directory with race detector.
-func runTests(testDir, problem string) {
+func runTests(testDir, problem string) error {
 	binCache, err := ioutil.TempDir("/tmp", "bincache")
 	if err != nil {
 		log.Fatal(err)
@@ -182,7 +195,7 @@ func runTests(testDir, problem string) {
 		log.Fatal(err)
 	}
 
-	runGo := func(arg ...string) {
+	runGo := func(arg ...string) error {
 		log.Printf("> go %s", strings.Join(arg, " "))
 
 		cmd := exec.Command("go", arg...)
@@ -190,9 +203,7 @@ func runTests(testDir, problem string) {
 		cmd.Dir = testDir
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			log.Fatal(err)
-		}
+		return cmd.Run()
 	}
 
 	binaries := map[string]string{}
@@ -202,7 +213,10 @@ func runTests(testDir, problem string) {
 	for binaryPkg := range binPkgs {
 		binPath := filepath.Join(binCache, randomName())
 		binaries[binaryPkg] = binPath
-		runGo("build", "-mod", "readonly", "-tags", "private", "-o", binPath, binaryPkg)
+
+		if err := runGo("build", "-mod", "readonly", "-tags", "private", "-o", binPath, binaryPkg); err != nil {
+			return fmt.Errorf("error building binary in %s: %w", binaryPkg, err)
+		}
 	}
 
 	binariesJSON, _ := json.Marshal(binaries)
@@ -210,7 +224,9 @@ func runTests(testDir, problem string) {
 	for testPkg := range testPkgs {
 		binPath := filepath.Join(binCache, randomName())
 		testBinaries[testPkg] = binPath
-		runGo("test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg)
+		if err := runGo("test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg); err != nil {
+			return fmt.Errorf("error building test in %s: %w", testPkg, err)
+		}
 	}
 
 	for testPkg, testBinary := range testBinaries {
@@ -229,9 +245,11 @@ func runTests(testDir, problem string) {
 		cmd.Stderr = os.Stderr
 
 		if err := cmd.Run(); err != nil {
-			log.Fatal(err)
+			return &TestFailedError{E: err}
 		}
 	}
+
+	return nil
 }
 
 // relPaths converts paths to relative (to the baseDir) ones.

tools/testtool/commands/test_submission_test.go

@@ -1,9 +1,8 @@
 package commands
 
 import (
+	"errors"
 	"io/ioutil"
-	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"testing"
@@ -39,7 +38,8 @@ func Test_testSubmission_correct(t *testing.T) {
 		t.Run(problem, func(t *testing.T) {
 			studentRepo := path.Join(dir, "student")
 			privateRepo := path.Join(dir, "private")
-			testSubmission(studentRepo, privateRepo, problem)
+
+			require.NoError(t, testSubmission(studentRepo, privateRepo, problem))
 		})
 	}
 }
@@ -54,24 +54,16 @@ func Test_testSubmission_incorrect(t *testing.T) {
 
 		problem := path.Base(dir)
 		t.Run(problem, func(t *testing.T) {
-			if os.Getenv("BE_CRASHER") == "1" {
-				studentRepo := path.Join(dir, "student")
-				privateRepo := path.Join(dir, "private")
-				testSubmission(studentRepo, privateRepo, problem)
-				return
+			studentRepo := path.Join(dir, "student")
+			privateRepo := path.Join(dir, "private")
+
+			err := testSubmission(studentRepo, privateRepo, problem)
+			require.Error(t, err)
+
+			if problem == "brokentest" {
+				var testFailedErr *TestFailedError
+				require.True(t, errors.As(err, &testFailedErr))
 			}
-
-			cmd := exec.Command(os.Args[0], "-test.run=Test_testSubmission_incorrect/"+problem)
-			cmd.Env = append(os.Environ(), "BE_CRASHER=1")
-			cmd.Stdout = nil
-			cmd.Stderr = os.Stderr
-
-			err := cmd.Run()
-			if e, ok := err.(*exec.ExitError); ok && !e.Success() {
-				return
-			}
-
-			t.Fatalf("process ran with err %v, want exit status != 0", err)
 		})
 	}
 }