From 6a83a4f1fc507b0367be8f071499d9fe067a637f Mon Sep 17 00:00:00 2001 From: Fedor Korotkiy Date: Thu, 13 Feb 2020 01:54:25 +0300 Subject: [PATCH] Task grading --- .grader-ci.yml | 4 +- go.mod | 1 + go.sum | 2 + tools/testtool/commands/deadlines.go | 108 ++++++++++++++++++ tools/testtool/commands/deadlines_test.go | 17 +++ tools/testtool/commands/git.go | 22 ++++ tools/testtool/commands/git_test.go | 13 +++ tools/testtool/commands/grade.go | 82 +++++++++++++ tools/testtool/commands/report.go | 44 +++++++ tools/testtool/commands/report_test.go | 15 +++ tools/testtool/commands/test_submission.go | 48 +++++--- .../testtool/commands/test_submission_test.go | 32 ++---- 12 files changed, 350 insertions(+), 38 deletions(-) create mode 100644 tools/testtool/commands/deadlines.go create mode 100644 tools/testtool/commands/deadlines_test.go create mode 100644 tools/testtool/commands/git.go create mode 100644 tools/testtool/commands/git_test.go create mode 100644 tools/testtool/commands/grade.go create mode 100644 tools/testtool/commands/report.go create mode 100644 tools/testtool/commands/report_test.go diff --git a/.grader-ci.yml b/.grader-ci.yml index 47db6f8..4e714d7 100644 --- a/.grader-ci.yml +++ b/.grader-ci.yml @@ -1,6 +1,4 @@ grade: image: eu.gcr.io/shad-ts/grader/go - only: - - /^submits/.*$/ script: - - echo "Not implemented" + - testtool grade diff --git a/go.mod b/go.mod index 395a3ee..029fc61 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.4.0 golang.org/x/tools v0.0.0-20200125223703-d33eef8e6825 + gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 542b0fd..4094bc3 100644 --- a/go.sum +++ b/go.sum @@ -53,3 +53,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/tools/testtool/commands/deadlines.go b/tools/testtool/commands/deadlines.go new file mode 100644 index 0000000..819d149 --- /dev/null +++ b/tools/testtool/commands/deadlines.go @@ -0,0 +1,108 @@ +package commands + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "sort" + "time" + + "gopkg.in/yaml.v2" +) + +const timeFormat = "02-01-2006 15:04" + +type ( + Task struct { + Name string `yaml:"task"` + Score int `yaml:"score"` + } + + Group struct { + Name string `yaml:"group"` + Start string `yaml:"start"` + Deadline string `yaml:"deadline"` + Tasks []Task `yaml:"tasks"` + } + + Deadlines []Group +) + +func (g Group) IsOpen() bool { + t, _ := time.Parse(timeFormat, g.Start) + return time.Until(t) < 0 +} + +func (d Deadlines) FindTask(name string) (*Group, *Task) { + for _, g := range d { + for _, t := range g.Tasks { + if t.Name == name { + return &g, &t + } + } + } + + return nil, nil +} + +func loadDeadlines(filename string) (Deadlines, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + var d Deadlines + if err := yaml.Unmarshal(b, &d); err != nil { + return nil, fmt.Errorf("error reading deadlines: %w", err) + } + + for _, g := range d { + if _, err := time.Parse(timeFormat, g.Start); err != nil { + return nil, fmt.Errorf("invalid time format in task %q: %w", g.Name, err) + } + + if _, err := time.Parse(timeFormat, g.Deadline); err != nil { + return nil, fmt.Errorf("invalid time format in task %q: %w", g.Name, err) + } + } + + return d, nil +} + +func findChangedTasks(d Deadlines, files []string) []string { + tasks := map[string]struct{}{} + + for _, f := range files { + for { + dir, _ := filepath.Split(f) + if dir == "" { + break + } + + f = dir + } + + if f == "" { + continue + } + + group, task := d.FindTask(f) + if task == nil { + continue + } + + if !group.IsOpen() { + continue + } + + tasks[f] = struct{}{} + } + + var l []string + for t := range tasks { + l = append(l, t) + } + + sort.Strings(l) + return l +} diff --git a/tools/testtool/commands/deadlines_test.go b/tools/testtool/commands/deadlines_test.go new file mode 100644 index 0000000..0ebc158 --- /dev/null +++ b/tools/testtool/commands/deadlines_test.go @@ -0,0 +1,17 @@ +package commands + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDeadlines(t *testing.T) { + d, err := loadDeadlines("../../../.deadlines.yml") + require.NoError(t, err) + require.NotEmpty(t, d) + + _, sum := d.FindTask("sum") + require.NotNil(t, sum) + require.Equal(t, "sum", sum.Name) +} diff --git a/tools/testtool/commands/git.go b/tools/testtool/commands/git.go new file mode 100644 index 0000000..73dc884 --- /dev/null +++ b/tools/testtool/commands/git.go @@ -0,0 +1,22 @@ +package commands + +import ( + "bytes" + "os" + "os/exec" + "strings" +) + +func listChangedFiles(gitPath string) ([]string, error) { + var gitOutput bytes.Buffer + + cmd := exec.Command("git", "diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD") + cmd.Dir = gitPath + cmd.Stdout = &gitOutput + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, err + } + + return strings.Split(gitOutput.String(), "\n"), nil +} diff --git a/tools/testtool/commands/git_test.go b/tools/testtool/commands/git_test.go new file mode 100644 index 0000000..b514809 --- /dev/null +++ b/tools/testtool/commands/git_test.go @@ -0,0 +1,13 @@ +package commands + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGit(t *testing.T) { + files, err := listChangedFiles(".") + require.NoError(t, err) + require.NotEmpty(t, files) +} diff --git a/tools/testtool/commands/grade.go b/tools/testtool/commands/grade.go new file mode 100644 index 0000000..de4f3e6 --- /dev/null +++ b/tools/testtool/commands/grade.go @@ -0,0 +1,82 @@ +package commands + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/spf13/cobra" +) + +const ( + privateRepoRoot = "/opt/shad" + deadlinesYML = ".deadlines.yml" +) + +func grade() error { + userID := os.Getenv("GITLAB_USER_ID") + testerToken := os.Getenv("TESTER_TOKEN") + submitRoot := os.Getenv("CI_PROJECT_DIR") + + changedFiles, err := listChangedFiles(submitRoot) + if err != nil { + return err + } + + deadlines, err := loadDeadlines(filepath.Join(privateRepoRoot, deadlinesYML)) + if err != nil { + return err + } + + changedTasks := findChangedTasks(deadlines, changedFiles) + log.Printf("detected change in tasks %v", changedTasks) + + var failed bool + for _, task := range changedTasks { + log.Printf("testing task %s", task) + + var testFailed bool + + err := testSubmission(submitRoot, privateRepoRoot, task) + if err != nil { + log.Printf("task %s failed: %s", task, err) + failed = true + + var testFailedErr *TestFailedError + testFailed = errors.As(err, &testFailedErr) + + if !testFailed { + continue + } + } else { + log.Printf("task %s passed", task) + } + + if err := reportTestResults(testerToken, task, userID, testFailed); err != nil { + log.Fatal(err) + } + } + + if failed { + return fmt.Errorf("some tasks failed") + } + + return nil +} + +var gradeCmd = &cobra.Command{ + Use: "grade", + Short: "test all tasks in the last commit", + Run: func(cmd *cobra.Command, args []string) { + if err := grade(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + }, +} + +func init() { + rootCmd.AddCommand(gradeCmd) +} diff --git a/tools/testtool/commands/report.go b/tools/testtool/commands/report.go new file mode 100644 index 0000000..e3c65fa --- /dev/null +++ b/tools/testtool/commands/report.go @@ -0,0 +1,44 @@ +package commands + +import ( + "fmt" + "log" + "net/http" + "net/url" +) + +var testingToken = "" + +const reportEndpoint = "https://go.manytask.org/api/report" + +func reportTestResults(token string, task string, userID string, failed bool) error { + form := url.Values{} + form.Set("token", token) + form.Set("task", task) + form.Set("user_id", userID) + + if failed { + form.Set("failed", "1") + } + + var rsp *http.Response + var err error + + for i := 0; i < 3; i++ { + rsp, err = http.PostForm(reportEndpoint, form) + if err != nil { + log.Printf("retrying report: %v", err) + continue + } + + if rsp.StatusCode != 200 { + err = fmt.Errorf("server returned status %d", rsp.StatusCode) + log.Printf("retrying report: %v", err) + continue + } + + return nil + } + + return err +} diff --git a/tools/testtool/commands/report_test.go b/tools/testtool/commands/report_test.go new file mode 100644 index 0000000..6c606c1 --- /dev/null +++ b/tools/testtool/commands/report_test.go @@ -0,0 +1,15 @@ +package commands + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReport(t *testing.T) { + if testingToken == "" { + t.Skip("token is missing") + } + + require.NoError(t, reportTestResults(testingToken, "sum", "1", false)) +} diff --git a/tools/testtool/commands/test_submission.go b/tools/testtool/commands/test_submission.go index 2e6a6a4..611264f 100644 --- a/tools/testtool/commands/test_submission.go +++ b/tools/testtool/commands/test_submission.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "fmt" "io/ioutil" "log" "os" @@ -27,10 +28,8 @@ const ( ) var testSubmissionCmd = &cobra.Command{ - Use: "test", - Aliases: []string{"check", "test-submission", "check-submission"}, - Short: "test submission", - Long: `run solution on private and private tests`, + Use: "check-task", + Short: "test single task", Run: func(cmd *cobra.Command, args []string) { problem, err := cmd.Flags().GetString(problemFlag) if err != nil { @@ -47,7 +46,9 @@ var testSubmissionCmd = &cobra.Command{ log.Fatalf("%s does not have %s directory", privateRepo, problem) } - testSubmission(studentRepo, privateRepo, problem) + if err := testSubmission(studentRepo, privateRepo, problem); err != nil { + log.Fatal(err) + } }, } @@ -85,7 +86,7 @@ func problemDirExists(repo, problem string) bool { return info.IsDir() } -func testSubmission(studentRepo, privateRepo, problem string) { +func testSubmission(studentRepo, privateRepo, problem string) error { // Create temp directory to store all files required to test the solution. tmpRepo, err := ioutil.TempDir("/tmp", problem+"-") if err != nil { @@ -125,7 +126,7 @@ func testSubmission(studentRepo, privateRepo, problem string) { // Run tests. log.Printf("running tests") - runTests(tmpRepo, problem) + return runTests(tmpRepo, problem) } // copyDir recursively copies src directory to dst. @@ -172,8 +173,20 @@ func randomName() string { return hex.EncodeToString(raw[:]) } +type TestFailedError struct { + E error +} + +func (e *TestFailedError) Error() string { + return fmt.Sprintf("test failed: %v", e) +} + +func (e *TestFailedError) Unwrap() error { + return e.E +} + // runTests runs all tests in directory with race detector. -func runTests(testDir, problem string) { +func runTests(testDir, problem string) error { binCache, err := ioutil.TempDir("/tmp", "bincache") if err != nil { log.Fatal(err) @@ -182,7 +195,7 @@ func runTests(testDir, problem string) { log.Fatal(err) } - runGo := func(arg ...string) { + runGo := func(arg ...string) error { log.Printf("> go %s", strings.Join(arg, " ")) cmd := exec.Command("go", arg...) @@ -190,9 +203,7 @@ func runTests(testDir, problem string) { cmd.Dir = testDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - log.Fatal(err) - } + return cmd.Run() } binaries := map[string]string{} @@ -202,7 +213,10 @@ func runTests(testDir, problem string) { for binaryPkg := range binPkgs { binPath := filepath.Join(binCache, randomName()) binaries[binaryPkg] = binPath - runGo("build", "-mod", "readonly", "-tags", "private", "-o", binPath, binaryPkg) + + if err := runGo("build", "-mod", "readonly", "-tags", "private", "-o", binPath, binaryPkg); err != nil { + return fmt.Errorf("error building binary in %s: %w", binaryPkg, err) + } } binariesJSON, _ := json.Marshal(binaries) @@ -210,7 +224,9 @@ func runTests(testDir, problem string) { for testPkg := range testPkgs { binPath := filepath.Join(binCache, randomName()) testBinaries[testPkg] = binPath - runGo("test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg) + if err := runGo("test", "-mod", "readonly", "-tags", "private", "-c", "-o", binPath, testPkg); err != nil { + return fmt.Errorf("error building test in %s: %w", testPkg, err) + } } for testPkg, testBinary := range testBinaries { @@ -229,9 +245,11 @@ func runTests(testDir, problem string) { cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - log.Fatal(err) + return &TestFailedError{E: err} } } + + return nil } // relPaths converts paths to relative (to the baseDir) ones. diff --git a/tools/testtool/commands/test_submission_test.go b/tools/testtool/commands/test_submission_test.go index deae525..845efae 100644 --- a/tools/testtool/commands/test_submission_test.go +++ b/tools/testtool/commands/test_submission_test.go @@ -1,9 +1,8 @@ package commands import ( + "errors" "io/ioutil" - "os" - "os/exec" "path" "path/filepath" "testing" @@ -39,7 +38,8 @@ func Test_testSubmission_correct(t *testing.T) { t.Run(problem, func(t *testing.T) { studentRepo := path.Join(dir, "student") privateRepo := path.Join(dir, "private") - testSubmission(studentRepo, privateRepo, problem) + + require.NoError(t, testSubmission(studentRepo, privateRepo, problem)) }) } } @@ -54,24 +54,16 @@ func Test_testSubmission_incorrect(t *testing.T) { problem := path.Base(dir) t.Run(problem, func(t *testing.T) { - if os.Getenv("BE_CRASHER") == "1" { - studentRepo := path.Join(dir, "student") - privateRepo := path.Join(dir, "private") - testSubmission(studentRepo, privateRepo, problem) - return + studentRepo := path.Join(dir, "student") + privateRepo := path.Join(dir, "private") + + err := testSubmission(studentRepo, privateRepo, problem) + require.Error(t, err) + + if problem == "brokentest" { + var testFailedErr *TestFailedError + require.True(t, errors.As(err, &testFailedErr)) } - - cmd := exec.Command(os.Args[0], "-test.run=Test_testSubmission_incorrect/"+problem) - cmd.Env = append(os.Environ(), "BE_CRASHER=1") - cmd.Stdout = nil - cmd.Stderr = os.Stderr - - err := cmd.Run() - if e, ok := err.(*exec.ExitError); ok && !e.Success() { - return - } - - t.Fatalf("process ran with err %v, want exit status != 0", err) }) } }