Resolve "word-count"
This commit is contained in:
parent
0de2390008
commit
dea434be37
11 changed files with 324 additions and 6 deletions
|
@ -1,8 +1,8 @@
|
|||
check:
|
||||
image: eu.gcr.io/shad-ts/grader/go-build
|
||||
script:
|
||||
- go test -tags private,solution ./...
|
||||
- go test -race -tags private,solution ./...
|
||||
- go test -v -tags private,solution ./...
|
||||
- go test -v -race -tags private,solution ./...
|
||||
|
||||
rebuild-base-image:
|
||||
only:
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
testtool - cobra-based cli. У всех комманд есть help.
|
||||
Использование можно начать с такого:
|
||||
```
|
||||
go run ./tools/testtool/main.go --help
|
||||
go run ./tools/testtool/cmd/testtool/main.go --help
|
||||
```
|
||||
|
||||
При тестирование посылки выполняются следующие шаги:
|
||||
|
|
1
go.sum
1
go.sum
|
@ -47,6 +47,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20200125223703-d33eef8e6825 h1:aNQeSIHKi0RWpKA5NO0CqyLjx6Beh5l0LLUEnndEjz0=
|
||||
golang.org/x/tools v0.0.0-20200125223703-d33eef8e6825/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
@ -2,6 +2,6 @@ FROM eu.gcr.io/shad-ts/grader/go-build:latest
|
|||
|
||||
COPY . /opt/shad
|
||||
|
||||
RUN cd /opt/shad && go install gitlab.com/slon/shad-go/tools/testtool
|
||||
RUN cd /opt/shad && go install gitlab.com/slon/shad-go/tools/testtool/cmd/....
|
||||
|
||||
RUN find /opt/shad | xargs chmod o-rwx
|
||||
|
|
111
tools/testtool/bincache.go
Normal file
111
tools/testtool/bincache.go
Normal file
|
@ -0,0 +1,111 @@
|
|||
package testtool
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const BinariesEnv = "TESTTOOL_BINARIES"
|
||||
|
||||
type BinCache interface {
|
||||
// GetBinary returns filesystem path to the compiled binary corresponding to given import path.
|
||||
GetBinary(importPath string) (string, error)
|
||||
}
|
||||
|
||||
type CloseFunc func()
|
||||
|
||||
func NewBinCache() (BinCache, CloseFunc) {
|
||||
if _, ok := os.LookupEnv(BinariesEnv); ok {
|
||||
return newCIBuildCache(), func() {}
|
||||
}
|
||||
|
||||
dir, err := ioutil.TempDir("", "bincache-")
|
||||
if err != nil {
|
||||
log.Fatalf("unable to create temp dir: %s", err)
|
||||
}
|
||||
|
||||
return newLocalBinCache(dir), func() { _ = os.RemoveAll(dir) }
|
||||
}
|
||||
|
||||
// localBinCache is a BinCache implementation that compiles queried binaries lazily.
|
||||
type localBinCache struct {
|
||||
// dir is a directory that stores compiled binaries.
|
||||
dir string
|
||||
|
||||
binaries sync.Map
|
||||
}
|
||||
|
||||
// newLocalBinCache creates localBinCache that uses given directory to store binaries.
|
||||
func newLocalBinCache(dir string) *localBinCache {
|
||||
return &localBinCache{dir: dir}
|
||||
}
|
||||
|
||||
func (c *localBinCache) GetBinary(importPath string) (string, error) {
|
||||
v, ok := c.binaries.Load(importPath)
|
||||
if ok {
|
||||
return v.(string), nil
|
||||
}
|
||||
|
||||
binPath := filepath.Join(c.dir, RandomName())
|
||||
if buildTags == "" {
|
||||
runGo("build", "-mod", "readonly", "-o", binPath, importPath)
|
||||
} else {
|
||||
runGo("build", "-mod", "readonly", "-tags", buildTags, "-o", binPath, importPath)
|
||||
}
|
||||
|
||||
c.binaries.Store(importPath, binPath)
|
||||
|
||||
return binPath, nil
|
||||
}
|
||||
|
||||
func runGo(arg ...string) {
|
||||
cmd := exec.Command("go", arg...)
|
||||
cmd.Env = append(os.Environ(), "GOFLAGS=")
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ciBuildCache is a BinCache implementation that uses precompiled binaries.
|
||||
type ciBuildCache struct {
|
||||
binaries map[string]string
|
||||
}
|
||||
|
||||
// newCIBuildCache creates ciBuildCache that loads locations of precompiled binaries from env variable.
|
||||
func newCIBuildCache() *ciBuildCache {
|
||||
binariesJSON, ok := os.LookupEnv(BinariesEnv)
|
||||
if !ok {
|
||||
log.Fatalf("%s env variable not set", BinariesEnv)
|
||||
}
|
||||
|
||||
binaries := make(map[string]string)
|
||||
if err := json.Unmarshal([]byte(binariesJSON), &binaries); err != nil {
|
||||
log.Fatalf("unexpected %s format: %s", binaries, err)
|
||||
}
|
||||
|
||||
return &ciBuildCache{binaries: binaries}
|
||||
}
|
||||
|
||||
func (c *ciBuildCache) GetBinary(importPath string) (string, error) {
|
||||
binary, ok := c.binaries[importPath]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s not found", importPath)
|
||||
}
|
||||
return binary, nil
|
||||
}
|
||||
|
||||
func RandomName() string {
|
||||
var raw [8]byte
|
||||
_, _ = rand.Read(raw[:])
|
||||
return hex.EncodeToString(raw[:])
|
||||
}
|
5
tools/testtool/buildtags.go
Normal file
5
tools/testtool/buildtags.go
Normal file
|
@ -0,0 +1,5 @@
|
|||
// +build !solution
|
||||
|
||||
package testtool
|
||||
|
||||
const buildTags = ""
|
|
@ -13,6 +13,8 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitlab.com/slon/shad-go/tools/testtool"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -203,7 +205,7 @@ func runTests(testDir, problem string) {
|
|||
runGo("build", "-mod", "readonly", "-tags", "private", "-o", binPath, binaryPkg)
|
||||
}
|
||||
|
||||
binariesJSON, _ := json.Marshal(binPkgs)
|
||||
binariesJSON, _ := json.Marshal(binaries)
|
||||
|
||||
for testPkg := range testPkgs {
|
||||
binPath := filepath.Join(binCache, randomName())
|
||||
|
@ -222,7 +224,7 @@ func runTests(testDir, problem string) {
|
|||
}
|
||||
|
||||
cmd.Dir = filepath.Join(testDir, relPath)
|
||||
cmd.Env = []string{"TESTTOOL_BINARIES=" + string(binariesJSON)}
|
||||
cmd.Env = []string{testtool.BinariesEnv + "=" + string(binariesJSON)}
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
|
|
64
wordcount/README.md
Normal file
64
wordcount/README.md
Normal file
|
@ -0,0 +1,64 @@
|
|||
## wordcount
|
||||
|
||||
В этой задаче нужно написать консольную утилиту, которая принимает на вход набор файлов
|
||||
и печатает в stdout уникальные строки и количество раз, которое они встречаются.
|
||||
В stdout попадают только те строки, что встречаются суммарно хотя бы дважды.
|
||||
|
||||
Формат вывода:
|
||||
```
|
||||
<LINE>\t<COUNT>
|
||||
<LINE>\t<COUNT>
|
||||
...
|
||||
```
|
||||
|
||||
#### Пример:
|
||||
|
||||
Если a.txt - это файл со следующим содержимым:
|
||||
```
|
||||
a
|
||||
b
|
||||
a
|
||||
c
|
||||
```
|
||||
а b.txt - со следующим:
|
||||
```
|
||||
a
|
||||
b
|
||||
a
|
||||
c
|
||||
```
|
||||
то результат выполнения команды `wordcount a.txt b.txt` должен выглядеть так (с точностью до перестановки строк):
|
||||
```
|
||||
2 c
|
||||
4 a
|
||||
2 b
|
||||
```
|
||||
|
||||
### Проверка решения
|
||||
|
||||
Для запуска тестов нужно выполнить следующую команду:
|
||||
|
||||
```
|
||||
go test -v ./wordcount/...
|
||||
```
|
||||
|
||||
### Компиляция
|
||||
|
||||
```
|
||||
go install ./wordcount/...
|
||||
```
|
||||
|
||||
После выполнения в `$GOPATH/bin` появится исполняемый файл с именем `wordcount`.
|
||||
|
||||
### Walkthrough
|
||||
|
||||
#### 1. Чтение аргументов командной строки
|
||||
https://gobyexample.com/command-line-arguments
|
||||
#### 2. Чтение файлов
|
||||
https://gobyexample.com/reading-files
|
||||
#### 3. Парсинг содержимого
|
||||
https://gobyexample.com/string-functions
|
||||
#### 4. Подсчёт вхождений
|
||||
https://gobyexample.com/maps
|
||||
#### 5. Вывод результатов
|
||||
https://gobyexample.com/string-formatting
|
7
wordcount/main.go
Normal file
7
wordcount/main.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
// +build !solution
|
||||
|
||||
package main
|
||||
|
||||
func main() {
|
||||
|
||||
}
|
128
wordcount/main_test.go
Normal file
128
wordcount/main_test.go
Normal file
|
@ -0,0 +1,128 @@
|
|||
// +build !change
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gitlab.com/slon/shad-go/tools/testtool"
|
||||
)
|
||||
|
||||
const wordcountImportPath = "gitlab.com/slon/shad-go/wordcount"
|
||||
|
||||
var binCache testtool.BinCache
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Exit(func() int {
|
||||
var teardown testtool.CloseFunc
|
||||
binCache, teardown = testtool.NewBinCache()
|
||||
defer teardown()
|
||||
|
||||
return m.Run()
|
||||
}())
|
||||
}
|
||||
|
||||
func TestWordCount(t *testing.T) {
|
||||
binary, err := binCache.GetBinary(wordcountImportPath)
|
||||
require.Nil(t, err)
|
||||
|
||||
type counts map[string]int64
|
||||
type files []string
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
files files
|
||||
expected map[string]int64
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
files: files{``},
|
||||
expected: make(counts),
|
||||
},
|
||||
{
|
||||
name: "simple",
|
||||
files: files{
|
||||
`a
|
||||
a
|
||||
b
|
||||
|
||||
|
||||
a
|
||||
b`,
|
||||
},
|
||||
expected: counts{"a": 3, "b": 2, "": 2},
|
||||
},
|
||||
{
|
||||
name: "multiple-files",
|
||||
files: files{
|
||||
`a
|
||||
a`,
|
||||
`a
|
||||
b`,
|
||||
`b`,
|
||||
},
|
||||
expected: counts{"a": 3, "b": 2},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Create temp directory.
|
||||
testDir, err := ioutil.TempDir("", "wordcount-testdata-")
|
||||
require.Nil(t, err)
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
|
||||
// Create test files in temp directory.
|
||||
var files []string
|
||||
for _, f := range tc.files {
|
||||
file := path.Join(testDir, testtool.RandomName())
|
||||
err = ioutil.WriteFile(file, []byte(f), 0644)
|
||||
require.Nil(t, err)
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
// Run wordcount executable.
|
||||
cmd := exec.Command(binary, files...)
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
output, err := cmd.Output()
|
||||
require.Nil(t, err)
|
||||
|
||||
// Parse output and compare with an expected one.
|
||||
counts, err := parseStdout(output)
|
||||
require.Nil(t, err)
|
||||
|
||||
require.True(t, reflect.DeepEqual(tc.expected, counts),
|
||||
fmt.Sprintf("expected: %v; got: %v", tc.expected, counts))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// parseStdout parses wordcount's output of the ['<COUNT>\t<LINE>'] format.
|
||||
func parseStdout(data []byte) (map[string]int64, error) {
|
||||
counts := make(map[string]int64)
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, "\t", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("unexpected line format: %s", parts)
|
||||
}
|
||||
c, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse line count: %w", err)
|
||||
}
|
||||
counts[parts[1]] = c
|
||||
}
|
||||
|
||||
return counts, nil
|
||||
}
|
Loading…
Reference in a new issue