From ac8e1b435d1be74d9961ced72b3b3d9308d86b2d Mon Sep 17 00:00:00 2001 From: Fedor Korotkiy Date: Wed, 26 Apr 2023 16:29:27 +0400 Subject: [PATCH] Cleanup distbuild --- distbuild/README.md | 45 +++++++++---------- {disttest => distbuild/disttest}/.gitignore | 0 {disttest => distbuild/disttest}/README.md | 0 {disttest => distbuild/disttest}/fixture.go | 2 + {disttest => distbuild/disttest}/recorder.go | 0 .../disttest}/single_worker_test.go | 0 .../disttest}/testdata/TestSourceFiles/a.txt | 0 .../testdata/TestSourceFiles/b/c.txt | 0 .../disttest}/three_workers_test.go | 11 ++--- distbuild/pkg/api/README.md | 4 +- distbuild/pkg/scheduler/README.md | 2 +- distbuild/pkg/scheduler/scheduler.go | 4 -- distbuild/pkg/tarstream/README.md | 4 ++ 13 files changed, 37 insertions(+), 35 deletions(-) rename {disttest => distbuild/disttest}/.gitignore (100%) rename {disttest => distbuild/disttest}/README.md (100%) rename {disttest => distbuild/disttest}/fixture.go (98%) rename {disttest => distbuild/disttest}/recorder.go (100%) rename {disttest => distbuild/disttest}/single_worker_test.go (100%) rename {disttest => distbuild/disttest}/testdata/TestSourceFiles/a.txt (100%) rename {disttest => distbuild/disttest}/testdata/TestSourceFiles/b/c.txt (100%) rename {disttest => distbuild/disttest}/three_workers_test.go (87%) create mode 100644 distbuild/pkg/tarstream/README.md diff --git a/distbuild/README.md b/distbuild/README.md index 5098e8e..87e93fb 100644 --- a/distbuild/README.md +++ b/distbuild/README.md @@ -96,7 +96,7 @@ type Job struct { После того, как все кубики будут готовы, нужно будет соединить их вместе, реализовав [`distbuild/pkg/worker`](./pkg/worker), [`distbuild/pkg/client`](./pkg/client) и [`distbuild/pkg/dist`](./pkg/dist). Код в этих пакетах нужно отлаживать на -интеграционных тестах в [`distbuild/disttest`](../disttest). +интеграционных тестах в [`distbuild/disttest`](./disttest). Код тестов в этом задании менять нельзя. Это значит, что вы не можете менять интерфейсы в тех местах, где код покрыт тестами. @@ -105,28 +105,25 @@ type Job struct { Сколько кода нужно написать? ``` -prime@bee ~/C/shad-go> find distbuild -iname '*.go' | grep -v test | grep -v mock | grep -v pkg/build | xargs wc -l - 23 distbuild/pkg/worker/state.go - 111 distbuild/pkg/worker/worker.go - 45 distbuild/pkg/worker/download.go - 281 distbuild/pkg/worker/job.go - 69 distbuild/pkg/api/heartbeat.go - 121 distbuild/pkg/api/build_client.go - 53 distbuild/pkg/api/build.go - 60 distbuild/pkg/api/heartbeat_handler.go - 142 distbuild/pkg/api/build_handler.go - 56 distbuild/pkg/api/heartbeat_client.go - 288 distbuild/pkg/scheduler/scheduler.go - 119 distbuild/pkg/dist/build.go - 120 distbuild/pkg/dist/coordinator.go - 98 distbuild/pkg/tarstream/stream.go - 42 distbuild/pkg/artifact/client.go - 191 distbuild/pkg/artifact/cache.go - 54 distbuild/pkg/artifact/handler.go - 124 distbuild/pkg/client/build.go - 83 distbuild/pkg/filecache/client.go - 99 distbuild/pkg/filecache/handler.go - 111 distbuild/pkg/filecache/filecache.go - 2290 total +prime@fedora ~/C/s/distbuild (master)> find -iname '*_solution.go' | grep -v scheduler_solution| xargs wc -l + 123 ./pkg/api/build_client_solution.go + 140 ./pkg/api/build_handler_solution.go + 58 ./pkg/api/heartbeat_client_solution.go + 62 ./pkg/api/heartbeat_handler_solution.go + 5 ./pkg/artifact/cache_solution.go + 44 ./pkg/artifact/client_solution.go + 56 ./pkg/artifact/handler_solution.go + 126 ./pkg/client/build_solution.go + 121 ./pkg/dist/build_solution.go + 122 ./pkg/dist/coordinator_solution.go + 85 ./pkg/filecache/client_solution.go + 5 ./pkg/filecache/filecache_solution.go + 101 ./pkg/filecache/handler_solution.go + 5 ./pkg/tarstream/stream_solution.go + 47 ./pkg/worker/download_solution.go + 283 ./pkg/worker/job_solution.go + 25 ./pkg/worker/state_solution.go + 113 ./pkg/worker/worker_solution.go + 1521 total ``` diff --git a/disttest/.gitignore b/distbuild/disttest/.gitignore similarity index 100% rename from disttest/.gitignore rename to distbuild/disttest/.gitignore diff --git a/disttest/README.md b/distbuild/disttest/README.md similarity index 100% rename from disttest/README.md rename to distbuild/disttest/README.md diff --git a/disttest/fixture.go b/distbuild/disttest/fixture.go similarity index 98% rename from disttest/fixture.go rename to distbuild/disttest/fixture.go index 97c0928..df84a7e 100644 --- a/disttest/fixture.go +++ b/distbuild/disttest/fixture.go @@ -37,6 +37,7 @@ type env struct { Client *client.Client Coordinator *dist.Coordinator Workers []*worker.Worker + WorkerCache []*artifact.Cache HTTP *http.Server } @@ -140,6 +141,7 @@ func newEnv(t *testing.T, config *Config) (e *env, cancel func()) { ) env.Workers = append(env.Workers, w) + env.WorkerCache = append(env.WorkerCache, artifacts) router.Handle(workerPrefix+"/", http.StripPrefix(workerPrefix, w)) } diff --git a/disttest/recorder.go b/distbuild/disttest/recorder.go similarity index 100% rename from disttest/recorder.go rename to distbuild/disttest/recorder.go diff --git a/disttest/single_worker_test.go b/distbuild/disttest/single_worker_test.go similarity index 100% rename from disttest/single_worker_test.go rename to distbuild/disttest/single_worker_test.go diff --git a/disttest/testdata/TestSourceFiles/a.txt b/distbuild/disttest/testdata/TestSourceFiles/a.txt similarity index 100% rename from disttest/testdata/TestSourceFiles/a.txt rename to distbuild/disttest/testdata/TestSourceFiles/a.txt diff --git a/disttest/testdata/TestSourceFiles/b/c.txt b/distbuild/disttest/testdata/TestSourceFiles/b/c.txt similarity index 100% rename from disttest/testdata/TestSourceFiles/b/c.txt rename to distbuild/disttest/testdata/TestSourceFiles/b/c.txt diff --git a/disttest/three_workers_test.go b/distbuild/disttest/three_workers_test.go similarity index 87% rename from disttest/three_workers_test.go rename to distbuild/disttest/three_workers_test.go index 82c4694..04d6ead 100644 --- a/disttest/three_workers_test.go +++ b/distbuild/disttest/three_workers_test.go @@ -5,9 +5,9 @@ import ( "os" "sync" "testing" - "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gitlab.com/slon/shad-go/distbuild/pkg/build" ) @@ -29,8 +29,6 @@ func TestArtifactTransferBetweenWorkers(t *testing.T) { var wg sync.WaitGroup wg.Add(3) - startTime := time.Now() - for i := 0; i < 3; i++ { depJobID := build.ID{'b', byte(i)} depJob := build.Job{ @@ -59,6 +57,9 @@ func TestArtifactTransferBetweenWorkers(t *testing.T) { wg.Wait() - testDuration := time.Since(startTime) - assert.True(t, testDuration < time.Second*5/2, "test duration should be less than 2.5 seconds") + for _, cache := range env.WorkerCache { + _, unlock, err := cache.Get(baseJob.ID) + require.NoError(t, err) + defer unlock() + } } diff --git a/distbuild/pkg/api/README.md b/distbuild/pkg/api/README.md index feced46..4a0f3aa 100644 --- a/distbuild/pkg/api/README.md +++ b/distbuild/pkg/api/README.md @@ -19,8 +19,10 @@ Client и Coordinator общаются через два вызова. - `POST /build` - стартует новый билд. * Client посылает в Body запроса json c описанием сборки. * Coordinator стримит в body ответа json сообщения, описывающие прогресс сборки. - * Первым сообщением в ответе Coordinator присылает `buildID`. * _Тут можно было бы использовать websocket, но нас устраивает более простое решение._ + * Чтобы послать клиенту неполный body, нужно использовать метод `Flush`. + Прочитайте про [`http.ResponseController`](https://pkg.go.dev/net/http#ResponseController) и используйте его. + * Первым сообщением в ответе Coordinator присылает `buildID`. - `POST /signal?build_id=12345` - посылает сигнал бегущему билду. * Запрос и ответ передаются в формате json. diff --git a/distbuild/pkg/scheduler/README.md b/distbuild/pkg/scheduler/README.md index 4772f49..4e7352c 100644 --- a/distbuild/pkg/scheduler/README.md +++ b/distbuild/pkg/scheduler/README.md @@ -10,7 +10,7 @@ могут вызвать даже для того джоба, который никто не шедулил. В этом случае планировщик просто должен запомнить, что результаты джоба сохранены в кеше на воркере. -Фукнция `LocateArtifact` должна возвращать имя любого воркера, который хранит в кеше заданный артефакт. +Функция `LocateArtifact` должна возвращать имя любого воркера, который хранит в кеше заданный артефакт. Эта функция не нужна в этой задаче, но он потребуется вам для реализации передачи артефактов между воркерами. diff --git a/distbuild/pkg/scheduler/scheduler.go b/distbuild/pkg/scheduler/scheduler.go index e026024..90ff054 100644 --- a/distbuild/pkg/scheduler/scheduler.go +++ b/distbuild/pkg/scheduler/scheduler.go @@ -36,10 +36,6 @@ func (c *Scheduler) LocateArtifact(id build.ID) (api.WorkerID, bool) { panic("implement me") } -func (c *Scheduler) RegisterWorker(workerID api.WorkerID) { - panic("implement me") -} - func (c *Scheduler) OnJobComplete(workerID api.WorkerID, jobID build.ID, res *api.JobResult) bool { panic("implement me") } diff --git a/distbuild/pkg/tarstream/README.md b/distbuild/pkg/tarstream/README.md new file mode 100644 index 0000000..ea5c25b --- /dev/null +++ b/distbuild/pkg/tarstream/README.md @@ -0,0 +1,4 @@ +# tarstream + +Пакет `tarstream` содержит функции для сериализации и десериализации директории. Вам не нужно +писать новый код в этом пакете, но нужно научиться пользоваться тем кодом, который вам дан.