From 110b00a017b685ad89a542a68da846f30224765e Mon Sep 17 00:00:00 2001 From: Fedor Korotkiy Date: Fri, 13 Mar 2020 01:33:54 +0300 Subject: [PATCH] Artifact cache --- distbuild/disttest/simple_test.go | 4 +- distbuild/pkg/artifact/cache.go | 168 +++++++++++++++++++++++++-- distbuild/pkg/artifact/cache_test.go | 70 +++++++++++ distbuild/pkg/build/graph.go | 13 +++ 4 files changed, 244 insertions(+), 11 deletions(-) create mode 100644 distbuild/pkg/artifact/cache_test.go diff --git a/distbuild/disttest/simple_test.go b/distbuild/disttest/simple_test.go index 397f6d7..834d3a7 100644 --- a/distbuild/disttest/simple_test.go +++ b/distbuild/disttest/simple_test.go @@ -3,6 +3,7 @@ package disttest import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/slon/shad-go/distbuild/pkg/build" @@ -27,5 +28,6 @@ func TestSingleCommand(t *testing.T) { var recorder Recorder require.NoError(t, env.Client.Build(env.Ctx, echoGraph, &recorder)) - require.Equal(t, &JobResult{Stdout: "OK", Code: new(int)}, recorder.Jobs[build.ID{'a'}]) + assert.Len(t, len(recorder.Jobs), 1) + assert.Equal(t, &JobResult{Stdout: "OK", Code: new(int)}, recorder.Jobs[build.ID{'a'}]) } diff --git a/distbuild/pkg/artifact/cache.go b/distbuild/pkg/artifact/cache.go index 153e2fd..183d2ca 100644 --- a/distbuild/pkg/artifact/cache.go +++ b/distbuild/pkg/artifact/cache.go @@ -1,8 +1,14 @@ package artifact import ( + "encoding/hex" "errors" + "fmt" + "io/ioutil" "net/http" + "os" + "path/filepath" + "sync" "gitlab.com/slon/shad-go/distbuild/pkg/build" ) @@ -13,26 +19,168 @@ var ( ErrReadLocked = errors.New("file is locked for read") ) -type Cache struct{} +type Cache struct { + tmpDir string + cacheDir string -func NewCache(root string) (*Cache, error) { - return &Cache{}, nil + mu sync.Mutex + writeLocked map[build.ID]struct{} + readLocked map[build.ID]int } -func (c *Cache) Range(artifactFn func(file build.ID) error) error { - panic("implement me") +func NewCache(root string) (*Cache, error) { + tmpDir := filepath.Join(root, "tmp") + + if err := os.RemoveAll(tmpDir); err != nil { + return nil, err + } + if err := os.MkdirAll(tmpDir, 0777); err != nil { + return nil, err + } + + cacheDir := filepath.Join(root, "c") + if err := os.MkdirAll(cacheDir, 0777); err != nil { + return nil, err + } + + for i := 0; i < 256; i++ { + d := hex.EncodeToString([]byte{uint8(i)}) + if err := os.MkdirAll(filepath.Join(cacheDir, d), 0777); err != nil { + return nil, err + } + } + + return &Cache{ + tmpDir: tmpDir, + cacheDir: cacheDir, + writeLocked: make(map[build.ID]struct{}), + readLocked: make(map[build.ID]int), + }, nil +} + +func (c *Cache) readLock(id build.ID) error { + c.mu.Lock() + defer c.mu.Unlock() + + if _, ok := c.writeLocked[id]; ok { + return ErrWriteLocked + } + + c.readLocked[id] += 1 + return nil +} + +func (c *Cache) readUnlock(id build.ID) { + c.mu.Lock() + defer c.mu.Unlock() + + c.readLocked[id] -= 1 + if c.readLocked[id] == 0 { + delete(c.readLocked, id) + } +} + +func (c *Cache) writeLock(id build.ID) error { + c.mu.Lock() + defer c.mu.Unlock() + + if _, ok := c.writeLocked[id]; ok { + return ErrWriteLocked + } + if c.readLocked[id] > 0 { + return ErrReadLocked + } + + c.writeLocked[id] = struct{}{} + return nil +} + +func (c *Cache) writeUnlock(id build.ID) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.writeLocked, id) +} + +func (c *Cache) Range(artifactFn func(artifact build.ID) error) error { + shards, err := ioutil.ReadDir(c.cacheDir) + if err != nil { + return err + } + + for _, shard := range shards { + dirs, err := ioutil.ReadDir(filepath.Join(c.cacheDir, shard.Name())) + if err != nil { + return err + } + + for _, d := range dirs { + var id build.ID + if err := id.UnmarshalText([]byte(d.Name())); err != nil { + return fmt.Errorf("invalid artifact name: %w", err) + } + + if err := artifactFn(id); err != nil { + return err + } + } + } + + return nil } func (c *Cache) Remove(artifact build.ID) error { - panic("implement me") + if err := c.writeLock(artifact); err != nil { + return err + } + defer c.writeUnlock(artifact) + + return os.RemoveAll(filepath.Join(c.cacheDir, artifact.Path())) } -func (c *Cache) Create(artifact build.ID) (path string, abort, commit func(), err error) { - panic("implement me") +func (c *Cache) Create(artifact build.ID) (path string, commit, abort func() error, err error) { + if err = c.writeLock(artifact); err != nil { + return + } + + path = filepath.Join(c.tmpDir, artifact.String()) + if err = os.MkdirAll(path, 0777); err != nil { + c.writeUnlock(artifact) + return + } + + abort = func() error { + defer c.writeUnlock(artifact) + return os.RemoveAll(path) + } + + commit = func() error { + defer c.writeUnlock(artifact) + return os.Rename(path, filepath.Join(c.cacheDir, artifact.Path())) + } + + return } -func (c *Cache) Get(file build.ID) (path string, unlock func(), err error) { - panic("implement me") +func (c *Cache) Get(artifact build.ID) (path string, unlock func(), err error) { + if err = c.readLock(artifact); err != nil { + return + } + + path = filepath.Join(c.cacheDir, artifact.Path()) + if _, err = os.Stat(path); err != nil { + c.readUnlock(artifact) + + if os.IsNotExist(err) { + err = ErrNotFound + } + return + } + + unlock = func() { + c.readUnlock(artifact) + } + return } func NewHandler(c *Cache) http.Handler { diff --git a/distbuild/pkg/artifact/cache_test.go b/distbuild/pkg/artifact/cache_test.go new file mode 100644 index 0000000..043d1a7 --- /dev/null +++ b/distbuild/pkg/artifact/cache_test.go @@ -0,0 +1,70 @@ +package artifact + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "gitlab.com/slon/shad-go/distbuild/pkg/build" +) + +func TestCache(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + c, err := NewCache(tmpDir) + require.NoError(t, err) + + idA := build.ID{'a'} + + path, commit, _, err := c.Create(idA) + require.NoError(t, err) + + _, _, _, err = c.Create(idA) + require.Equal(t, ErrWriteLocked, err) + + _, err = os.Create(filepath.Join(path, "a.txt")) + require.NoError(t, err) + + require.NoError(t, commit()) + + path, unlock, err := c.Get(idA) + require.NoError(t, err) + defer unlock() + + _, err = os.Stat(filepath.Join(path, "a.txt")) + require.NoError(t, err) + + require.Equal(t, ErrReadLocked, c.Remove(idA)) + + idB := build.ID{'b'} + _, _, err = c.Get(idB) + require.Equal(t, ErrNotFound, err) + + require.NoError(t, c.Range(func(artifact build.ID) error { + require.Equal(t, idA, artifact) + return nil + })) +} + +func TestAbortWrite(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + c, err := NewCache(tmpDir) + require.NoError(t, err) + + idA := build.ID{'a'} + + _, _, abort, err := c.Create(idA) + require.NoError(t, err) + require.NoError(t, abort()) + + _, _, err = c.Get(idA) + require.Equal(t, ErrNotFound, err) +} diff --git a/distbuild/pkg/build/graph.go b/distbuild/pkg/build/graph.go index 66c9c48..af2d1fc 100644 --- a/distbuild/pkg/build/graph.go +++ b/distbuild/pkg/build/graph.go @@ -5,6 +5,7 @@ import ( "encoding" "encoding/hex" "fmt" + "path/filepath" ) type ID [sha1.Size]byte @@ -14,6 +15,14 @@ var ( _ = encoding.TextUnmarshaler(&ID{}) ) +func (id ID) String() string { + return hex.EncodeToString(id[:]) +} + +func (id ID) Path() string { + return filepath.Join(hex.EncodeToString(id[:1]), hex.EncodeToString(id[:])) +} + func (id ID) MarshalText() ([]byte, error) { return []byte(hex.EncodeToString(id[:])), nil } @@ -95,6 +104,10 @@ type Cmd struct { CatOutput string } +func (cmd Cmd) Render(outputDir, sourceDir string, deps map[ID]string) Cmd { + panic("implement me") +} + type Graph struct { SourceFiles map[ID]string