Artifact cache

distbuild/disttest/simple_test.go

@@ -3,6 +3,7 @@ package disttest
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"gitlab.com/slon/shad-go/distbuild/pkg/build"
@@ -27,5 +28,6 @@ func TestSingleCommand(t *testing.T) {
 	var recorder Recorder
 	require.NoError(t, env.Client.Build(env.Ctx, echoGraph, &recorder))
 
-	require.Equal(t, &JobResult{Stdout: "OK", Code: new(int)}, recorder.Jobs[build.ID{'a'}])
+	assert.Len(t, len(recorder.Jobs), 1)
+	assert.Equal(t, &JobResult{Stdout: "OK", Code: new(int)}, recorder.Jobs[build.ID{'a'}])
 }

distbuild/pkg/artifact/cache.go

@@ -1,8 +1,14 @@
 package artifact
 
 import (
+	"encoding/hex"
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
 
 	"gitlab.com/slon/shad-go/distbuild/pkg/build"
 )
@@ -13,26 +19,168 @@ var (
 	ErrReadLocked  = errors.New("file is locked for read")
 )
 
-type Cache struct{}
+type Cache struct {
+	tmpDir   string
+	cacheDir string
 
-func NewCache(root string) (*Cache, error) {
+	mu          sync.Mutex
-	return &Cache{}, nil
+	writeLocked map[build.ID]struct{}
+	readLocked  map[build.ID]int
 }
 
-func (c *Cache) Range(artifactFn func(file build.ID) error) error {
+func NewCache(root string) (*Cache, error) {
-	panic("implement me")
+	tmpDir := filepath.Join(root, "tmp")
+
+	if err := os.RemoveAll(tmpDir); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(tmpDir, 0777); err != nil {
+		return nil, err
+	}
+
+	cacheDir := filepath.Join(root, "c")
+	if err := os.MkdirAll(cacheDir, 0777); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < 256; i++ {
+		d := hex.EncodeToString([]byte{uint8(i)})
+		if err := os.MkdirAll(filepath.Join(cacheDir, d), 0777); err != nil {
+			return nil, err
+		}
+	}
+
+	return &Cache{
+		tmpDir:      tmpDir,
+		cacheDir:    cacheDir,
+		writeLocked: make(map[build.ID]struct{}),
+		readLocked:  make(map[build.ID]int),
+	}, nil
+}
+
+func (c *Cache) readLock(id build.ID) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if _, ok := c.writeLocked[id]; ok {
+		return ErrWriteLocked
+	}
+
+	c.readLocked[id] += 1
+	return nil
+}
+
+func (c *Cache) readUnlock(id build.ID) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.readLocked[id] -= 1
+	if c.readLocked[id] == 0 {
+		delete(c.readLocked, id)
+	}
+}
+
+func (c *Cache) writeLock(id build.ID) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if _, ok := c.writeLocked[id]; ok {
+		return ErrWriteLocked
+	}
+	if c.readLocked[id] > 0 {
+		return ErrReadLocked
+	}
+
+	c.writeLocked[id] = struct{}{}
+	return nil
+}
+
+func (c *Cache) writeUnlock(id build.ID) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	delete(c.writeLocked, id)
+}
+
+func (c *Cache) Range(artifactFn func(artifact build.ID) error) error {
+	shards, err := ioutil.ReadDir(c.cacheDir)
+	if err != nil {
+		return err
+	}
+
+	for _, shard := range shards {
+		dirs, err := ioutil.ReadDir(filepath.Join(c.cacheDir, shard.Name()))
+		if err != nil {
+			return err
+		}
+
+		for _, d := range dirs {
+			var id build.ID
+			if err := id.UnmarshalText([]byte(d.Name())); err != nil {
+				return fmt.Errorf("invalid artifact name: %w", err)
+			}
+
+			if err := artifactFn(id); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
 func (c *Cache) Remove(artifact build.ID) error {
-	panic("implement me")
+	if err := c.writeLock(artifact); err != nil {
+		return err
+	}
+	defer c.writeUnlock(artifact)
+
+	return os.RemoveAll(filepath.Join(c.cacheDir, artifact.Path()))
 }
 
-func (c *Cache) Create(artifact build.ID) (path string, abort, commit func(), err error) {
+func (c *Cache) Create(artifact build.ID) (path string, commit, abort func() error, err error) {
-	panic("implement me")
+	if err = c.writeLock(artifact); err != nil {
+		return
+	}
+
+	path = filepath.Join(c.tmpDir, artifact.String())
+	if err = os.MkdirAll(path, 0777); err != nil {
+		c.writeUnlock(artifact)
+		return
+	}
+
+	abort = func() error {
+		defer c.writeUnlock(artifact)
+		return os.RemoveAll(path)
+	}
+
+	commit = func() error {
+		defer c.writeUnlock(artifact)
+		return os.Rename(path, filepath.Join(c.cacheDir, artifact.Path()))
+	}
+
+	return
 }
 
-func (c *Cache) Get(file build.ID) (path string, unlock func(), err error) {
+func (c *Cache) Get(artifact build.ID) (path string, unlock func(), err error) {
-	panic("implement me")
+	if err = c.readLock(artifact); err != nil {
+		return
+	}
+
+	path = filepath.Join(c.cacheDir, artifact.Path())
+	if _, err = os.Stat(path); err != nil {
+		c.readUnlock(artifact)
+
+		if os.IsNotExist(err) {
+			err = ErrNotFound
+		}
+		return
+	}
+
+	unlock = func() {
+		c.readUnlock(artifact)
+	}
+	return
 }
 
 func NewHandler(c *Cache) http.Handler {

distbuild/pkg/artifact/cache_test.go

@@ -0,0 +1,70 @@
+package artifact
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/slon/shad-go/distbuild/pkg/build"
+)
+
+func TestCache(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	c, err := NewCache(tmpDir)
+	require.NoError(t, err)
+
+	idA := build.ID{'a'}
+
+	path, commit, _, err := c.Create(idA)
+	require.NoError(t, err)
+
+	_, _, _, err = c.Create(idA)
+	require.Equal(t, ErrWriteLocked, err)
+
+	_, err = os.Create(filepath.Join(path, "a.txt"))
+	require.NoError(t, err)
+
+	require.NoError(t, commit())
+
+	path, unlock, err := c.Get(idA)
+	require.NoError(t, err)
+	defer unlock()
+
+	_, err = os.Stat(filepath.Join(path, "a.txt"))
+	require.NoError(t, err)
+
+	require.Equal(t, ErrReadLocked, c.Remove(idA))
+
+	idB := build.ID{'b'}
+	_, _, err = c.Get(idB)
+	require.Equal(t, ErrNotFound, err)
+
+	require.NoError(t, c.Range(func(artifact build.ID) error {
+		require.Equal(t, idA, artifact)
+		return nil
+	}))
+}
+
+func TestAbortWrite(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	c, err := NewCache(tmpDir)
+	require.NoError(t, err)
+
+	idA := build.ID{'a'}
+
+	_, _, abort, err := c.Create(idA)
+	require.NoError(t, err)
+	require.NoError(t, abort())
+
+	_, _, err = c.Get(idA)
+	require.Equal(t, ErrNotFound, err)
+}

distbuild/pkg/build/graph.go

@@ -5,6 +5,7 @@ import (
 	"encoding"
 	"encoding/hex"
 	"fmt"
+	"path/filepath"
 )
 
 type ID [sha1.Size]byte
@@ -14,6 +15,14 @@ var (
 	_ = encoding.TextUnmarshaler(&ID{})
 )
 
+func (id ID) String() string {
+	return hex.EncodeToString(id[:])
+}
+
+func (id ID) Path() string {
+	return filepath.Join(hex.EncodeToString(id[:1]), hex.EncodeToString(id[:]))
+}
+
 func (id ID) MarshalText() ([]byte, error) {
 	return []byte(hex.EncodeToString(id[:])), nil
 }
@@ -95,6 +104,10 @@ type Cmd struct {
 	CatOutput string
 }
 
+func (cmd Cmd) Render(outputDir, sourceDir string, deps map[ID]string) Cmd {
+	panic("implement me")
+}
+
 type Graph struct {
 	SourceFiles map[ID]string