implement mask for hash fields

Makefile

@@ -12,3 +12,4 @@ gen:
 	cd plugin && buf generate
 	cd plugin/pluginexec && buf generate
 	cd plugin/plugingroup && buf generate
+	cd internal/hproto && buf generate

internal/engine/engine.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1"
+
 	"github.com/hephbuild/heph/internal/hcore/hlog"
 	"github.com/hephbuild/heph/internal/hcore/hstep"
 	"github.com/hephbuild/heph/internal/hfs"
@@ -65,6 +67,7 @@ type Engine struct {
 	Drivers       []pluginv1connect.DriverClient
 	DriversHandle map[pluginv1connect.DriverClient]PluginHandle
 	DriversByName map[string]pluginv1connect.DriverClient
+	DriversConfig map[string]*pluginv1.ConfigResponse
 }
 
 func New(ctx context.Context, root string, cfg Config) (*Engine, error) {

internal/engine/local_cache.go

@@ -3,11 +3,9 @@ package engine
 import (
 	"context"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"time"
 
 	"github.com/hephbuild/heph/internal/hartifact"
@@ -109,15 +107,13 @@ func (e *Engine) CacheLocally(ctx context.Context, def *LightLinkedTarget, hashi
 		})
 	}
 
-	manifestfs := hfs.At(cachedir, ArtifactManifestName)
-
-	m := Manifest{
+	m := hartifact.Manifest{
 		Version:   "v1",
 		CreatedAt: time.Now(),
 		Hashin:    hashin,
 	}
 	for _, artifact := range cacheArtifacts {
-		m.Artifacts = append(m.Artifacts, ManifestArtifact{
+		m.Artifacts = append(m.Artifacts, hartifact.ManifestArtifact{
 			Hashout:  artifact.Hashout,
 			Group:    artifact.Group,
 			Name:     artifact.Name,
@@ -126,32 +122,18 @@ func (e *Engine) CacheLocally(ctx context.Context, def *LightLinkedTarget, hashi
 		})
 	}
 
-	b, err := json.Marshal(m) //nolint:musttag
-	if err != nil {
-		return nil, err
-	}
-
-	err = hfs.WriteFile(manifestfs, "", b, os.ModePerm)
+	manifestArtifact, err := hartifact.NewManifestArtifact(cachedir, m)
 	if err != nil {
 		return nil, err
 	}
 
 	cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{
-		Artifact: manifestV1Artifact(cachedir),
+		Artifact: manifestArtifact,
 	})
 
 	return cacheArtifacts, nil
 }
 
-func manifestV1Artifact(fs hfs.OS) *pluginv1.Artifact {
-	return &pluginv1.Artifact{
-		Name:     ArtifactManifestName,
-		Type:     pluginv1.Artifact_TYPE_MANIFEST_V1,
-		Encoding: pluginv1.Artifact_ENCODING_NONE,
-		Uri:      "file://" + hfs.At(fs, ArtifactManifestName).Path(),
-	}
-}
-
 func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) {
 	multi := hlocks.NewMulti()
 
@@ -182,26 +164,20 @@ func (e *Engine) resultFromLocalCacheInner(
 	dirfs := hfs.At(e.Cache, def.Ref.GetPackage(), "__"+def.Ref.GetName(), hashin)
 
 	{
-		l := hlocks.NewFlock2(dirfs, "", ArtifactManifestName, false)
+		l := hlocks.NewFlock2(dirfs, "", hartifact.ManifestName, false)
 		err := l.RLock(ctx)
 		if err != nil {
 			return nil, false, err
 		}
 		locks.Add(l.RUnlock)
 	}
 
-	mainfestb, err := hfs.ReadFile(dirfs, ArtifactManifestName)
+	manifest, err := hartifact.ManifestFromFS(dirfs)
 	if err != nil {
 		return nil, false, err
 	}
 
-	var manifest Manifest
-	err = json.Unmarshal(mainfestb, &manifest) //nolint:musttag
-	if err != nil {
-		return nil, false, err
-	}
-
-	var artifacts []ManifestArtifact
+	var artifacts []hartifact.ManifestArtifact
 	for _, output := range outputs {
 		outputArtifacts := manifest.GetArtifacts(output)
 
@@ -231,8 +207,13 @@ func (e *Engine) resultFromLocalCacheInner(
 		})
 	}
 
+	manifestArtifact, err := hartifact.NewManifestArtifact(dirfs, manifest)
+	if err != nil {
+		return nil, false, err
+	}
+
 	execArtifacts = append(execArtifacts, ExecuteResultArtifact{
-		Artifact: manifestV1Artifact(dirfs),
+		Artifact: manifestArtifact,
 	})
 
 	return &ExecuteResult{

internal/engine/plugins.go

@@ -182,10 +182,14 @@ func (e *Engine) RegisterDriver(ctx context.Context, handler pluginv1connect.Dri
 	if e.DriversHandle == nil {
 		e.DriversHandle = map[pluginv1connect.DriverClient]PluginHandle{}
 	}
+	if e.DriversConfig == nil {
+		e.DriversConfig = map[string]*pluginv1.ConfigResponse{}
+	}
 
 	e.Drivers = append(e.Drivers, client)
 	e.DriversByName[res.Msg.GetName()] = client
 	e.DriversHandle[client] = pluginh
+	e.DriversConfig[res.Msg.GetName()] = res.Msg
 
 	err = e.initPlugin(ctx, handler)
 	if err != nil {

internal/engine/schedule.go

@@ -13,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hephbuild/heph/internal/hproto"
+
 	"connectrpc.com/connect"
 	"github.com/dlsniper/debugger"
 	"github.com/hephbuild/heph/internal/hcore/hlog"
@@ -266,8 +268,15 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []*
 		return "", err
 	}
 
-	// TODO support fieldmask of things to include in hashin
-	b, err = proto.Marshal(def.Def)
+	defHash := def.Def
+	if ignoreFromHash := e.DriversConfig[def.Ref.GetDriver()].GetIgnoreFromHash(); len(ignoreFromHash) > 0 {
+		defHash, err = hproto.RemoveMasked(defHash, ignoreFromHash)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	b, err = proto.Marshal(defHash)
 	if err != nil {
 		return "", err
 	}

internal/enginee2e/hash_deps_test.go

@@ -0,0 +1,83 @@
+package enginee2e
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hephbuild/heph/internal/engine"
+	"github.com/hephbuild/heph/internal/hartifact"
+	pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1"
+	"github.com/hephbuild/heph/plugin/pluginexec"
+	"github.com/hephbuild/heph/plugin/pluginstaticprovider"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func TestHashDeps(t *testing.T) {
+	ctx := context.Background()
+
+	dir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	e, err := engine.New(ctx, dir, engine.Config{})
+	require.NoError(t, err)
+
+	staticprovider := pluginstaticprovider.NewFunc(func() []pluginstaticprovider.Target {
+		return []pluginstaticprovider.Target{
+			{
+				Spec: &pluginv1.TargetSpec{
+					Ref: &pluginv1.TargetRef{
+						Package: "some/package",
+						Name:    "sometarget",
+						Driver:  "sh",
+					},
+					Config: map[string]*structpb.Value{
+						"run": newValueMust([]any{`echo hello > out`}),
+						"out": newValueMust([]any{"out"}),
+						"runtime_env": newValueMust(map[string]any{
+							"VAR1": time.Now().String(),
+						}),
+					},
+				},
+			},
+		}
+	})
+
+	_, err = e.RegisterProvider(ctx, staticprovider)
+	require.NoError(t, err)
+
+	_, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil)
+	require.NoError(t, err)
+
+	var at time.Time
+	{
+		ch := e.Result(ctx, "some/package", "sometarget", []string{""}, engine.ResultOptions{})
+
+		res := <-ch
+		require.NoError(t, res.Err)
+
+		require.Len(t, res.Artifacts, 2)
+
+		m, err := hartifact.ManifestFromArtifact(ctx, res.Artifacts[1].Artifact)
+		require.NoError(t, err)
+
+		at = m.CreatedAt
+	}
+
+	{
+		ch := e.Result(ctx, "some/package", "sometarget", []string{""}, engine.ResultOptions{})
+
+		res := <-ch
+		require.NoError(t, res.Err)
+
+		require.Len(t, res.Artifacts, 2)
+
+		m, err := hartifact.ManifestFromArtifact(ctx, res.Artifacts[1].Artifact)
+		require.NoError(t, err)
+
+		require.Equal(t, at, m.CreatedAt)
+	}
+}

internal/enginee2e/env_test.go → internal/enginee2e/src_out_env_test.go

@@ -14,7 +14,7 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 )
 
-func TestEnv(t *testing.T) {
+func TestSrcOutEnv(t *testing.T) {
 	ctx := context.Background()
 
 	dir, err := os.MkdirTemp("", "")

internal/hartifact/manifest.go

@@ -0,0 +1,93 @@
+package hartifact
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"time"
+
+	"github.com/hephbuild/heph/internal/hfs"
+
+	pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1"
+)
+
+var ManifestName = "manifest.v1.json"
+
+type ManifestArtifact struct {
+	Hashout string
+
+	Group    string
+	Name     string
+	Type     pluginv1.Artifact_Type
+	Encoding pluginv1.Artifact_Encoding
+}
+
+type Manifest struct {
+	Version   string
+	CreatedAt time.Time
+	Hashin    string
+	Artifacts []ManifestArtifact
+}
+
+func (m Manifest) GetArtifacts(output string) []ManifestArtifact {
+	a := make([]ManifestArtifact, 0)
+	for _, artifact := range m.Artifacts {
+		if artifact.Group != output {
+			continue
+		}
+
+		a = append(a, artifact)
+	}
+
+	return a
+}
+
+func NewManifestArtifact(fs hfs.FS, m Manifest) (*pluginv1.Artifact, error) {
+	b, err := json.Marshal(m) //nolint:musttag
+	if err != nil {
+		return nil, err
+	}
+
+	err = hfs.WriteFile(fs, ManifestName, b, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pluginv1.Artifact{
+		Name:     ManifestName,
+		Type:     pluginv1.Artifact_TYPE_MANIFEST_V1,
+		Encoding: pluginv1.Artifact_ENCODING_NONE,
+		Uri:      "file://" + hfs.At(fs, ManifestName).Path(),
+	}, nil
+}
+
+func ManifestFromArtifact(ctx context.Context, a *pluginv1.Artifact) (Manifest, error) {
+	r, err := Reader(ctx, a)
+	if err != nil {
+		return Manifest{}, err
+	}
+	defer r.Close()
+
+	return openManifest(r)
+}
+
+func ManifestFromFS(fs hfs.FS) (Manifest, error) {
+	f, err := hfs.Open(fs, ManifestName)
+	if err != nil {
+		return Manifest{}, err
+	}
+	defer f.Close()
+
+	return openManifest(f)
+}
+
+func openManifest(r io.Reader) (Manifest, error) {
+	var manifest Manifest
+	err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag
+	if err != nil {
+		return Manifest{}, err
+	}
+
+	return manifest, nil
+}