From 3a695f78efd6658e1450d978cdd4c7b993d0b2bc Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sat, 11 May 2024 14:58:48 +0100 Subject: [PATCH] Fix golang ro cache --- BUILD | 6 --- backend/go/BUILD | 83 ++++++++++++++++++++-------------------- backend/go/go.sh | 4 +- dev_run.sh | 1 + lcache/lcache.go | 46 +++++++++++++--------- sandbox/sandbox.go | 2 + test/features/deps.BUILD | 2 +- 7 files changed, 75 insertions(+), 69 deletions(-) diff --git a/BUILD b/BUILD index 1dcfb53e..964b1a41 100644 --- a/BUILD +++ b/BUILD @@ -5,12 +5,6 @@ load("//backend/node", "yarn_toolchain") go_toolchain( name = "go", version = "1.22.2", - architectures = [ - "darwin_amd64", - "darwin_arm64", - "linux_amd64", - "linux_arm64", - ], env = { "GOEXPERIMENT": "rangefunc", }, diff --git a/backend/go/BUILD b/backend/go/BUILD index 653d230f..0bfdb882 100644 --- a/backend/go/BUILD +++ b/backend/go/BUILD @@ -1,3 +1,5 @@ +backend_pkg = heph.pkg.addr() + cfg = CONFIG["go_backend"] go = cfg["go"] @@ -18,23 +20,44 @@ go_toolchain_gosh = group( deps = ["go.sh"], ) -def go_toolchain(name, version, architectures = [], runtime_env = {}, **kwargs): +def go_std(os, arch, name = None): + if not name: + name = "_std_lib_{}_{}".format(os, arch) + + xargs = "xargs " + ("-S" if get_os() == "darwin" else "-s") + " 100000" + + return target( + name = name, + pkg = backend_pkg, + tools = go, + run = [ + 'export LGOROOT=$(pwd)/goroot', + 'rm -rf $LGOROOT', + 'cp -r $(go env GOROOT) $LGOROOT', + 'export GOROOT=$LGOROOT', + 'chmod -R 777 $GOROOT', + 'go install --trimpath std', + 'go list std > list', + 'cat list | {} -I[] echo "packagefile []=$OUT_PKG/[].a" | sort -u > $SANDBOX/$OUT_IMPORTCFG'.format(xargs), + ], + out_env = "rel_root", + env = { + 'GOARCH': arch, + 'GOOS': os, + 'GODEBUG': 'installgoroot=all', + }, + out = { + 'pkg': 'goroot/pkg/{}_{}'.format(os, arch), + 'importcfg': 'goroot/pkg/{}_{}.importcfg'.format(os, arch), + }, + ) + +def go_toolchain(name, version, runtime_env = {}, **kwargs): run = [ "./$SRC_INSTALL '{}'".format(version), "mv $SRC_GO $OUT_GO", - "export GO_OUTDIR=$(pwd)", - "export GODEBUG='installgoroot=all'", ] - for arch in architectures: - (goos, _, goarch) = arch.partition("_") - run.append( - "(export GOOS={} && export GOARCH={} && ./$OUT_GO install --trimpath std)".format( - goos, - goarch, - ), - ) - return target( name = name, run = run, @@ -52,7 +75,6 @@ def go_toolchain(name, version, architectures = [], runtime_env = {}, **kwargs): }, cache = heph.cache(history = 1), support_files = "go", - runtime_env = {"GO_SHARED": "$(shared_stage_dir)"}, transitive = heph.target_spec( runtime_env = runtime_env | {"GO_OUTDIR": "$(outdir)", "GO_SHARED": "$(shared_stage_dir)"}, **kwargs, @@ -86,10 +108,6 @@ generate_testmain = target( }, ) -backend_dir = heph.pkg.dir() - -backend_pkg = heph.pkg.addr() - def go_install(name, pkg, version, bin_name): return target( name = name, @@ -182,10 +200,8 @@ def go_mod( env = goenv, ) - (pkg, _, _) = heph.split(go) - mod_pkgs_gen = [ - "//" + pkg + ":_std_pkgs_*", + backend_pkg + ":_std_lib_*", "go_lib", "go_build_bin", "test", @@ -276,9 +292,7 @@ def _go_gen_importcfg(): xargs = "xargs " + ("-S" if get_os() == "darwin" else "-s") + " 100000" return [ - 'cat "$SRC_STD" | {} -I[] echo "packagefile []=$GO_OUTDIR/go/pkg/${{GOOS}}_${{GOARCH}}/[].a" | sort -u > $SANDBOX/importconfig'.format(xargs), - 'echo "" >> $SANDBOX/importconfig', - 'find "$SANDBOX" -name "*.importcfg" | {} -I[] cat [] | sed -e "s:=:=$SANDBOX/:" | sort -u >> $SANDBOX/importconfig'.format(xargs), + 'find "$SANDBOX" -name "*.importcfg" | {} -I[] cat [] | sed -e "s:=:=$SANDBOX/:" | sort -u > $SANDBOX/importconfig'.format(xargs), ] def _go_compile_cmd(name, import_path, abi, complete, embed_cfg): @@ -352,7 +366,7 @@ def go_library( if len(s_files) > 0: abi = target( name = pvt_name + "#abi", - deps = src_dep if src_dep else s_files, + deps = (src_dep if src_dep else s_files) + [go_std(os = os, arch = arch)], run = [ "eval $(go env)", "touch $OUT_H", # the go Toolchain does this @@ -391,7 +405,7 @@ def go_library( deps = { "lib": lib + "|a", "hdr": lib + "|h", - "_": src_dep if src_dep else s_file, + "_": (src_dep if src_dep else s_file) + [go_std(os = os, arch = arch)], }, run = [ "eval $(go env)", @@ -473,7 +487,7 @@ def go_library( embed_cfg = None deps = deps | { - "std": _std_pkgs(os, arch), + "std": go_std(os = os, arch = arch), "embed": embed_cfg, } @@ -497,21 +511,6 @@ def go_library( labels = ["go_lib"], ) -def _std_pkgs(os, arch): - (pkg, _, _) = heph.split(go) - - return target( - name = "_std_pkgs_{}_{}".format(os, arch), - pkg = "//" + pkg, - tools = [go], - run = "export CGO_ENABLED=1 && go list std > std_list", - out = "std_list", - env = { - "GOOS": os, - "GOARCH": arch, - }, - ) - def go_build_bin( name, main, @@ -534,7 +533,7 @@ def go_build_bin( _deps = { "libs": libs, "main": main, - "std": _std_pkgs(os, arch), + "std": go_std(os, arch), } return target( diff --git a/backend/go/go.sh b/backend/go/go.sh index 2cad073f..83c53370 100755 --- a/backend/go/go.sh +++ b/backend/go/go.sh @@ -2,9 +2,9 @@ export GOPATH=${GOPATH:-$GO_SHARED/path} export GOCACHE=${GOCACHE:-$GO_SHARED/cache} -export GOROOT=$GO_OUTDIR/go +export GOROOT=${GOROOT:-$GO_OUTDIR/go} export CGO_ENABLED=${CGO_ENABLED:-0} set -u -exec $GO_OUTDIR/go/bin/go "$@" +exec $GOROOT/bin/go "$@" diff --git a/dev_run.sh b/dev_run.sh index 0edfb7da..028169d6 100755 --- a/dev_run.sh +++ b/dev_run.sh @@ -4,6 +4,7 @@ set -e export HEPH_CWD=$(pwd) export HEPH_SRC_ROOT="" +export GOEXPERIMENT=rangefunc cd $HEPH_SRC_ROOT diff --git a/lcache/lcache.go b/lcache/lcache.go index 6742dab9..b9bba7e4 100644 --- a/lcache/lcache.go +++ b/lcache/lcache.go @@ -364,6 +364,30 @@ func (e *LocalCacheState) tarListPath(artifact artifacts.Artifact, target graph. return dir.Join(artifact.Name() + ".list").Abs(), nil } +type OutDirMeta struct { + Version int + Outputs []string + CacheRW bool +} + +func (expectedMeta OutDirMeta) decide(currentMeta OutDirMeta) (expand bool, cleanExpand bool) { + if !ads.ContainsAll(currentMeta.Outputs, expectedMeta.Outputs) { + expand = true + } + + if currentMeta.Version != expectedMeta.Version { + cleanExpand = true + } else if currentMeta.CacheRW != expectedMeta.CacheRW { + cleanExpand = true + } + + if cleanExpand { + expand = true + } + + return expand, cleanExpand +} + func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, outputs []string) (xfs.Path, error) { target := ttarget.GraphTarget() @@ -397,17 +421,10 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou outDir := cacheDir.Join("_output") // Legacy... - outDirHashPath := cacheDir.Join("_output_hash").Abs() - _ = os.Remove(outDirHashPath) + _ = os.Remove(cacheDir.Join("_output_hash").Abs()) outDirMetaPath := cacheDir.Join("_output_meta").Abs() - type OutDirMeta struct { - Version int - Outputs []string - CacheRW bool - } - expectedMeta := OutDirMeta{ Version: 1, Outputs: outputs, @@ -427,15 +444,8 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou var currentMeta OutDirMeta currentMeta.CacheRW = true // Legacy behavior _ = json.Unmarshal(b, ¤tMeta) - if currentMeta.Version != expectedMeta.Version || !ads.ContainsAll(currentMeta.Outputs, expectedMeta.Outputs) { - shouldExpand = true - } - if currentMeta.Version != expectedMeta.Version { - shouldCleanExpand = true - } else if currentMeta.CacheRW != expectedMeta.CacheRW { - shouldCleanExpand = true - } + shouldExpand, shouldCleanExpand = expectedMeta.decide(currentMeta) } if len(outputs) == 0 { @@ -443,6 +453,8 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou } if shouldExpand { + xfs.MakeDirsReadWrite(outDir.Abs()) + status.Emit(ctx, tgt.TargetStatus(target, "Expanding cache...")) if shouldCleanExpand { err = os.RemoveAll(outDir.Abs()) @@ -456,8 +468,6 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou return outDir, err } - xfs.MakeDirsReadWrite(outDir.Abs()) - untarDedup := sets.NewStringSet(0) for _, name := range outputs { diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 658998d2..050b5625 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -84,6 +84,8 @@ func Make(ctx context.Context, cfg MakeConfig) error { log.Debugf("Make %v took %v", cfg.Dir, time.Since(start)) }() + xfs.MakeDirsReadWrite(cfg.Dir) + err := os.RemoveAll(cfg.Dir) if err != nil { return err diff --git a/test/features/deps.BUILD b/test/features/deps.BUILD index c87f6dc9..0262832b 100644 --- a/test/features/deps.BUILD +++ b/test/features/deps.BUILD @@ -7,8 +7,8 @@ e2e_test( name = "e2e_gen_deps", cmd = "heph q deps '{}'".format(bin), expect_output_contains = """ -//:_std_pkgs_OS_ARCH //:go +//go_backend:_std_lib_OS_ARCH //test/go/mod-transitive-gen:_go_lib_ """.replace("OS", get_os()).replace("ARCH", get_arch()).strip(), )