Fix golang ro cache

hephbuild · May 11, 2024 · 3a695f7 · 3a695f7
1 parent 11bcf1d
commit 3a695f7
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 69 deletions.
diff --git a/BUILD b/BUILD
@@ -5,12 +5,6 @@ load("//backend/node", "yarn_toolchain")
 go_toolchain(
     name = "go",
     version = "1.22.2",
-    architectures = [
-        "darwin_amd64",
-        "darwin_arm64",
-        "linux_amd64",
-        "linux_arm64",
-    ],
     env = {
         "GOEXPERIMENT": "rangefunc",
     },

diff --git a/backend/go/BUILD b/backend/go/BUILD
@@ -1,3 +1,5 @@
+backend_pkg = heph.pkg.addr()
+
 cfg = CONFIG["go_backend"]
 
 go = cfg["go"]
@@ -18,23 +20,44 @@ go_toolchain_gosh = group(
     deps = ["go.sh"],
 )
 
-def go_toolchain(name, version, architectures = [], runtime_env = {}, **kwargs):
+def go_std(os, arch, name = None):
+    if not name:
+        name = "_std_lib_{}_{}".format(os, arch)
+
+    xargs = "xargs " + ("-S" if get_os() == "darwin" else "-s") + " 100000"
+
+    return target(
+        name = name,
+        pkg = backend_pkg,
+        tools = go,
+        run = [
+            'export LGOROOT=$(pwd)/goroot',
+            'rm -rf $LGOROOT',
+            'cp -r $(go env GOROOT) $LGOROOT',
+            'export GOROOT=$LGOROOT',
+            'chmod -R 777 $GOROOT',
+            'go install --trimpath std',
+            'go list std > list',
+            'cat list | {} -I[] echo "packagefile []=$OUT_PKG/[].a" | sort -u > $SANDBOX/$OUT_IMPORTCFG'.format(xargs),
+        ],
+        out_env = "rel_root",
+        env = {
+            'GOARCH': arch,
+            'GOOS': os,
+            'GODEBUG': 'installgoroot=all',
+        },
+        out = {
+            'pkg': 'goroot/pkg/{}_{}'.format(os, arch),
+            'importcfg': 'goroot/pkg/{}_{}.importcfg'.format(os, arch),
+        },
+    )
+
+def go_toolchain(name, version, runtime_env = {}, **kwargs):
     run = [
         "./$SRC_INSTALL '{}'".format(version),
         "mv $SRC_GO $OUT_GO",
-        "export GO_OUTDIR=$(pwd)",
-        "export GODEBUG='installgoroot=all'",
     ]
 
-    for arch in architectures:
-        (goos, _, goarch) = arch.partition("_")
-        run.append(
-            "(export GOOS={} && export GOARCH={} && ./$OUT_GO install --trimpath std)".format(
-                goos,
-                goarch,
-            ),
-        )
-
     return target(
         name = name,
         run = run,
@@ -52,7 +75,6 @@ def go_toolchain(name, version, architectures = [], runtime_env = {}, **kwargs):
         },
         cache = heph.cache(history = 1),
         support_files = "go",
-        runtime_env = {"GO_SHARED": "$(shared_stage_dir)"},
         transitive = heph.target_spec(
             runtime_env = runtime_env | {"GO_OUTDIR": "$(outdir)", "GO_SHARED": "$(shared_stage_dir)"},
             **kwargs,
@@ -86,10 +108,6 @@ generate_testmain = target(
     },
 )
 
-backend_dir = heph.pkg.dir()
-
-backend_pkg = heph.pkg.addr()
-
 def go_install(name, pkg, version, bin_name):
     return target(
         name = name,
@@ -182,10 +200,8 @@ def go_mod(
         env = goenv,
     )
 
-    (pkg, _, _) = heph.split(go)
-
     mod_pkgs_gen = [
-        "//" + pkg + ":_std_pkgs_*",
+        backend_pkg + ":_std_lib_*",
         "go_lib",
         "go_build_bin",
         "test",
@@ -276,9 +292,7 @@ def _go_gen_importcfg():
     xargs = "xargs " + ("-S" if get_os() == "darwin" else "-s") + " 100000"
 
     return [
-        'cat "$SRC_STD" | {} -I[] echo "packagefile []=$GO_OUTDIR/go/pkg/${{GOOS}}_${{GOARCH}}/[].a" | sort -u > $SANDBOX/importconfig'.format(xargs),
-        'echo "" >> $SANDBOX/importconfig',
-        'find "$SANDBOX" -name "*.importcfg" | {} -I[] cat [] | sed -e "s:=:=$SANDBOX/:" | sort -u >> $SANDBOX/importconfig'.format(xargs),
+        'find "$SANDBOX" -name "*.importcfg" | {} -I[] cat [] | sed -e "s:=:=$SANDBOX/:" | sort -u > $SANDBOX/importconfig'.format(xargs),
     ]
 
 def _go_compile_cmd(name, import_path, abi, complete, embed_cfg):
@@ -352,7 +366,7 @@ def go_library(
     if len(s_files) > 0:
         abi = target(
             name = pvt_name + "#abi",
-            deps = src_dep if src_dep else s_files,
+            deps = (src_dep if src_dep else s_files) + [go_std(os = os, arch = arch)],
             run = [
                 "eval $(go env)",
                 "touch $OUT_H", # the go Toolchain does this
@@ -391,7 +405,7 @@ def go_library(
                 deps = {
                     "lib": lib + "|a",
                     "hdr": lib + "|h",
-                    "_": src_dep if src_dep else s_file,
+                    "_": (src_dep if src_dep else s_file) + [go_std(os = os, arch = arch)],
                 },
                 run = [
                     "eval $(go env)",
@@ -473,7 +487,7 @@ def go_library(
         embed_cfg = None
 
     deps = deps | {
-        "std": _std_pkgs(os, arch),
+        "std": go_std(os = os, arch = arch),
         "embed": embed_cfg,
     }
 
@@ -497,21 +511,6 @@ def go_library(
         labels = ["go_lib"],
     )
 
-def _std_pkgs(os, arch):
-    (pkg, _, _) = heph.split(go)
-
-    return target(
-        name = "_std_pkgs_{}_{}".format(os, arch),
-        pkg = "//" + pkg,
-        tools = [go],
-        run = "export CGO_ENABLED=1 && go list std > std_list",
-        out = "std_list",
-        env = {
-            "GOOS": os,
-            "GOARCH": arch,
-        },
-    )
-
 def go_build_bin(
     name,
     main,
@@ -534,7 +533,7 @@ def go_build_bin(
     _deps = {
         "libs": libs,
         "main": main,
-        "std": _std_pkgs(os, arch),
+        "std": go_std(os, arch),
     }
 
     return target(

diff --git a/backend/go/go.sh b/backend/go/go.sh
@@ -2,9 +2,9 @@
 
 export GOPATH=${GOPATH:-$GO_SHARED/path}
 export GOCACHE=${GOCACHE:-$GO_SHARED/cache}
-export GOROOT=$GO_OUTDIR/go
+export GOROOT=${GOROOT:-$GO_OUTDIR/go}
 export CGO_ENABLED=${CGO_ENABLED:-0}
 
 set -u
 
-exec $GO_OUTDIR/go/bin/go "$@"
+exec $GOROOT/bin/go "$@"
diff --git a/dev_run.sh b/dev_run.sh
@@ -4,6 +4,7 @@ set -e
 
 export HEPH_CWD=$(pwd)
 export HEPH_SRC_ROOT="<HEPH_SRC_ROOT>"
+export GOEXPERIMENT=rangefunc
 
 cd $HEPH_SRC_ROOT
 

diff --git a/lcache/lcache.go b/lcache/lcache.go
@@ -364,6 +364,30 @@ func (e *LocalCacheState) tarListPath(artifact artifacts.Artifact, target graph.
 	return dir.Join(artifact.Name() + ".list").Abs(), nil
 }
 
+type OutDirMeta struct {
+	Version int
+	Outputs []string
+	CacheRW bool
+}
+
+func (expectedMeta OutDirMeta) decide(currentMeta OutDirMeta) (expand bool, cleanExpand bool) {
+	if !ads.ContainsAll(currentMeta.Outputs, expectedMeta.Outputs) {
+		expand = true
+	}
+
+	if currentMeta.Version != expectedMeta.Version {
+		cleanExpand = true
+	} else if currentMeta.CacheRW != expectedMeta.CacheRW {
+		cleanExpand = true
+	}
+
+	if cleanExpand {
+		expand = true
+	}
+
+	return expand, cleanExpand
+}
+
 func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, outputs []string) (xfs.Path, error) {
 	target := ttarget.GraphTarget()
 
@@ -397,17 +421,10 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 	outDir := cacheDir.Join("_output")
 
 	// Legacy...
-	outDirHashPath := cacheDir.Join("_output_hash").Abs()
-	_ = os.Remove(outDirHashPath)
+	_ = os.Remove(cacheDir.Join("_output_hash").Abs())
 
 	outDirMetaPath := cacheDir.Join("_output_meta").Abs()
 
-	type OutDirMeta struct {
-		Version int
-		Outputs []string
-		CacheRW bool
-	}
-
 	expectedMeta := OutDirMeta{
 		Version: 1,
 		Outputs: outputs,
@@ -427,22 +444,17 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 		var currentMeta OutDirMeta
 		currentMeta.CacheRW = true // Legacy behavior
 		_ = json.Unmarshal(b, &currentMeta)
-		if currentMeta.Version != expectedMeta.Version || !ads.ContainsAll(currentMeta.Outputs, expectedMeta.Outputs) {
-			shouldExpand = true
-		}
 
-		if currentMeta.Version != expectedMeta.Version {
-			shouldCleanExpand = true
-		} else if currentMeta.CacheRW != expectedMeta.CacheRW {
-			shouldCleanExpand = true
-		}
+		shouldExpand, shouldCleanExpand = expectedMeta.decide(currentMeta)
 	}
 
 	if len(outputs) == 0 {
 		shouldExpand = false
 	}
 
 	if shouldExpand {
+		xfs.MakeDirsReadWrite(outDir.Abs())
+
 		status.Emit(ctx, tgt.TargetStatus(target, "Expanding cache..."))
 		if shouldCleanExpand {
 			err = os.RemoveAll(outDir.Abs())
@@ -456,8 +468,6 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 			return outDir, err
 		}
 
-		xfs.MakeDirsReadWrite(outDir.Abs())
-
 		untarDedup := sets.NewStringSet(0)
 
 		for _, name := range outputs {

diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go
@@ -84,6 +84,8 @@ func Make(ctx context.Context, cfg MakeConfig) error {
 		log.Debugf("Make %v took %v", cfg.Dir, time.Since(start))
 	}()
 
+	xfs.MakeDirsReadWrite(cfg.Dir)
+
 	err := os.RemoveAll(cfg.Dir)
 	if err != nil {
 		return err

diff --git a/test/features/deps.BUILD b/test/features/deps.BUILD
@@ -7,8 +7,8 @@ e2e_test(
     name = "e2e_gen_deps",
     cmd = "heph q deps '{}'".format(bin),
     expect_output_contains = """
-//:_std_pkgs_OS_ARCH
 //:go
+//go_backend:_std_lib_OS_ARCH
 //test/go/mod-transitive-gen:_go_lib_
 """.replace("OS", get_os()).replace("ARCH", get_arch()).strip(),
 )