Allow inclusion of extra tools in sandbox for go backend (#21309)

This adds the git binary to the go backend so that `go mod download` can
retrieve modules in private repositories.
Addresses issues #21292 specifically for private modules and #12625 
This PR also includes a short document on how to use pants with a go
module from a private git repo.

docs/docs/go/private-modules/_category_.json

@@ -0,0 +1,4 @@
+{
+  "label": "Private Modules",
+  "position": 2
+}

docs/docs/go/private-modules/index.mdx

@@ -0,0 +1,46 @@
+---
+    title: Private Modules
+    sidebar_position: 1
+---
+
+Use Golang modules from private repositories
+
+---
+# Using Private Modules in Golang
+
+Pants can build go binaries that use modules from private git repositories.
+To do this specify the private repo(s) in `GOPRIVATE` and provide credentials for the git repo in your `$HOME/.netrc`.
+
+Define the `GOPRIVATE` variable in the `subprocess_env_vars` section of your `pants.toml`. The example below shows the `.netrc` file so that git can authentcate. A simple `go.mod` shows the inclusion of the private module, nothing special here.
+
+
+```toml tab={"label":"pants.toml"}
+[GLOBAL]
+backend_packages.add = [
+  "pants.backend.experimental.go",
+]
+
+
+[golang]
+subprocess_env_vars = [
+    'GOPRIVATE=github.com/your-user/your-module/*',
+    'HOME',
+]
+extra_tools = [
+  'git',
+]
+```
+
+```go tab={"label":"go.mod"}
+module consumer
+
+go 1.22
+
+require github.com/your-user/your-repo/your-module v0.0.1
+```
+
+``` tab={"label":".netrc"}
+machine github.com
+login your-user
+password your-token
+```

docs/notes/2.23.x.md

@@ -117,6 +117,10 @@ Docker inference is improved. Pants can now make inferences by target address fo
 
 Experimental support for a rust-based dockerfile parser can be enabled via `[dockerfile-parser].use_rust_parser` option.
 
+#### Go
+
+Support for including additional binaries in the pants sandbox through the `--golang-extra-tools` option. The `go` tools may require other binaries in certain cases. E.g. When using `go` modules from a private git repository, `go mod download` will invoke `git`. See the [documentation on Go Private Modules](https://www.pantsbuild.org/2.23/docs/go/private-modules) for details
+
 #### JVM
 
 When [the `jvm.reproducible_jars` flag](https://www.pantsbuild.org/2.21/reference/subsystems/jvm#reproducible_jars) is set resources jars are now also made reproducible, previously it was assumed resources jars are reproducible without additional action.

src/python/pants/backend/go/subsystems/golang.py

@@ -30,7 +30,7 @@ class EnvironmentAware(Subsystem.EnvironmentAware):
             default=["<PATH>"],
             help=softwrap(
                 f"""
-                A list of paths to search for Go.
+                A list of paths to search for Go and extra tools needed by go.
 
                 Specify absolute paths to directories with the `go` binary, e.g. `/usr/bin`.
                 Earlier entries will be searched first.
@@ -71,6 +71,18 @@ class EnvironmentAware(Subsystem.EnvironmentAware):
             ),
         )
 
+        _extra_tools = StrListOption(
+            default=[],
+            help=softwrap(
+                """
+                List any additional executable tools required for the `go` tool to work.
+                The paths to these tools will be included in the PATH used in the execution sandbox.
+                E.g. `go mod download` may require the `git` tool to download private modules.
+                """
+            ),
+            advanced=True,
+        )
+
         cgo_gcc_binary_name = StrOption(
             default="gcc",
             advanced=True,
@@ -184,6 +196,10 @@ def iter_path_entries():
 
             return tuple(OrderedSet(iter_path_entries()))
 
+        @property
+        def extra_tools(self) -> tuple[str, ...]:
+            return tuple(sorted(set(self._extra_tools)))
+
     minimum_expected_version = StrOption(
         default="1.17",
         help=softwrap(

src/python/pants/backend/go/util_rules/sdk.py

@@ -9,8 +9,9 @@
 
 from pants.backend.go.subsystems.golang import GolangSubsystem
 from pants.backend.go.util_rules import goroot
+from pants.backend.go.util_rules.go_bootstrap import GoBootstrap
 from pants.backend.go.util_rules.goroot import GoRoot
-from pants.core.util_rules.system_binaries import BashBinary
+from pants.core.util_rules.system_binaries import BashBinary, BinaryShims, BinaryShimsRequest
 from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest
 from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.internals.selectors import Get, MultiGet
@@ -105,11 +106,15 @@ async def go_sdk_invoke_setup(goroot: GoRoot) -> GoSdkRunSetup:
 @rule
 async def setup_go_sdk_process(
     request: GoSdkProcess,
+    go_bootstrap: GoBootstrap,
     go_sdk_run: GoSdkRunSetup,
     bash: BashBinary,
     golang_env_aware: GolangSubsystem.EnvironmentAware,
     goroot: GoRoot,
 ) -> Process:
+    # Use go search path to find extra tools
+    search_path = go_bootstrap.go_search_paths
+
     input_digest, env_vars = await MultiGet(
         Get(Digest, MergeDigests([go_sdk_run.digest, request.input_digest])),
         Get(
@@ -125,6 +130,26 @@ async def setup_go_sdk_process(
         "__PANTS_GO_SDK_CACHE_KEY": f"{goroot.full_version}/{goroot.goos}/{goroot.goarch}",
     }
 
+    immutable_input_digests: dict[str, Digest] = {}
+
+    # Add path to additional tools, such as git, that may be needed by the go tool
+    if golang_env_aware.extra_tools:
+        extra_tools = await Get(
+            BinaryShims,
+            BinaryShimsRequest,
+            BinaryShimsRequest.for_binaries(
+                *golang_env_aware.extra_tools,
+                rationale="allow additional tools for go tools",
+                search_path=search_path,
+            ),
+        )
+        # Prepend path to additional tools
+        if "PATH" in env:
+            env["PATH"] = f"{extra_tools.path_component}:{env['PATH']}"
+        else:
+            env["PATH"] = extra_tools.path_component
+        immutable_input_digests.update(extra_tools.immutable_input_digests)
+
     if request.replace_sandbox_root_in_args:
         env[GoSdkRunSetup.SANDBOX_ROOT_ENV] = "1"
 
@@ -140,6 +165,7 @@ async def setup_go_sdk_process(
     return Process(
         argv=[bash.path, go_sdk_run.script.path, *request.command],
         env=env,
+        immutable_input_digests=immutable_input_digests,
         input_digest=input_digest,
         description=request.description,
         output_files=request.output_files,