[ABLD-256] First version of rules to make installer symlinks

MODULE.bazel

@@ -2,7 +2,21 @@
 
 http_archive = use_repo_rule("//third_party/bazel/tools/build_defs/repo:http.bzl", "http_archive")
 
-visual_studio = use_repo_rule("//bazel/rules:visual_studio.bzl", "visual_studio")
+env_vars = use_repo_rule("//bazel/repo:env_vars.bzl", "env_vars")
+
+# Environment values available for packaging.
+env_vars(
+    name = "agent_volatile",
+    # Each variable needs a justification. Every time you use one, the
+    # target using it is potentially uncachable.
+    variables = [
+        # For some tests we want to run with higher compression, but
+        # not break caching and rebuild the binaries.
+        "FORCED_PACKAGE_COMPRESSION_LEVEL",
+        # This is the build pipeline id, which is used in some symlink paths.
+        "PACKAGE_VERSION",
+    ],
+)
 
 # package_metadata and rules_license are dependencies of every module in the
 # Bazel ecosystem.  We pin those first to get the versions we require.
@@ -85,6 +99,8 @@ register_toolchains(
     dev_dependency = True,
 )
 
+visual_studio = use_repo_rule("//bazel/rules:visual_studio.bzl", "visual_studio")
+
 ######################################
 ## Compilers and toolchains        ##
 ####################################

bazel/repo/BUILD.bazel

@@ -0,0 +1 @@
+"""Local repository rules."""

bazel/repo/env_vars.bzl

@@ -0,0 +1,77 @@
+"""Import selected build environment variables to a repo where we can use them.
+
+This method allows us to pull in an isolated set of variables without
+polluting the build configuration and breaking caching.
+
+Bazel will treat changes to the value of these variables across builds
+as a change to the repository, and cause the repo to be rebuilt, so it
+is important to depend on these from as few places, and as high in the
+build graph, as possible.  That is, it is acceptable to use them from
+final packaging steps, but not to use for select clauses, or as input
+to libraries and binaries.
+
+Usage:
+    env_vars = use_repo_rule("//bazel/repo:env_vars.bzl", "env_vars")
+    env_vars(
+        name = "agent_volatile",
+        variables = [
+            "FORCED_PACKAGE_COMPRESSION_LEVEL",
+            "PACKAGE_VERSION",
+        ],
+    )
+
+Future ideas:
+
+1. Add a rule in the .bzl file that provides rules_pkg PackageVariables. That
+   makes them immediately usable for things like the .deb file name.
+
+2. If we find that we need this technique more often, and it becomes a caching problem,
+we could consider emitting one .bzl file per variable. so PACKAGE_VERSION would be
+found in package_version.bzl. But that is overly complex until we prove we need it.
+"""
+
+BUILD_FILE_CONTENT = """
+exports_files(
+    ["env_vars.bzl"],
+    # We may expand this in the future, but let's limit inventiveness for now.
+    visibility([
+        "@agent//bazel/rules/...",
+        "@agent//packages/...",
+    ]),
+)
+"""
+
+VARIABLE_FILE_CONTENT = """
+
+env_vars = struct(
+%s
+)
+"""
+
+def _env_vars_impl(rctx):
+    rctx.file("BUILD.bazel", BUILD_FILE_CONTENT)
+    var_list = []
+    for var in rctx.attr.variables:
+        value = rctx.getenv(var)
+        rctx.report_progress("Importing %s='%s'" % (var, value))
+
+        # Not importing floats.  If we are doing that, we are thinking wrong.
+        if value == None:
+            var_list.append("    %s = None" % var)
+        elif type(value) == "int":
+            var_list.append("    %s = %d" % (var, value))
+        else:
+            var_list.append("    %s = \"%s\"" % (var, value))
+
+    rctx.file("env_vars.bzl", VARIABLE_FILE_CONTENT % ",\n".join(var_list))
+
+env_vars = repository_rule(
+    implementation = _env_vars_impl,
+    attrs = {
+        "variables": attr.string_list(
+            doc = "List of environment variables to export.",
+        ),
+    },
+    configure = True,
+    doc = """Import environment variables into a build.""",
+)

bazel/rules/BUILD.bazel

@@ -25,6 +25,12 @@ bzl_library(
     visibility = ["//visibility:public"],
 )
 
+bzl_library(
+    name = "dd_agent_pkg_mklink_bzl",
+    srcs = ["dd_agent_pkg_mklink.bzl"],
+    visibility = ["//visibility:public"],
+)
+
 bzl_library(
     name = "preprocessor_bzl",
     srcs = ["preprocessor.bzl"],

bazel/rules/dd_agent_pkg_mklink.bzl

@@ -0,0 +1,75 @@
+"""dd_agent_pkg_mklink. Expand a template, splicing in agent specific flags."""
+
+load("@agent_volatile//:env_vars.bzl", "env_vars")
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
+load("@rules_pkg//pkg:providers.bzl", "PackageSymlinkInfo")
+
+# The place we will install to if we run bazel pkg_install without a destdir
+# We use /tmp for lack of a better safe space.
+DEFAULT_OUTPUT_CONFIG_DIR = "/tmp"
+
+# The location where the product should be installed on a user system.
+DEFAULT_PRODUCT_DIR = "/opt/datadog-agent"
+
+def _dd_agent_pkg_mklink_impl(ctx):
+    # Set up a fallback default.
+    subs = {}
+
+    # TODO: Consider sharing common logic with dd_agent_expand_template IFF we find
+    #       the alignment in variable names gets larger.
+    # TODO: should this be different for windows? Or should we have different variables for windows?
+    subs["output_config_dir"] = DEFAULT_OUTPUT_CONFIG_DIR
+    if ctx.attr._output_config_dir and BuildSettingInfo in ctx.attr._output_config_dir:
+        output_config_dir = ctx.attr._output_config_dir[BuildSettingInfo].value.rstrip("/")
+        subs["output_config_dir"] = output_config_dir
+    subs["install_dir"] = DEFAULT_PRODUCT_DIR
+    if ctx.attr._install_dir and BuildSettingInfo in ctx.attr._install_dir:
+        install_dir = ctx.attr._install_dir[BuildSettingInfo].value
+        subs["install_dir"] = install_dir
+
+    # The environment variable is PACKAGE_VERSION but the omnibus scripts
+    # use build_version. Let's unify that in the future. For now, it is not
+    # clear which direction we should move towards.
+    subs["build_version"] = env_vars.PACKAGE_VERSION or "_build_version_unset_"
+
+    # Default our link mode to 755
+    out_attributes = json.decode(ctx.attr.attributes)
+    out_attributes.setdefault("mode", "0755")
+    return [
+        PackageSymlinkInfo(
+            destination = ctx.attr.link_name.format(**subs),
+            target = ctx.attr.target.format(**subs),
+            attributes = out_attributes,
+        ),
+    ]
+
+dd_agent_pkg_mklink = rule(
+    implementation = _dd_agent_pkg_mklink_impl,
+    doc = """pkg_mklink with substitution from select Agent flags.
+
+This performs a simple search over the template file for the keys in
+substitutions, and replaces them with the corresponding values.  There are
+some default substitutions which are computed from the build environment.
+
+Default substitutions:
+  {install_dir}:  The value of the flag //:install_dir
+  {output_config_dir}:  The value of the flag //:output_config_dir
+  {build_version}: The pipeline build version.
+""",
+    attrs = {
+        "target": attr.string(
+            doc = """See @rules_pkg for documentation.""",
+            mandatory = True,
+        ),
+        "link_name": attr.string(
+            doc = """See @rules_pkg for documentation.""",
+            mandatory = True,
+        ),
+        "attributes": attr.string(
+            doc = """See @rules_pkg for documentation.""",
+            default = "{}",  # Empty JSON
+        ),
+        "_install_dir": attr.label(default = "@@//:install_dir"),
+        "_output_config_dir": attr.label(default = "@@//:output_config_dir"),
+    },
+)

omnibus/config/software/datadog-agent-installer-symlinks.rb

@@ -17,6 +17,12 @@
       link "/opt/datadog-packages/datadog-agent/stable", "/opt/datadog-packages/datadog-agent/experiment"
       project.extra_package_file "/opt/datadog-packages/datadog-agent"
       project.extra_package_file "/opt/datadog-packages/run"
+
+      # TODO: Next PR, enable this and delete the rest.
+      # NOTE TO REVIEWER: I will delete these lines before commit. They are only for
+      command_on_repo_root "bazelisk run -- //packages/agent/linux:install --destdir='/opt/datadog-packages/pr45475'"
+      command_on_repo_root "find /opt/datadog-packages", :live_stream => Omnibus.logger.live_stream(:info)
+      command_on_repo_root "ls -lR /opt/datadog-packages", :live_stream => Omnibus.logger.live_stream(:info)
     end
   end
 end

packages/agent/linux/BUILD.bazel

@@ -1,7 +1,17 @@
-# agent linux distribution packages
+"""Agent package components specific to linux."""
 
+load("@rules_pkg//pkg:install.bzl", "pkg_install")
+load(
+    "@rules_pkg//pkg:mappings.bzl",
+    "pkg_attributes",
+    "pkg_filegroup",
+    "pkg_mkdirs",
+)
+load("//bazel/rules:dd_agent_pkg_mklink.bzl", "dd_agent_pkg_mklink")
 load("//compliance:package_licenses.bzl", "package_licenses")
 
+package(default_visibility = ["//packages:__subpackages__"])
+
 # This is a placeholder for an eventual group containg (the future) top level binaries
 # such as //cmd/agent:agent
 filegroup(
@@ -20,3 +30,60 @@ package_licenses(
     name = "license_files",
     src = ":all_deps",
 )
+
+# Replacement for datadog-agent-installer-symlinks.rb
+
+pkg_mkdirs(
+    name = "dirs",
+    attributes = pkg_attributes(mode = "755"),
+    dirs = [
+        "opt/datadog-packages",
+        "opt/datadog-packages/datadog-agent",
+        "opt/datadog-packages/run",
+        "opt/datadog-packages/run/datadog-agent",
+    ],
+)
+
+dd_agent_pkg_mklink(
+    name = "experiment_link",
+    attributes = pkg_attributes(mode = "755"),
+    link_name = "/opt/datadog-packages/datadog-agent/experiment",
+    target = "/opt/datadog-packages/datadog-agent/stable",
+)
+
+# TODO: Figure out how to point to the build version
+# stable -> /opt/datadog-packages/run/datadog-agent/7.75.0-devel.git.510.0eb2d89.pipeline.87895082
+dd_agent_pkg_mklink(
+    name = "stable_link",
+    attributes = pkg_attributes(mode = "755"),
+    link_name = "/opt/datadog-packages/datadog-agent/stable",
+    target = "/opt/datadog-packages/run/datadog-agent/{build_version}",
+    #link "/opt/datadog-agent", "/opt/datadog-packages/run/datadog-agent/{build_version}"
+)
+
+dd_agent_pkg_mklink(
+    name = "version_to_install_link",
+    attributes = pkg_attributes(mode = "755"),
+    link_name = "opt/datadog-packages/run/datadog-agent/{build_version}",
+    target = "{install_dir}",
+)
+
+# note that this does not follow the naming pattern of "all_files".
+# Instead, I'm using the name of the omnibus script it will replace.
+# That provides a clearer meaning during the migration than the name
+# of the package (packages/agent/linux) alone.
+pkg_filegroup(
+    name = "datadog_agent_installer_symlinks",
+    srcs = [
+        ":experiment_link",
+        ":stable_link",
+        ":version_to_install_link",
+    ],
+)
+
+pkg_install(
+    name = "install",
+    srcs = [
+        ":datadog_agent_installer_symlinks",
+    ],
+)