(PUP-5482) Add a benchmark to missing type caching

This commit adds a simple benchmark to measure the effect of caching
type load misses. It works by creating a single populated module with a
defined type that is defined in init.pp. It also creates a number of
other empty modules to expand the required search_path.

Author: haus

benchmarks/missing_type_caching/benchmarker.rb

@@ -0,0 +1,63 @@
+require 'erb'
+require 'ostruct'
+require 'fileutils'
+require 'json'
+
+class Benchmarker
+  include FileUtils
+
+  def initialize(target, size)
+    @target = target
+    @size = size
+  end
+
+  def setup
+    require 'puppet'
+    config = File.join(@target, 'puppet.conf')
+    Puppet.initialize_settings(['--config', config])
+    Puppet[:always_retry_plugins] = false
+  end
+
+  def run(args=nil)
+    env = Puppet.lookup(:environments).get('benchmarking')
+    node = Puppet::Node.new("testing", :environment => env)
+    Puppet::Resource::Catalog.indirection.find("testing", :use_node => node)
+  end
+
+  def generate
+    environment = File.join(@target, 'environments', 'benchmarking')
+    templates = File.join('benchmarks', 'missing_type_caching')
+    test_module_dir = File.join(environment, 'modules', 'testmodule',
+                                'manifests')
+
+    mkdir_p(File.join(environment, 'modules'))
+    @size.times.each do |i|
+      mkdir_p(File.join(environment, 'modules', "mymodule_#{i}", 'lib',
+                        'puppet', 'type'))
+      mkdir_p(File.join(environment, 'modules', "mymodule_#{i}", 'manifests'))
+    end
+    mkdir_p(File.join(environment, 'manifests'))
+
+    mkdir_p(test_module_dir)
+
+
+    render(File.join(templates, 'site.pp.erb'),
+           File.join(environment, 'manifests', 'site.pp'),
+           :size => @size)
+
+    render(File.join(templates, 'module', 'testmodule.pp.erb'),
+           File.join(test_module_dir, 'init.pp'),
+           :name => "foo")
+
+    render(File.join(templates, 'puppet.conf.erb'),
+           File.join(@target, 'puppet.conf'),
+           :location => @target)
+  end
+
+  def render(erb_file, output_file, bindings)
+    site = ERB.new(File.read(erb_file))
+    File.open(output_file, 'w') do |fh|
+      fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding }))
+    end
+  end
+end

benchmarks/missing_type_caching/description

@@ -0,0 +1,3 @@
+Benchmark scenario: heavy use of defined types found in init.pp
+Benchmark target: catalog compilation
+

benchmarks/missing_type_caching/module/testmodule.pp.erb

@@ -0,0 +1,3 @@
+define testmodule {
+  notify { "in <%= name %>: $title": }
+}

benchmarks/missing_type_caching/puppet.conf.erb

@@ -0,0 +1,3 @@
+confdir = <%= location %>
+vardir = <%= location %>
+environmentpath = <%= File.join(location, 'environments') %>

benchmarks/missing_type_caching/site.pp.erb

@@ -0,0 +1,5 @@
+if true {
+<% size.times do |i| %>
+  testmodule { "foo<%= i %>": }
+<% end %>
+}