Add --jobs (parallel git fetch) (#539)

Co-authored-by: Johannes Müller <[email protected]>

docs/shards.adoc

@@ -58,7 +58,7 @@ Exit status:
 *init*::
 Initializes a default _shard.yml_ in the current folder.
 
-*install* [--frozen] [--without-development] [--production] [--skip-postinstall] [--skip-executables]::
+*install* [--frozen] [--without-development] [--production] [--skip-postinstall] [--skip-executables] [--jobs=N]::
 Resolves and installs dependencies into the _lib_ folder. If not already
 present, generates a _shard.lock_ file from resolved dependencies, locking
 version numbers or Git commits.
@@ -74,6 +74,7 @@ doesn't generate a conflict, thus generating a new _shard.lock_ file.
 --production:: same as _--frozen_ and _--without-development_
 --skip-postinstall:: Does not run postinstall of dependencies.
 --skip-executables:: Does not install executables.
+--jobs:: Number of repository downloads to perform in parallel (default: 8). Currently only for git.
 --
 
 *list* [--tree]::

src/cli.cr

@@ -48,6 +48,7 @@ module Shards
         self.skip_executables = true
       end
       opts.on("--local", "Don't update remote repositories, use the local cache only.") { self.local = true }
+      opts.on("--jobs=N", "Number of repository downloads to perform in parallel (default: 8). Currently only for git.") { |n| self.jobs = n.to_i }
       # TODO: remove in the future
       opts.on("--ignore-crystal-version", "Has no effect. Kept for compatibility, to be removed in the future.") { }
       opts.on("-v", "--verbose", "Increase the log verbosity, printing all debug statements.") { self.set_debug_log_level }

src/config.cr

@@ -108,4 +108,6 @@ module Shards
   class_property? local = false
   class_property? skip_postinstall = false
   class_property? skip_executables = false
+
+  class_property jobs : Int32 = 8
 end

src/molinillo_solver.cr

@@ -47,7 +47,39 @@ module Shards
       end
     end
 
+    private def prefetch_local_caches(deps)
+      return unless Shards.jobs > 1
+
+      count = 0
+      active = Atomic.new(0)
+      ch = Channel(Exception?).new(deps.size + 1)
+      deps.each do |dep|
+        count += 1
+        active.add(1)
+        while active.get > Shards.jobs
+          sleep 0.1
+        end
+        spawn do
+          begin
+            dep.resolver.update_local_cache if dep.resolver.is_a? GitResolver
+            ch.send(nil)
+          rescue ex : Exception
+            ch.send(ex)
+          ensure
+            active.sub(1)
+          end
+        end
+      end
+
+      count.times do
+        obj = ch.receive
+        raise obj if obj.is_a? Exception
+      end
+    end
+
     private def add_lock(base, lock_index, deps : Array(Dependency))
+      prefetch_local_caches(deps)
+
       deps.each do |dep|
         if lock = lock_index[dep.name]?
           next unless dep.matches?(lock.version)
@@ -64,6 +96,8 @@ module Shards
              end
       deps = apply_overrides(deps)
 
+      prefetch_local_caches(deps)
+
       base = Molinillo::DependencyGraph(Dependency, Dependency).new
       if locks = @locks
         lock_index = locks.to_h { |d| {d.name, d} }

src/resolvers/fossil.cr

@@ -333,7 +333,7 @@ module Shards
       end
     end
 
-    private def update_local_cache
+    def update_local_cache
       if cloned_repository? && origin_changed?
         delete_repository
         @updated_cache = false

src/resolvers/git.cr

@@ -283,7 +283,7 @@ module Shards
       end
     end
 
-    private def update_local_cache
+    def update_local_cache
       if cloned_repository? && origin_changed?
         delete_repository
         @updated_cache = false
@@ -317,7 +317,7 @@ module Shards
       # This configuration can be overridden by defining the environment
       # variable `GIT_ASKPASS`.
       git_retry(err: "Failed to clone #{git_url}") do
-        run_in_current_folder "git clone -c core.askPass=true -c init.templateDir= --mirror --quiet -- #{Process.quote(git_url)} #{Process.quote(local_path)}"
+        run_in_folder "git clone -c core.askPass=true -c init.templateDir= --mirror --quiet -- #{Process.quote(git_url)} #{Process.quote(local_path)}"
       end
     end
 
@@ -411,12 +411,12 @@ module Shards
         dependency_name = File.basename(path, ".git")
         raise Error.new("Missing repository cache for #{dependency_name.inspect}. Please run without --local to fetch it.")
       end
-      Dir.cd(path) do
-        run_in_current_folder(command, capture)
-      end
+      run_in_folder(command, path, capture)
     end
 
-    private def run_in_current_folder(command, capture = false)
+    # Chdir to a folder and run command.
+    # Runs in current folder if `path` is nil.
+    private def run_in_folder(command, path : String? = nil, capture = false)
       unless GitResolver.has_git_command?
         raise Error.new("Error missing git command line tool. Please install Git first!")
       end
@@ -425,7 +425,7 @@ module Shards
 
       output = capture ? IO::Memory.new : Process::Redirect::Close
       error = IO::Memory.new
-      status = Process.run(command, shell: true, output: output, error: error)
+      status = Process.run(command, shell: true, output: output, error: error, chdir: path)
 
       if status.success?
         output.to_s if capture

src/resolvers/hg.cr

@@ -302,7 +302,7 @@ module Shards
       end
     end
 
-    private def update_local_cache
+    def update_local_cache
       if cloned_repository? && origin_changed?
         delete_repository
         @updated_cache = false

src/resolvers/resolver.cr

@@ -88,6 +88,9 @@ module Shards
     abstract def install_sources(version : Version, install_dir : String)
     abstract def report_version(version : Version) : String
 
+    def update_local_cache
+    end
+
     def parse_requirement(params : Hash(String, String)) : Requirement
       if version = params["version"]?
         VersionReq.new version