Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --blob-exec to run system commands for each blob #169

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package com.madgag.git.bfg.cleaner

import com.google.common.io.ByteStreams
import com.madgag.git.bfg.model.TreeBlobEntry
import com.madgag.git.ThreadLocalObjectDatabaseResources
import java.util.{Arrays => JavaArrays}
import org.eclipse.jgit.lib.Constants.OBJ_BLOB
import scala.collection.JavaConversions._

trait BlobExecModifier extends TreeBlobModifier {

def command: String

val threadLocalObjectDBResources: ThreadLocalObjectDatabaseResources

def fix(entry: TreeBlobEntry) = {
val loader = threadLocalObjectDBResources.reader().open(entry.objectId)
val objectStream = loader.openStream
val variables = System.getenv().map { case (key, value) => s"$key=$value" }.toSeq :+
s"BFG_BLOB=${entry.objectId.name}"
val process = Runtime.getRuntime.exec(command, variables.toArray)

val bytes = ByteStreams.toByteArray(objectStream)
objectStream.close()
process.getOutputStream.write(bytes)
process.getOutputStream.close()

ByteStreams.copy(process.getErrorStream, System.err)
process.getErrorStream.close()

val newBytes = ByteStreams.toByteArray(process.getInputStream)
process.getInputStream.close()

val exitCode = process.waitFor()
if(exitCode != 0) {
throw new RuntimeException(s"Process exited with code $exitCode")
}

if(JavaArrays.equals(bytes, newBytes)) {
entry.withoutName
} else {
val objectId = threadLocalObjectDBResources.inserter().insert(OBJ_BLOB, newBytes)
entry.copy(objectId = objectId).withoutName
}
}

}
15 changes: 14 additions & 1 deletion bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ object CLIConfig {
opt[String]("filter-content-size-threshold").abbr("fs").valueName("<size>").text("only do file-content filtering on files smaller than <size> (default is %1$d bytes)".format(CLIConfig().filterSizeThreshold)).action {
(v, c) => c.copy(filterSizeThreshold = ByteSize.parse(v))
}
opt[String]("blob-exec").abbr("be").valueName("<cmd>").text("execute the system command for each blob").action {
(v, c) => c.copy(blobExec = Some(v))
}
opt[String]('p', "protect-blobs-from").valueName("<refs>").text("protect blobs that appear in the most recent versions of the specified refs (default is 'HEAD')").action {
(v, c) => c.copy(protectBlobsFromRevisions = v.split(',').toSet)
}
Expand Down Expand Up @@ -131,6 +134,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None,
filenameFilters: Seq[Filter[String]] = Nil,
filterSizeThreshold: Int = BlobTextModifier.DefaultSizeThreshold,
textReplacementExpressions: Traversable[String] = List.empty,
blobExec: Option[String] = None,
stripBlobsWithIds: Option[Set[ObjectId]] = None,
lfsConversion: Option[String] = None,
strictObjectChecking: Boolean = false,
Expand Down Expand Up @@ -180,6 +184,15 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None,
new LfsBlobConverter(lfsGlobExpr, repo)
}

lazy val blobExecModifier: Option[BlobExecModifier] = blobExec.map {
execCommand =>
new BlobExecModifier {
val command = execCommand

val threadLocalObjectDBResources: ThreadLocalObjectDatabaseResources = repo.getObjectDatabase.threadLocalResources
}
}

lazy val privateDataRemoval = sensitiveData.getOrElse(Seq(fileDeletion, folderDeletion, blobTextModifier).flatten.nonEmpty)

lazy val objectIdSubstitutor = if (privateDataRemoval) ObjectIdSubstitutor.OldIdsPrivate else ObjectIdSubstitutor.OldIdsPublic
Expand Down Expand Up @@ -217,7 +230,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None,
}
}

Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier, lfsBlobConverter).flatten
Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier, lfsBlobConverter, blobExecModifier).flatten
}

lazy val definesNoWork = treeBlobCleaners.isEmpty && folderDeletion.isEmpty && treeEntryListCleaners.isEmpty
Expand Down