add benchmarks for FossilDelta algorithm. #1

endel · Nov 7, 2016 · e3d891b · e3d891b
1 parent e47411b
commit e3d891b
Show file tree

Hide file tree

Showing 8 changed files with 182 additions and 22 deletions.
diff --git a/Benchmarks/Benchmarks.csproj b/Benchmarks/Benchmarks.csproj
@@ -53,23 +53,24 @@
     <Reference Include="System.Threading.Tasks.Extensions">
       <HintPath>..\packages\System.Threading.Tasks.Extensions.4.0.0\lib\portable-net45+win8+wp8+wpa81\System.Threading.Tasks.Extensions.dll</HintPath>
     </Reference>
-    <Reference Include="BenchmarkDotNet.Core">
-      <HintPath>..\packages\BenchmarkDotNet.Core.0.9.9\lib\net45\BenchmarkDotNet.Core.dll</HintPath>
-    </Reference>
     <Reference Include="System.Management" />
     <Reference Include="System.Xml" />
+    <Reference Include="BenchmarkDotNet.Core">
+      <HintPath>..\packages\BenchmarkDotNet.Core.0.9.9.182\lib\net45\BenchmarkDotNet.Core.dll</HintPath>
+    </Reference>
     <Reference Include="BenchmarkDotNet.Toolchains.Roslyn">
-      <HintPath>..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll</HintPath>
+      <HintPath>..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9.182\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll</HintPath>
     </Reference>
     <Reference Include="BenchmarkDotNet">
-      <HintPath>..\packages\BenchmarkDotNet.0.9.9\lib\net45\BenchmarkDotNet.dll</HintPath>
+      <HintPath>..\packages\BenchmarkDotNet.0.9.9.182\lib\net45\BenchmarkDotNet.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="FossilDelta.cs" />
     <Compile Include="Samples.cs" />
+    <Compile Include="DeltaCompressionDotNet.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>

diff --git a/Benchmarks/DeltaCompressionDotNet.cs b/Benchmarks/DeltaCompressionDotNet.cs
@@ -0,0 +1,69 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Jobs;
+using BenchmarkDotNet.Running;
+
+namespace Benchmarks
+{
+	public class DeltaCompressionDotNet
+	{
+		byte[] sample1Delta;
+		byte[] sample2Delta;
+		byte[] sample3Delta;
+		byte[] sample4Delta;
+
+		[Benchmark]
+		public byte[] CreateDelta1()
+		{
+			sample1Delta = Fossil.Delta.Create(Samples.origin1, Samples.target1);
+			return sample1Delta;
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta2()
+		{
+			sample2Delta = Fossil.Delta.Create(Samples.origin2, Samples.target2);
+			return sample2Delta;
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta3()
+		{
+			sample3Delta = Fossil.Delta.Create(Samples.origin3, Samples.target3);
+			return sample3Delta;
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta4()
+		{
+			sample4Delta = Fossil.Delta.Create(Samples.origin4, Samples.target4);
+			return sample4Delta;
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta1()
+		{
+			return Fossil.Delta.Apply(Samples.origin1, sample1Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta2()
+		{
+			return Fossil.Delta.Apply(Samples.origin2, sample2Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta3()
+		{
+			return Fossil.Delta.Apply(Samples.origin3, sample3Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta4()
+		{
+			return Fossil.Delta.Apply(Samples.origin4, sample4Delta);
+		}
+
+	}
+}
+
diff --git a/Benchmarks/FossilDelta.cs b/Benchmarks/FossilDelta.cs
@@ -7,12 +7,72 @@ namespace Benchmarks
 {
 	public class FossilDelta
 	{
+		static byte[] sample1Delta = Fossil.Delta.Create(Samples.origin1, Samples.target1);
+		static byte[] sample2Delta = Fossil.Delta.Create(Samples.origin2, Samples.target2);
+		static byte[] sample3Delta = Fossil.Delta.Create(Samples.origin3, Samples.target3);
+		static byte[] sample4Delta = Fossil.Delta.Create(Samples.origin4, Samples.target4);
+		static byte[] sample5Delta = Fossil.Delta.Create(Samples.origin5, Samples.target5);
 
 		[Benchmark]
-		public byte[] CreateDelta()
+		public byte[] CreateDelta1()
 		{
 			return Fossil.Delta.Create(Samples.origin1, Samples.target1);
 		}
+
+		[Benchmark]
+		public byte[] CreateDelta2()
+		{
+			return Fossil.Delta.Create(Samples.origin2, Samples.target2);
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta3()
+		{
+			return Fossil.Delta.Create(Samples.origin3, Samples.target3);
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta4()
+		{
+			return Fossil.Delta.Create(Samples.origin4, Samples.target4);
+		}
+
+		[Benchmark]
+		public byte[] CreateDelta5()
+		{
+			return Fossil.Delta.Create(Samples.origin5, Samples.target5);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta1()
+		{
+			return Fossil.Delta.Apply(Samples.origin1, sample1Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta2()
+		{
+			return Fossil.Delta.Apply(Samples.origin2, sample2Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta3()
+		{
+			return Fossil.Delta.Apply(Samples.origin3, sample3Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta4()
+		{
+			return Fossil.Delta.Apply(Samples.origin4, sample4Delta);
+		}
+
+		[Benchmark]
+		public byte[] ApplyDelta5()
+		{
+			return Fossil.Delta.Apply(Samples.origin5, sample5Delta);
+		}
+
 	}
 }
 
diff --git a/Benchmarks/Program.cs b/Benchmarks/Program.cs
@@ -7,8 +7,10 @@ class MainClass
 	{
 		public static void Main (string[] args)
 		{
-			var summary = BenchmarkRunner.Run<FossilDelta>();
-			Console.WriteLine (summary.ToString());
+			var switcher = new BenchmarkSwitcher(new[] {
+				typeof(FossilDelta)
+			});
+			switcher.Run(args);
 		}
 	}
 }
diff --git a/Benchmarks/Samples.cs b/Benchmarks/Samples.cs
@@ -1,22 +1,22 @@
-using System;
+using System;
 namespace Benchmarks
 {
 	public class Samples
 	{
-		public static byte[] origin1 = System.IO.File.ReadAllBytes("../../../Tests/data/1/origin");
-		public static byte[] target1 = System.IO.File.ReadAllBytes("../../../Tests/data/1/target");
+		public static byte[] origin1 = System.IO.File.ReadAllBytes("Tests/data/1/origin");
+		public static byte[] target1 = System.IO.File.ReadAllBytes("Tests/data/1/target");
 
-		public static byte[] origin2 = System.IO.File.ReadAllBytes("../../../Tests/data/2/origin");
-		public static byte[] target2 = System.IO.File.ReadAllBytes("../../../Tests/data/2/target");
+		public static byte[] origin2 = System.IO.File.ReadAllBytes("Tests/data/2/origin");
+		public static byte[] target2 = System.IO.File.ReadAllBytes("Tests/data/2/target");
 
-		public static byte[] origin3 = System.IO.File.ReadAllBytes("../../../Tests/data/3/origin");
-		public static byte[] target3 = System.IO.File.ReadAllBytes("../../../Tests/data/3/target");
+		public static byte[] origin3 = System.IO.File.ReadAllBytes("Tests/data/3/origin");
+		public static byte[] target3 = System.IO.File.ReadAllBytes("Tests/data/3/target");
 
-		public static byte[] origin4 = System.IO.File.ReadAllBytes("../../../Tests/data/4/origin");
-		public static byte[] target4 = System.IO.File.ReadAllBytes("../../../Tests/data/4/target");
+		public static byte[] origin4 = System.IO.File.ReadAllBytes("Tests/data/4/origin");
+		public static byte[] target4 = System.IO.File.ReadAllBytes("Tests/data/4/target");
 
-		public static byte[] origin5 = System.IO.File.ReadAllBytes("../../../Tests/data/5/origin");
-		public static byte[] target5 = System.IO.File.ReadAllBytes("../../../Tests/data/5/target");
+		public static byte[] origin5 = System.IO.File.ReadAllBytes("Tests/data/5/origin");
+		public static byte[] target5 = System.IO.File.ReadAllBytes("Tests/data/5/target");
 
 		public Samples()
 		{

diff --git a/Benchmarks/packages.config b/Benchmarks/packages.config
@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="BenchmarkDotNet" version="0.9.9" targetFramework="net45" />
-  <package id="BenchmarkDotNet.Core" version="0.9.9" targetFramework="net45" />
-  <package id="BenchmarkDotNet.Toolchains.Roslyn" version="0.9.9" targetFramework="net45" />
+  <package id="BenchmarkDotNet" version="0.9.9.182" targetFramework="net45" />
+  <package id="BenchmarkDotNet.Core" version="0.9.9.182" targetFramework="net45" />
+  <package id="BenchmarkDotNet.Toolchains.Roslyn" version="0.9.9.182" targetFramework="net45" />
   <package id="DeltaCompressionDotNet" version="1.1.0" targetFramework="net40" />
   <package id="Microsoft.CodeAnalysis.Analyzers" version="1.1.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Common" version="1.3.2" targetFramework="net45" />

diff --git a/Makefile b/Makefile
@@ -7,3 +7,8 @@ publish:
 	rm *.nupkg
 	nuget pack FossilDelta.nuspec
 	nuget push *.nupkg
+
+benchmark:
+	xbuild /p:Configuration=Release FossilDelta.sln
+	mono ./Benchmarks/bin/Release/Benchmarks.exe
+
diff --git a/README.md b/README.md
@@ -51,3 +51,26 @@ Throws an error if it fails to apply the delta
 Returns a size of target for this delta.
 
 Throws an error if it can't read the size from delta.
+
+Benchmark
+---
+
+[See the inputs used for benchmarking](Tests/data). Run the benchmarks
+locally using the `make benchmark` in your commandline.
+
+**Results:**
+
+```
+       Method |           Mean |        StdErr |         StdDev |         Median |
+------------- |--------------- |-------------- |--------------- |--------------- |
+ CreateDelta1 |  5,426.4132 ns |   787.5304 ns |  6,201.0206 ns |  4,286.4851 ns |
+ CreateDelta2 | 21,837.1107 ns | 1,661.4695 ns | 13,900.8509 ns | 25,942.1491 ns |
+ CreateDelta3 | 11,697.2018 ns | 1,213.1634 ns | 12,607.5636 ns |  9,260.4452 ns |
+ CreateDelta4 |    253.4085 ns |    25.1048 ns |    214.4952 ns |    252.6454 ns |
+ CreateDelta5 |    150.4963 ns |    29.0635 ns |    311.6718 ns |      0.0000 ns |
+  ApplyDelta1 |  3,547.0234 ns |   493.4131 ns |  4,357.7065 ns |  3,086.8397 ns |
+  ApplyDelta2 | 20,336.7691 ns | 2,488.0257 ns | 27,254.9560 ns |  9,233.5811 ns |
+  ApplyDelta3 |  1,441.5354 ns |   209.2071 ns |  1,995.7090 ns |    855.9650 ns |
+  ApplyDelta4 |    252.5743 ns |    29.7323 ns |    234.1123 ns |    236.0480 ns |
+  ApplyDelta5 |     68.5550 ns |     9.2923 ns |     92.4574 ns |     39.0918 ns |
+```