improve benchmarking

jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java

@@ -4,13 +4,15 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import util.BenchmarkWriter;
 import util.Solutions;
 import algorithms.VehicleRoutingAlgorithms;
 import basics.VehicleRoutingAlgorithm;
@@ -20,11 +22,11 @@ import basics.algo.VehicleRoutingAlgorithmListeners.Priority;
 
 public class ConcurrentBenchmarker {
 	
-	static class Problem {
+	public static class BenchmarkInstance {
 		public final String name;
 		public final VehicleRoutingProblem vrp;
 		public final Double bestKnown;
-		public Problem(String name, VehicleRoutingProblem vrp, Double bestKnown) {
+		public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnown) {
 			super();
 			this.name = name;
 			this.vrp = vrp;
@@ -32,16 +34,16 @@ public class ConcurrentBenchmarker {
 		}
 	}
 	
-	static class Result {
+	public static class BenchmarkResult {
 		public final double result;
 		public final double time;
-		public final Problem problem;
+		public final BenchmarkInstance instance;
 		public Double delta = null;
-		public Result(Problem p, double result, double time) {
+		public BenchmarkResult(BenchmarkInstance p, double result, double time) {
 			super();
 			this.result = result;
 			this.time = time;
-			this.problem = p;
+			this.instance = p;
 		}
 		void setBestKnownDelta(double delta){
 			this.delta = delta;
@@ -50,39 +52,62 @@ public class ConcurrentBenchmarker {
 	
 	private String algorithmConfig;
 	
-	private List<Problem> problems = new ArrayList<Problem>();
+	private List<BenchmarkInstance> problems = new ArrayList<BenchmarkInstance>();
+
+	private int runs = 1;
+	
+	private Collection<BenchmarkWriter> writers = new ArrayList<BenchmarkWriter>();
+	
+	private Collection<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
 	
 	public ConcurrentBenchmarker(String algorithmConfig) {
 		super();
 		this.algorithmConfig = algorithmConfig;
+		Logger.getRootLogger().setLevel(Level.ERROR);
+	}
+	
+	public void addBenchmarkWriter(BenchmarkWriter writer){
+		writers.add(writer);
 	}
 
 	public void addProblem(String name, VehicleRoutingProblem problem){
-		problems.add(new Problem(name,problem,null));
+		problems.add(new BenchmarkInstance(name,problem,null));
 	}
 	
 	public void addProblem(String name, VehicleRoutingProblem problem, double bestKnown){
-		problems.add(new Problem(name,problem,bestKnown));
+		problems.add(new BenchmarkInstance(name,problem,bestKnown));
+	}
+	
+	public void setNuOfRuns(int runs){
+		this.runs = runs;
 	}
 	
 	public void run(){
+		System.out.println("start benchmarking [nuOfInstances=" + problems.size() + "]");
+		double startTime = System.currentTimeMillis();
 		ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()+1);
-		List<Future<Result>> futures = new ArrayList<Future<Result>>();
+		List<Future<BenchmarkResult>> futures = new ArrayList<Future<BenchmarkResult>>();
-		List<Result> results = new ArrayList<ConcurrentBenchmarker.Result>();
+//		List<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
-		for(final Problem p : problems){
+		for(final BenchmarkInstance p : problems){
-			Future<Result> futureResult = executor.submit(new Callable<Result>(){
+			for(int run=0;run<runs;run++){
+				Future<BenchmarkResult> futureResult = executor.submit(new Callable<BenchmarkResult>(){
 
-				@Override
+					@Override
-				public Result call() throws Exception {
+					public BenchmarkResult call() throws Exception {
-					return runAlgoAndGetResult(p);
+						return runAlgoAndGetResult(p);
-				}
+					}
 
-			});
+				});
-			futures.add(futureResult);
+				futures.add(futureResult);
+			}
 		}
 		try {
-			for(Future<Result> f : futures){
+			int count = 1;
+			for(Future<BenchmarkResult> f : futures){
+				BenchmarkResult r = f.get();
+				print(r,count);
 				results.add(f.get());
+				count++;
 			}
 		} catch (InterruptedException e) {
 			// TODO Auto-generated catch block
@@ -93,33 +118,43 @@ public class ConcurrentBenchmarker {
 		}
 		executor.shutdown();
 		print(results);
+		System.out.println("done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]");
 	}
 
-	private Result runAlgoAndGetResult(Problem p) {
+	private BenchmarkResult runAlgoAndGetResult(BenchmarkInstance p) {
 		VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
 		StopWatch stopwatch = new StopWatch();
 		vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
 		Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
 		VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
-		Result result = new Result(p,best.getCost(),stopwatch.getCompTimeInSeconds());
+		BenchmarkResult result = new BenchmarkResult(p,best.getCost(),stopwatch.getCompTimeInSeconds());
 		if(p.bestKnown != null) result.setBestKnownDelta((best.getCost()/p.bestKnown-1));
 		return result;
 	}
 
-	private void print(List<Result> results) {
+	private void print(Collection<BenchmarkResult> results) {
-		System.out.println("instance,time [in sec],result,delta [in percent to bestKnown]");
 		double sumTime=0.0;
 		double sumResult=0.0;
-		for(Result r : results){
+		for(BenchmarkResult r : results){
 			sumTime+=r.time;
 			sumResult+=r.result;
-			System.out.println("[instance="+r.problem.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+"][delta="+round(r.delta,3)+"]");
+//			print(r);
 		}
 		System.out.println("[avgTime="+round(sumTime/(double)results.size(),2)+"][avgResult="+round(sumResult/(double)results.size(),2)+"]");
+		for(BenchmarkWriter writer : writers){
+			writer.write(results);
+		}
 	}
 
-	private double round(Double delta, int i) {
+	private void print(BenchmarkResult r, int count) {
-		long roundedVal = Math.round(delta*Math.pow(10, i));
+		System.out.println("("+count+"/"+problems.size() +")"+ "\t[instance="+r.instance.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+"][delta="+round(r.delta,3)+"]");
+//		for(BenchmarkWriter writer : writers){
+//			writer.write(r);
+//		}
+	}
+
+	private double round(Double value, int i) {
+		long roundedVal = Math.round(value*Math.pow(10, i));
 		return (double)roundedVal/(double)(Math.pow(10, i));
 	}
 

jsprit-analysis/src/main/java/util/BenchmarkWriter.java

@@ -0,0 +1,9 @@
+package util;
+
+import java.util.Collection;
+
+import analysis.ConcurrentBenchmarker.BenchmarkResult;
+
+public interface BenchmarkWriter {
+	public void write(Collection<BenchmarkResult> results);
+}

jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java

@@ -0,0 +1,98 @@
+package util;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Collection;
+
+import analysis.ConcurrentBenchmarker.BenchmarkResult;
+
+public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
+
+	private String filename;
+	
+	public HtmlBenchmarkTableWriter(String filename) {
+		this.filename = filename;
+	}
+
+	@Override
+	public void write(Collection<BenchmarkResult> results) {
+		
+		try {
+			BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filename)));
+			writer.write(openTable() + newline());
+			//table head
+			writer.write(openRow() + newline());
+			writer.write(head("instance") + newline());
+			writer.write(head("compTime [sec]") + newline());
+			writer.write(head("result") + newline());
+			writer.write(head("&Delta; bestKnown [in %]") + newline());
+			writer.write(closeRow() + newline());
+			//data
+			double sum_time = 0.0;
+			double sum_result = 0.0;
+			double sum_delta = 0.0;
+			for(BenchmarkResult result : results){
+				sum_time+=result.time;
+				sum_result+=result.result;
+				sum_delta+=result.delta;
+				writer.write(openRow() + newline());
+				writer.write(date(result.instance.name) + newline());
+				writer.write(date(Double.valueOf(round(result.time,2)).toString()) + newline());
+				writer.write(date(Double.valueOf(round(result.result,2)).toString()) + newline());
+				writer.write(date(Double.valueOf(round(result.delta*100.0,2)).toString()) + newline());
+				writer.write(closeRow() + newline());
+			}
+			writer.write(openRow() + newline());
+			writer.write(date("avg") + newline());
+			writer.write(date(Double.valueOf(round(sum_time/(double)results.size(),2)).toString()) + newline());
+			writer.write(date(Double.valueOf(round(sum_result/(double)results.size(),2)).toString()) + newline());
+			writer.write(date(Double.valueOf(round(sum_delta/(double)results.size()*100.0,2)).toString()) + newline());
+			writer.write(closeRow() + newline());
+			
+			writer.write(closeTable() + newline());
+			writer.close();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+	}
+	
+	private double round(Double value, int i) {
+		long roundedVal = Math.round(value*Math.pow(10, i));
+		return (double)roundedVal/(double)(Math.pow(10, i));
+	}
+
+	private String head(String head) {
+		return "<th>"+head+"</th>";
+	}
+
+	private String closeTable() {
+		return "</table>";
+	}
+
+	private String openTable() {
+		return "<table>";
+	}
+
+	private String closeRow() {
+		return "</tr>";
+	}
+
+	private String date(String date) {
+		return "<td>"+date+"</td>";
+	}
+
+	private String newline() {
+		return "\n";
+	}
+
+	private String openRow() {
+		return "<tr>";
+	}
+
+	
+
+}