refine and improve benchmarking

jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java

@@ -30,9 +30,12 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
+import util.BenchmarkInstance;
+import util.BenchmarkResult;
 import util.BenchmarkWriter;
 import util.Solutions;
 import algorithms.VehicleRoutingAlgorithms;
@@ -43,45 +46,32 @@ import basics.algo.VehicleRoutingAlgorithmListeners.Priority;
 
 public class ConcurrentBenchmarker {
 	
-	public static class BenchmarkInstance {
+	public static interface Cost {
-		public final String name;
+		public double getCost(VehicleRoutingProblemSolution sol);
-		public final VehicleRoutingProblem vrp;
-		public final Double bestKnown;
-		public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnown) {
-			super();
-			this.name = name;
-			this.vrp = vrp;
-			this.bestKnown = bestKnown;
-		}
 	}
 	
-	public static class BenchmarkResult {
+	
-		public final double result;
-		public final double time;
-		public final int nuOfVehicles;
-		public final BenchmarkInstance instance;
-		public Double delta = null;
-		public BenchmarkResult(BenchmarkInstance p, double result, double time, int nuOfVehicles) {
-			super();
-			this.result = result;
-			this.time = time;
-			this.instance = p;
-			this.nuOfVehicles = nuOfVehicles;
-		}
-		void setBestKnownDelta(double delta){
-			this.delta = delta;
-		}
-	}
 	
 	private String algorithmConfig;
 	
-	private List<BenchmarkInstance> problems = new ArrayList<BenchmarkInstance>();
+	private List<BenchmarkInstance> benchmarkInstances = new ArrayList<BenchmarkInstance>();
 
 	private int runs = 1;
 	
 	private Collection<BenchmarkWriter> writers = new ArrayList<BenchmarkWriter>();
 	
-	private Collection<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
+	private Collection<BenchmarkResult> results = new ArrayList<BenchmarkResult>();
+	
+	private Cost cost = new Cost(){
+
+		@Override
+		public double getCost(VehicleRoutingProblemSolution sol) {
+			return sol.getCost();
+		}
+		
+	};
+	
+	public void setCost(Cost cost){ this.cost = cost; }
 	
 	public ConcurrentBenchmarker(String algorithmConfig) {
 		super();
@@ -93,12 +83,20 @@ public class ConcurrentBenchmarker {
 		writers.add(writer);
 	}
 
-	public void addProblem(String name, VehicleRoutingProblem problem){
+	public void addInstance(String name, VehicleRoutingProblem problem){
-		problems.add(new BenchmarkInstance(name,problem,null));
+		benchmarkInstances.add(new BenchmarkInstance(name,problem,null,null));
 	}
 	
-	public void addProblem(String name, VehicleRoutingProblem problem, double bestKnown){
+	public void addInstane(BenchmarkInstance instance){
-		problems.add(new BenchmarkInstance(name,problem,bestKnown));
+		benchmarkInstances.add(instance);
+	}
+	
+	public void addAllInstances(Collection<BenchmarkInstance> instances){
+		benchmarkInstances.addAll(instances);
+	}
+	
+	public void addInstance(String name, VehicleRoutingProblem problem, Double bestKnownResult, Double bestKnownVehicles){
+		benchmarkInstances.add(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
 	}
 	
 	public void setNuOfRuns(int runs){
@@ -106,13 +104,12 @@ public class ConcurrentBenchmarker {
 	}
 	
 	public void run(){
-		System.out.println("start benchmarking [nuOfInstances=" + problems.size() + "]");
+		System.out.println("start benchmarking [nuOfInstances=" + benchmarkInstances.size() + "][runsPerInstance=" + runs + "]");
 		double startTime = System.currentTimeMillis();
 		ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()+1);
 		List<Future<BenchmarkResult>> futures = new ArrayList<Future<BenchmarkResult>>();
-//		List<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
+		for(final BenchmarkInstance p : benchmarkInstances){
-		for(final BenchmarkInstance p : problems){
+			
-			for(int run=0;run<runs;run++){
 				Future<BenchmarkResult> futureResult = executor.submit(new Callable<BenchmarkResult>(){
 
 					@Override
@@ -122,7 +119,7 @@ public class ConcurrentBenchmarker {
 
 				});
 				futures.add(futureResult);
-			}
+			
 		}
 		try {
 			int count = 1;
@@ -145,22 +142,30 @@ public class ConcurrentBenchmarker {
 	}
 
 	private BenchmarkResult runAlgoAndGetResult(BenchmarkInstance p) {
-		VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
+		double[] vehicles = new double[runs];
-		StopWatch stopwatch = new StopWatch();
+		double[] results = new double[runs];
-		vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
+		double[] times = new double[runs];
-		Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
+		
-		VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
+		for(int run=0;run<runs;run++){
-		BenchmarkResult result = new BenchmarkResult(p,best.getCost(),stopwatch.getCompTimeInSeconds(), best.getRoutes().size());
+			VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
-		if(p.bestKnown != null) result.setBestKnownDelta((best.getCost()/p.bestKnown-1));
+			StopWatch stopwatch = new StopWatch();
-		return result;
+			vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
+			Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
+			VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
+			vehicles[run] = best.getRoutes().size();
+			results[run] = cost.getCost(best);
+			times[run] = stopwatch.getCompTimeInSeconds();
+		}
+		
+		return new BenchmarkResult(p, runs, results, times, vehicles);
 	}
 
 	private void print(Collection<BenchmarkResult> results) {
 		double sumTime=0.0;
 		double sumResult=0.0;
 		for(BenchmarkResult r : results){
-			sumTime+=r.time;
+			sumTime+=r.getTimesStats().getMean();
-			sumResult+=r.result;
+			sumResult+=r.getResultStats().getMean();
 //			print(r);
 		}
 		System.out.println("[avgTime="+round(sumTime/(double)results.size(),2)+"][avgResult="+round(sumResult/(double)results.size(),2)+"]");
@@ -170,9 +175,27 @@ public class ConcurrentBenchmarker {
 	}
 
 	private void print(BenchmarkResult r, int count) {
-		System.out.println("("+count+"/"+problems.size() +")"+ "\t[instance="+r.instance.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+
+		Double avgDelta = null;
-				"][delta="+round(r.delta,3)+"][nuOfVehicles="+r.nuOfVehicles+"]");
+		Double bestDelta = null;
+		Double worstDelta = null;
+		if(r.instance.bestKnownResult != null){
+			avgDelta = (r.getResultStats().getMean() / r.instance.bestKnownResult - 1) * 100;
+			bestDelta = (r.getResultStats().getMin() / r.instance.bestKnownResult - 1) * 100;
+			worstDelta = (r.getResultStats().getMax() / r.instance.bestKnownResult - 1) * 100;
+		}
+		System.out.println("("+count+"/"+benchmarkInstances.size() +")"+ "\t[instance="+r.instance.name+
+				"][avgTime="+round(r.getTimesStats().getMean(),2)+"]" +
+				"[Result=" + getString(r.getResultStats()) + "]" +
+				"[Vehicles=" + getString(r.getVehicleStats()) + "]" +
+				"[Delta[%]=" + getString(bestDelta,avgDelta,worstDelta) + "]");
+	}
 	
+	private String getString(Double bestDelta, Double avgDelta,Double worstDelta) {
+		return "[best="+round(bestDelta,2)+"][avg="+round(avgDelta,2)+"][worst="+round(worstDelta,2)+"]";
+	}
+
+	private String getString(DescriptiveStatistics stats){
+		return "[best="+round(stats.getMin(),2)+"][avg="+round(stats.getMean(),2)+"][worst="+round(stats.getMax(),2)+"][stdDev=" + round(stats.getStandardDeviation(),2)+"]";
 	}
 
 	private Double round(Double value, int i) {

jsprit-analysis/src/main/java/util/BenchmarkWriter.java

@@ -23,8 +23,6 @@ package util;
 
 import java.util.Collection;
 
-import analysis.ConcurrentBenchmarker.BenchmarkResult;
-
 public interface BenchmarkWriter {
 	public void write(Collection<BenchmarkResult> results);
 }

jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java

@@ -27,7 +27,7 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Collection;
 
-import analysis.ConcurrentBenchmarker.BenchmarkResult;
+import org.jfree.chart.renderer.xy.DeviationRenderer;
 
 public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
 
@@ -45,34 +45,170 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
 			writer.write(openTable() + newline());
 			//table head
 			writer.write(openRow() + newline());
-			writer.write(head("instance") + newline());
+			writer.write(head("inst") + newline());
-			writer.write(head("compTime [sec]") + newline());
+			writer.write(head("runs") + newline());
-			writer.write(head("result") + newline());
+			writer.write(head("Ø time [sec]") + newline());
-			writer.write(head("Δ bestKnown [in %]") + newline());
+			writer.write(head("results",4));
+			writer.write(head("vehicles",4));
+			writer.write(head("res*") + newline());
+			writer.write(head("veh*") + newline());
 			writer.write(closeRow() + newline());
+			
+			writer.write(openRow() + newline());
+			writer.write(head("") + newline());
+			writer.write(head("") + newline());
+			writer.write(head("") + newline());
+			writer.write(head("best") + newline());
+			writer.write(head("avg") + newline());
+			writer.write(head("worst") + newline());
+			writer.write(head("stdev") + newline());
+			writer.write(head("best") + newline());
+			writer.write(head("avg") + newline());
+			writer.write(head("worst") + newline());
+			writer.write(head("stdev") + newline());
+			writer.write(head("") + newline());
+			writer.write(head("") + newline());
+			writer.write(closeRow() + newline());
+			
 			//data
-			double sum_time = 0.0;
+			double sum_avg_time = 0.0;
-			double sum_result = 0.0;
+			double sum_best_result = 0.0;
-			double sum_delta = 0.0;
+			double sum_avg_result = 0.0;
+			double sum_worst_result = 0.0;
+			double sum_dev_result = 0.0;
+			
+			double sum_best_veh = 0.0;
+			double sum_avg_veh = 0.0;
+			double sum_worst_veh = 0.0;
+			double sum_dev_veh = 0.0;
+			
+			Integer runs = null; 
+			Double sum_res_star=null;
+			Double sum_veh_star=null;
+			
 			for(BenchmarkResult result : results){
-				sum_time+=result.time;
+				if(runs==null) runs=result.runs;
-				sum_result+=result.result;
-				sum_delta+=result.delta;
 				writer.write(openRow() + newline());
 				writer.write(date(result.instance.name) + newline());
-				writer.write(date(Double.valueOf(round(result.time,2)).toString()) + newline());
+				writer.write(date(Integer.valueOf(result.runs).toString()) + newline());
-				writer.write(date(Double.valueOf(round(result.result,2)).toString()) + newline());
+				
-				writer.write(date(Double.valueOf(round(result.delta*100.0,2)).toString()) + newline());
+				Double avg_time = round(result.getTimesStats().getMean(),2); 
+				writer.write(date(Double.valueOf(avg_time).toString()) + newline());
+				//bestRes
+				Double best_result = round(result.getResultStats().getMin(),2);
+				writer.write(date(Double.valueOf(best_result).toString()) + newline());
+				//avgRes
+				Double avg_result = round(result.getResultStats().getMean(),2);
+				writer.write(date(Double.valueOf(avg_result).toString()) + newline());
+				//worstRes
+				Double worst_result = round(result.getResultStats().getMax(),2);
+				writer.write(date(Double.valueOf(worst_result).toString()) + newline());
+				//stdevRes
+				Double std_result = round(result.getResultStats().getStandardDeviation(),2);
+				writer.write(date(Double.valueOf(std_result).toString()) + newline());
+				//bestVeh
+				Double best_vehicle = round(result.getVehicleStats().getMin(),2);
+				writer.write(date(Double.valueOf(best_vehicle).toString()) + newline());
+				//avgVeh
+				Double avg_vehicle = round(result.getVehicleStats().getMean(),2);
+				writer.write(date(Double.valueOf(avg_vehicle).toString()) + newline());
+				//worstVeh
+				Double worst_vehicle = round(result.getVehicleStats().getMax(),2);
+				writer.write(date(Double.valueOf(worst_vehicle).toString()) + newline());
+				//stdevVeh
+				Double std_vehicle = round(result.getVehicleStats().getStandardDeviation(),2);
+				writer.write(date(Double.valueOf(std_vehicle).toString()) + newline());
+				//bestKnownRes
+				writer.write(date("" + result.instance.bestKnownResult + newline()));
+				//bestKnownVeh
+				writer.write(date("" + result.instance.bestKnownVehicles + newline()));
 				writer.write(closeRow() + newline());
+				
+				sum_avg_time+=avg_time;
+				sum_best_result+=best_result;
+				sum_avg_result+=avg_result;
+				sum_worst_result+=worst_result;
+				sum_dev_result+=std_result;
+				
+				sum_best_veh+=best_vehicle;
+				sum_avg_veh+=avg_vehicle;
+				sum_worst_veh+=worst_vehicle;
+				sum_dev_veh+=std_vehicle;
+				
+				if(result.instance.bestKnownResult != null){
+					if(sum_res_star==null) sum_res_star=result.instance.bestKnownResult;
+					else sum_res_star+=result.instance.bestKnownResult;
+				}
+				if(result.instance.bestKnownVehicles != null){
+					if(sum_veh_star==null) sum_veh_star=result.instance.bestKnownVehicles;
+					else sum_veh_star+=result.instance.bestKnownVehicles;
+				}
+				
 			}
 			writer.write(openRow() + newline());
-			writer.write(date("avg") + newline());
+			writer.write(date("Ø") + newline());
-			writer.write(date(Double.valueOf(round(sum_time/(double)results.size(),2)).toString()) + newline());
+			writer.write(date(""+runs) + newline());
-			writer.write(date(Double.valueOf(round(sum_result/(double)results.size(),2)).toString()) + newline());
+			 
-			writer.write(date(Double.valueOf(round(sum_delta/(double)results.size()*100.0,2)).toString()) + newline());
+			Double average_time = round(sum_avg_time/(double)results.size(),2);
+			writer.write(date(Double.valueOf(average_time).toString()) + newline());
+			//bestRes
+			writer.write(date(Double.valueOf(round(sum_best_result/(double)results.size(),2)).toString()) + newline());
+			//avgRes
+			Double average_result = round(sum_avg_result/(double)results.size(),2);
+			writer.write(date(Double.valueOf(average_result).toString()) + newline());
+			//worstRes
+			writer.write(date(Double.valueOf(round(sum_worst_result/(double)results.size(),2)).toString()) + newline());
+			//stdevRes
+			writer.write(date(Double.valueOf(round(sum_dev_result/(double)results.size(),2)).toString()) + newline());
+			//bestVeh
+			writer.write(date(Double.valueOf(round(sum_best_veh/(double)results.size(),2)).toString()) + newline());
+			//avgVeh
+			Double average_vehicles = round(sum_avg_veh/(double)results.size(),2);
+			writer.write(date(Double.valueOf(average_vehicles).toString()) + newline());
+			//worstVeh
+			writer.write(date(Double.valueOf(round(sum_worst_veh/(double)results.size(),2)).toString()) + newline());
+			//stdevVeh
+			writer.write(date(Double.valueOf(round(sum_dev_veh/(double)results.size(),2)).toString()) + newline());
+			//bestKnownRes
+			Double delta_res = null;
+			if(sum_res_star != null){
+				writer.write(date(Double.valueOf(round(sum_res_star.doubleValue()/(double)results.size(),2)).toString()) + newline());
+				delta_res = (sum_avg_result/sum_res_star - 1)*100;
+			}
+			else writer.write(date("null") + newline());
+			//bestKnownVeh
+			Double delta_veh = null;
+			if(sum_veh_star != null){
+				writer.write(date(Double.valueOf(round(sum_veh_star.doubleValue()/(double)results.size(),2)).toString()) + newline());
+				delta_veh = (sum_avg_veh - sum_veh_star)/(double)results.size();
+			}
+			else writer.write(date("null") + newline());
 			writer.write(closeRow() + newline());
 			
 			writer.write(closeTable() + newline());
+			
+			writer.write("avg. percentage deviation to best-known result: " + round(delta_res,2) + newline() + newline());
+			writer.write("avg. absolute deviation to best-known vehicles: " + round(delta_veh,2) + newline());
+			
+			writer.write(openTable() + newline());
+			writer.write(openRow() + newline());
+			writer.write(date("") + newline());
+			writer.write(date("") + newline());
+			writer.write(date("") + newline());
+			writer.write(date("") + newline());
+			writer.write(date(Double.valueOf(average_time).toString(),"align=\"right\"") + newline());
+			writer.write(date(Double.valueOf(average_result).toString(),"align=\"right\"") + newline());
+			writer.write(date(Double.valueOf(average_vehicles).toString(),"align=\"right\"") + newline());
+			if(delta_res != null){
+				writer.write(date(Double.valueOf(round(delta_res,2)).toString(),"align=\"right\"") + newline());
+			}else writer.write(date("n.a.") + newline()); 
+			if(delta_veh != null){
+				writer.write(date(Double.valueOf(round(delta_veh,2)).toString(),"align=\"right\"") + newline());
+			}else writer.write(date("n.a.") + newline());
+			writer.write(closeRow() + newline());
+			writer.write(closeTable() + newline());
+			
+			
 			writer.close();
 		} catch (IOException e) {
 			// TODO Auto-generated catch block
@@ -81,7 +217,12 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
 		
 	}
 	
-	private double round(Double value, int i) {
+	private String head(String string, int i) {
+		return "<th colspan=\""+i+"\">"+string+"</th>";
+	}
+
+	private Double round(Double value, int i) {
+		if(value==null) return null;
 		long roundedVal = Math.round(value*Math.pow(10, i));
 		return (double)roundedVal/(double)(Math.pow(10, i));
 	}
@@ -106,6 +247,10 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
 		return "<td>"+date+"</td>";
 	}
 	
+	private String date(String date, String metaData) {
+	return "<td " + metaData + ">"+date+"</td>";
+	}
+
 	private String newline() {
 		return "\n";
 	}

jsprit-core/src/main/java/util/BenchmarkInstance.java

@@ -0,0 +1,17 @@
+package util;
+
+import basics.VehicleRoutingProblem;
+
+public class BenchmarkInstance {
+	public final String name;
+	public final VehicleRoutingProblem vrp;
+	public final Double bestKnownResult;
+	public Double bestKnownVehicles;
+	public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnownResult, Double bestKnowVehicles) {
+		super();
+		this.name = name;
+		this.vrp = vrp;
+		this.bestKnownResult = bestKnownResult;
+		this.bestKnownVehicles = bestKnowVehicles;
+	}
+}

jsprit-core/src/main/java/util/BenchmarkResult.java

@@ -0,0 +1,56 @@
+package util;
+
+import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
+
+
+
+public class BenchmarkResult {
+	private double[] results;
+	private double[] vehicles;
+	private double[] times;
+	
+	private DescriptiveStatistics statsResults;
+	private DescriptiveStatistics statsVehicles;
+	private DescriptiveStatistics statsTimes;
+	
+	public final BenchmarkInstance instance;
+	
+	public final int runs;
+	
+	public BenchmarkResult(BenchmarkInstance instance, int runs, double[] results, double[] compTimes, double[] vehicles) {
+		super();
+		this.results = results;
+		this.runs = runs;
+		this.times = compTimes;
+		this.instance = instance;
+		this.vehicles = vehicles;
+		this.statsResults = new DescriptiveStatistics(results);
+		this.statsTimes = new DescriptiveStatistics(times);
+		this.statsVehicles = new DescriptiveStatistics(vehicles);
+	}
+	
+	public double[] getResults(){
+		return results;
+	}
+	
+	public double[] getVehicles(){
+		return vehicles;
+	}
+	
+	public double[] getCompTimes(){
+		return times;
+	}
+	
+	public DescriptiveStatistics getResultStats(){
+		return statsResults;
+	}
+	
+	public DescriptiveStatistics getVehicleStats(){
+		return statsVehicles;
+	}
+	
+	public DescriptiveStatistics getTimesStats(){
+		return statsTimes;
+	}
+	
+}

jsprit-instances/src/main/java/util/Instances.java

@@ -0,0 +1,226 @@
+package util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import readers.ChristofidesReader;
+import readers.CordeauReader;
+import readers.SolomonReader;
+import basics.VehicleRoutingProblem;
+
+public class Instances {
+
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Cordeau's p instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 12 p-instances are located with their original name, i.e. p01,p02,...,p12.
+	 * <p>It also assumes that solution files are also located in inputFolder ending with .res
+	 * 
+	 * @param inputFolder where cordeau's p instances are located. It must end without '/' such as instances/cordeau. 
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllCordeauP(String inputFolder){
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<12;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/p"+  getInstanceNu(i+1);
+			new CordeauReader(builder,true).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("p" + getInstanceNu(i+1), p, getBestKnown(file), null));
+		}
+		return instances;
+	}
+	
+	private static double getBestKnown(String file) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(new File(file+".res")));
+			String first = reader.readLine();
+			Double result = Double.valueOf(first);
+			reader.close();
+			return result;
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		return 0;
+	}
+	
+	private static String getInstanceNu(int i) {
+		if(i<10) return "0"+i;
+		return ""+i;
+	}
+
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Cordeau's pr instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 10 p-instances are located with their original name, i.e. pr01,pr02,...,pr10.
+	 * <p>It also assumes that solution files are also located in inputFolder ending with .res
+	 * @param inputFolder TODO
+	 * @param inputFolder where cordeau's pr instances are located. It must end without '/' such as instances/cordeau.
+	 * 
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllCordeauPR(String inputFolder){
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<10;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/pr"+  getInstanceNu(i+1);
+			new CordeauReader(builder,true).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("pr" + getInstanceNu(i+1), p, getBestKnown(file),null));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Christofides vrpnc instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 14 vrpnc-instances are located with their original name, i.e. vrpnc1,vrpnc2,...,vrpnc14.
+	 * 
+	 * @param inputFolder where christofides vrpnc instances are located. It must end without '/' such as instances/christofides.
+	 * 
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllChristofides(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(524.61,835.26,826.14,1028.42,1291.29,555.43,909.68,865.49,1162.55,1395.85,1042.11,819.56,1541.14,866.37);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<14;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/vrpnc"+ (i+1) + ".txt";
+			new ChristofidesReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("vrpnc" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), null));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 9 C1-instances are located with their original name, i.e. C101.txt,C102.txt,...,C109.txt.
+	 * @param inputFolder where solomon C1 instances are located. It must end without '/' such as instances/solomon.
+	 * 
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonC1(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(828.94,828.94,828.06,824.78,828.94,828.94,828.94,828.94,828.94);
+		List<Double> bestKnowVehicles = Arrays.asList(10.0,10.0,10.0,10.0,10.0,10.0,10.0,10.0,10.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<9;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/C1"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("C1" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 8 C2-instances are located with their original name, i.e. C201.txt,C202.txt,...,C208.txt.
+	 * @param inputFolder where solomon C2 instances are located. It must end without '/' such as instances/solomon.
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonC2(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(591.56,591.56,591.17,590.60,588.88,588.49,588.29,588.32);
+		List<Double> bestKnowVehicles = Arrays.asList(3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<8;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/C2"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("C2" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 12 R1-instances are located with their original name, i.e. R101.txt,R102.txt,...,R112.txt.
+	 * @param inputFolder where solomon R1 instances are located. It must end without '/' such as instances/solomon.
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonR1(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(1650.80,1486.12,1292.68,1007.31,1377.11,1252.03,1104.66,960.88,1194.73,1118.84,1096.72,982.14);
+		List<Double> bestKnowVehicles = Arrays.asList(19.0,17.0,13.0,9.0,14.0,12.0,10.0,9.0,11.0,10.0,10.0,9.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<12;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/R1"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("R1" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 11 R1-instances are located with their original name, i.e. R201.txt,R202.txt,...,R111.txt.
+	 * @param inputFolder TODO
+	 * @param inputFolder where solomon R2 instances are located. It must end without '/' such as instances/solomon.
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonR2(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(1252.37,1191.70,939.50,825.52,994.42,906.14,890.61,726.82,909.16,939.37,885.71);
+		List<Double> bestKnowVehicles = Arrays.asList(4.0,3.0,3.0,2.0,3.0,3.0,2.0,2.0,3.0,3.0,2.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<11;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/R2"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("R2" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 8 RC1-instances are located with their original name, i.e. RC101.txt,RC102.txt,...,RC108.txt.
+	 * @param inputFolder where solomon RC1 instances are located. It must end without '/' such as instances/solomon.
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonRC1(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(1696.94,1554.75,1261.67,1135.48,1629.44,1424.73,1230.48,1139.82);
+		List<Double> bestKnowVehicles = Arrays.asList(14.0,12.0,11.0,10.0,13.0,11.0,11.0,10.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<8;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/RC1"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("RC1" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+	
+	/**
+	 * Returns a collection of {@link BenchmarkInstance} which are Solomon instances.
+	 * <p>Note that this assumes that within the folder 'inputFolder' 8 RC2-instances are located with their original name, i.e. RC201.txt,RC202.txt,...,RC208.txt.
+	 * @param inputFolder TODO
+	 * @param inputFolder where solomon RC2 instances are located. It must end without '/' such as instances/solomon.
+	 * @return a collection of {@link BenchmarkInstance}
+	 */
+	public static Collection<BenchmarkInstance> getAllSolomonRC2(String inputFolder){
+		List<Double> bestKnown = Arrays.asList(1406.94,1365.65,1049.62,798.46,1297.65,1146.32,1061.14,828.14);
+		List<Double> bestKnowVehicles = Arrays.asList(4.0,3.0,3.0,3.0,4.0,3.0,3.0,3.0);
+		Collection<BenchmarkInstance> instances = new ArrayList<BenchmarkInstance>();
+		for(int i=0;i<8;i++){
+			VehicleRoutingProblem.Builder builder = VehicleRoutingProblem.Builder.newInstance();
+			String file = inputFolder + "/RC2"+ getInstanceNu(i+1) + ".txt";
+			new SolomonReader(builder).read(file);
+			VehicleRoutingProblem p = builder.build();
+			instances.add(new BenchmarkInstance("RC2" + getInstanceNu(i+1), p, bestKnown.get(i).doubleValue(), bestKnowVehicles.get(i).doubleValue()));
+		}
+		return instances;
+	}
+}