refined, simplified, accelerated and documented

analysis.toolbox.ComputationalLaboratory.java

jsprit-analysis/src/main/java/jsprit/analysis/toolbox/ComputationalLaboratory.java

@@ -2,83 +2,198 @@ package jsprit.analysis.toolbox;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
-import java.util.concurrent.Callable;
+import java.util.Set;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 import jsprit.core.algorithm.VehicleRoutingAlgorithm;
 import jsprit.core.algorithm.VehicleRoutingAlgorithmFactory;
-import jsprit.core.algorithm.listener.VehicleRoutingAlgorithmListeners.Priority;
 import jsprit.core.problem.VehicleRoutingProblem;
 import jsprit.core.problem.solution.VehicleRoutingProblemSolution;
 import jsprit.core.util.BenchmarkInstance;
-import jsprit.core.util.Solutions;
 
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
 public class ComputationalLaboratory {
 	
-	public abstract static interface Result {
+	/**
+	 * Listener-interface to listen to calculation.
+	 * 
+	 * <p>Note that calculations are run concurrently, i.e. a unique task that is distributed to an available thread is
+	 * {algorithm, instance, run}.
+	 * 
+	 * @author schroeder
+	 *
+	 */
+	public static interface CalculationListener {
 		
-		public abstract String getIndicatorName();
+		public void calculationStarts(final BenchmarkInstance p, final String algorithmName, final VehicleRoutingAlgorithm algorithm, final int run);
 		
-		public abstract double getResult();
+		public void calculationEnds(final BenchmarkInstance p, final String algorithmName, final VehicleRoutingAlgorithm algorithm, final int run, final Collection<VehicleRoutingProblemSolution> solutions);
-	}
-	
-	public abstract static interface Indicator {
-				
-		public abstract void calculationStarts();
-		
-		public abstract void runStarts(VehicleRoutingAlgorithm vra);
-		
-		public abstract void runEnds(VehicleRoutingProblemSolution vrs);
-
-		public abstract Collection<Result> getResults();
 		
 	}
 	
-	public static class ResultContainer {
+	/**
+	 * Collects whatever indicators you require by algorithmName, instanceName, run and indicator.
+	 * 
+	 * @author schroeder
+	 *
+	 */
+	public static class DataCollector {
 		
-		private List<Result> results = new ArrayList<Result>();
+		public static class Key {
+			private String instanceName;
+			private String algorithmName;
+			private int run;
+			private String indicatorName;
 			
-		private String instanceName;
+			public Key(String instanceName, String algorithmName, int run,String indicatorName) {
+				super();
+				this.instanceName = instanceName;
+				this.algorithmName = algorithmName;
+				this.run = run;
+				this.indicatorName = indicatorName;
+			}
+			@Override
+			public int hashCode() {
+				final int prime = 31;
+				int result = 1;
+				result = prime
+						* result
+						+ ((algorithmName == null) ? 0 : algorithmName
+								.hashCode());
+				result = prime
+						* result
+						+ ((indicatorName == null) ? 0 : indicatorName
+								.hashCode());
+				result = prime
+						* result
+						+ ((instanceName == null) ? 0 : instanceName.hashCode());
+				result = prime * result + run;
+				return result;
+			}
+			@Override
+			public boolean equals(Object obj) {
+				if (this == obj)
+					return true;
+				if (obj == null)
+					return false;
+				if (getClass() != obj.getClass())
+					return false;
+				Key other = (Key) obj;
+				if (algorithmName == null) {
+					if (other.algorithmName != null)
+						return false;
+				} else if (!algorithmName.equals(other.algorithmName))
+					return false;
+				if (indicatorName == null) {
+					if (other.indicatorName != null)
+						return false;
+				} else if (!indicatorName.equals(other.indicatorName))
+					return false;
+				if (instanceName == null) {
+					if (other.instanceName != null)
+						return false;
+				} else if (!instanceName.equals(other.instanceName))
+					return false;
+				if (run != other.run)
+					return false;
+				return true;
+			}
+			public String getInstanceName() {
+				return instanceName;
+			}
+			public String getAlgorithmName() {
+				return algorithmName;
+			}
+			public int getRun() {
+				return run;
+			}
+			public String getIndicatorName() {
+				return indicatorName;
+			}
 			
-		private String algorithmName;
+			@Override
+			public String toString() {
+				return "[algorithm="+algorithmName+"][instance="+instanceName+"][run="+run+"][indicator="+indicatorName+"]";
+			}
 			
-		public ResultContainer(String instanceName, String algorithmName) {
-			this.instanceName = instanceName;
-			this.algorithmName = algorithmName;
 		}
 		
-		public void addResult(Result result){
+		private ConcurrentHashMap<Key, Double> data = new ConcurrentHashMap<ComputationalLaboratory.DataCollector.Key, Double>();
-			results.add(result);
+		
+		/**
+		 * Adds a single date by instanceName, algorithmName, run and indicatorName. 
+		 * <p>If there is already an entry for this instance, algorithm, run and indicatorName, it is overwritten.
+		 * 
+		 * @param instanceName
+		 * @param algorithmName
+		 * @param run
+		 * @param indicatorName
+		 * @param value
+		 */
+		public void addDate(String instanceName, String algorithmName, int run, String indicatorName, double value){
+			Key key = new Key(instanceName,algorithmName,run,indicatorName);
+			data.put(key, value);
 		}
 		
-		public List<Result> getResults() {
+		/**
-			return results;
+		 * Returns a collections of indicator values representing the calculated values of individual runs.
+		 * 
+		 * @param instanceName
+		 * @param algorithmName
+		 * @param indicator
+		 * @return
+		 */
+		public Collection<Double> getData(String instanceName, String algorithmName, String indicator){
+			List<Double> values = new ArrayList<Double>();
+			for(Key key : data.keySet()){
+				if(key.getAlgorithmName().equals(algorithmName) && key.getInstanceName().equals(instanceName) && key.getIndicatorName().equals(indicator)){
+					values.add(data.get(key));
+				}
+			}
+			return values;
 		}
 		
-		public String getInstanceName() { return instanceName; }
+		/**
+		 * Returns indicator value.
+		 * 
+		 * @param instanceName
+		 * @param algorithmName
+		 * @param run
+		 * @param indicator
+		 * @return
+		 */
+		public Double getDate(String instanceName, String algorithmName, int run, String indicator){
+			return data.get(new Key(instanceName,algorithmName,run,indicator));
+		}
 
-		public String getAlgorithmName() { return algorithmName; }
+		/**
+		 * Returns all keys that have been created. A key is a unique combination of algorithmName, instanceName, run and indicator.
+		 * 
+		 * @return
+		 */
+		public Set<Key> keySet(){
+			return data.keySet();
+		}
 		
-		public void addAllResults(Collection<Result> results) {
+		/**
-			this.results.addAll(results);
+		 * Returns date associated to specified key.
+		 * 
+		 * @param key
+		 * @return
+		 */
+		public Double getData(Key key){
+			return data.get(key);
 		}
 		
 	}
 	
-	public abstract static interface IndicatorFactory {
-		public abstract Indicator createIndicator(VehicleRoutingProblem vrp);
-	}
-	
-	public static interface ResultWriter {
-		public void writeResults(Collection<ResultContainer> results);
-	}
 	
 	private static class Algorithm {
 		
@@ -98,46 +213,90 @@ public class ComputationalLaboratory {
 
 	private int runs = 1;
 	
-	private Collection<ResultWriter> writers = new ArrayList<ResultWriter>();
+	private Collection<CalculationListener> listeners = new ArrayList<ComputationalLaboratory.CalculationListener>();
-	
-	private Collection<ResultContainer> results = new ArrayList<ResultContainer>();
-	
-	private Collection<IndicatorFactory> indicatorFactories = new ArrayList<IndicatorFactory>();
 	
 	private List<Algorithm> algorithms = new ArrayList<ComputationalLaboratory.Algorithm>();
 	
+	private Set<String> algorithmNames = new HashSet<String>();
+	
+	private Set<String> instanceNames = new HashSet<String>();
+	
 	private int threads = Runtime.getRuntime().availableProcessors()+1;
 	
 	public ComputationalLaboratory() {
 		Logger.getRootLogger().setLevel(Level.ERROR);
 	}
 	
-	public void addResultWriter(ResultWriter writer){
+	/**
-		writers.add(writer);
+	 * Adds algorithmFactory by name.
-	}
+	 * 
-	
+	 * @param name
+	 * @param factory
+	 * @throws IllegalStateException if there is already an algorithmFactory with the same name
+	 */
 	public void addAlgorithmFactory(String name, VehicleRoutingAlgorithmFactory factory){
+		if(algorithmNames.contains(name)) throw new IllegalStateException("there is already a algorithmFactory with the same name (algorithmName="+name+"). unique names are required.");
 		algorithms.add(new Algorithm(name,factory));
+		algorithmNames.add(name);
 	}
 
-	public void addIndicatorFactory(IndicatorFactory indicatorFactory){
+	/**
-		indicatorFactories.add(indicatorFactory);
+	 * Adds instance by name.
-	}
+	 * 
-
+	 * @param name
+	 * @param problem
+	 * @throws IllegalStateException if there is already an instance with the same name.
+	 */
 	public void addInstance(String name, VehicleRoutingProblem problem){
+		if(benchmarkInstances.contains(name)) throw new IllegalStateException("there is already an instance with the same name (instanceName="+name+"). unique names are required.");
 		benchmarkInstances.add(new BenchmarkInstance(name,problem,null,null));
+		instanceNames.add(name);
 	}
 	
+	/**
+	 * Adds instance.
+	 * 
+	 * @param name
+	 * @param problem
+	 * @throws IllegalStateException if there is already an instance with the same name.
+	 */
 	public void addInstance(BenchmarkInstance instance){
+		if(benchmarkInstances.contains(instance.name)) throw new IllegalStateException("there is already an instance with the same name (instanceName="+instance.name+"). unique names are required.");
 		benchmarkInstances.add(instance);
+		instanceNames.add(instance.name);
 	}
 	
+	/**
+	 * Adds collection of instances.
+	 * 
+	 * @param name
+	 * @param problem
+	 * @throws IllegalStateException if there is already an instance with the same name.
+	 */
 	public void addAllInstances(Collection<BenchmarkInstance> instances){
-		benchmarkInstances.addAll(instances);
+		for(BenchmarkInstance i : instances){
+			addInstance(i);
+		}
 	}
 	
+	/**
+	 * Adds instance by name, and with best known results.
+	 * 
+	 * @param name
+	 * @param problem
+	 * @throws IllegalStateException if there is already an instance with the same name.
+	 */
 	public void addInstance(String name, VehicleRoutingProblem problem, Double bestKnownResult, Double bestKnownVehicles){
-		benchmarkInstances.add(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
+		addInstance(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
+	}
+	
+	/**
+	 * Adds listener to listen computational experiments.
+	 * 
+	 * @param listener
+	 */
+	public void addListener(CalculationListener listener){
+		listeners.add(listener);
 	}
 	
 	/**
@@ -150,6 +309,19 @@ public class ComputationalLaboratory {
 		this.runs = runs;
 	}
 	
+	/**
+	 * Runs experiments.
+	 * 
+	 * <p>If nuThreads > 1 it runs them concurrently, i.e. individual runs are distributed to available threads. Therefore 
+	 * a unique task is defined by its algorithmName, instanceName and its runNumber.
+	 * <p>If you have one algorithm called "myAlgorithm" and one instance called "myInstance", and you need to run "myAlgorithm" on "myInstance" three times
+	 * with three threads then "myAlgorithm","myInstance",run1 runs on the first thread, "myAlgorithm", "myInstance", run2 on the second etc.
+	 * <p>You can register whatever analysisTool you require by implementing and registering CalculationListener. Then your tool is informed just
+	 * before a calculation starts as well as just after a calculation has been finished.
+	 * 
+	 * @see CalculationListener
+	 * @throws IllegalStateException if either no algorithm or no instance has been specified 
+	 */
 	public void run(){
 		if(algorithms.isEmpty()){
 			throw new IllegalStateException("no algorithm specified. at least one algorithm needs to be specified.");
@@ -157,96 +329,60 @@ public class ComputationalLaboratory {
 		if(benchmarkInstances.isEmpty()){
 			throw new IllegalStateException("no instance specified. at least one instance needs to be specified.");
 		}
-		if(indicatorFactories.isEmpty()){
-			throw new IllegalStateException("no indicator specified. at least one indicator needs to be specified.");
-		}
 		System.out.println("start benchmarking [nuAlgorithms="+algorithms.size()+"][nuInstances=" + benchmarkInstances.size() + "][runsPerInstance=" + runs + "]");
 		double startTime = System.currentTimeMillis();
 		ExecutorService executor = Executors.newFixedThreadPool(threads);
-		List<Future<ResultContainer>> futures = new ArrayList<Future<ResultContainer>>();
 		for(final Algorithm algorithm : algorithms){
 			for(final BenchmarkInstance p : benchmarkInstances){
+				for(int run=0;run<runs;run++){
+					final int r = run;
+					executor.submit(new Runnable(){
 
-				Future<ResultContainer> futureResult = executor.submit(new Callable<ResultContainer>(){
+						@Override
+						public void run() {
+							runAlgorithm(p, algorithm, r+1);
+						}
 						
-					@Override
+					});
-					public ResultContainer call() throws Exception {
+				}
-						return runAlgoAndGetResult(p, algorithm);
-					}
-
-				});
-				futures.add(futureResult);
 			}
 		}
 		try {
-			for(Future<ResultContainer> f : futures){
+			executor.shutdown();
-				results.add(f.get());
+			executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
-			}
+			
 		} catch (InterruptedException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		} catch (ExecutionException e) {
-			// TODO Auto-generated catch block
 			e.printStackTrace();
 		}
-		executor.shutdown();
+		System.out.println("benchmarking done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]");
-		write(results);
-		System.out.println("done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]");
 	}
 
+	/**
+	 * Sets number of threads.
+	 * <p>By default: <code>nuThreads = Runtime.getRuntime().availableProcessors()+1</code>
+	 * 
+	 * @param threads
+	 */
 	public void setThreads(int threads) {
 		this.threads = threads;
 	}
 
-	private ResultContainer runAlgoAndGetResult(BenchmarkInstance p, Algorithm algorithm) {
+	private void runAlgorithm(BenchmarkInstance p, Algorithm algorithm, int run) {
-		System.out.println("run " + algorithm.name + " on " + p.name);
+		System.out.println("[algorithm=" + algorithm.name + "][instance="+p.name+"][run="+run+"][status=start]");
-		List<Indicator> indicators = createIndicators(p.vrp);
+		VehicleRoutingAlgorithm vra = algorithm.factory.createAlgorithm(p.vrp);
-		informCalculationStarts(indicators);
+		informCalculationStarts(p, algorithm.name, vra, run);
-		for(int run=0;run<runs;run++){
+		Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
-			System.out.println("algorithmName=" + algorithm.name + "; instanceName=" + p.name + "; run=" + (run+1));
+		System.out.println("[algorithm=" + algorithm.name + "][instance="+p.name+"][run="+run+"][status=finished]");
-			VehicleRoutingAlgorithm vra = algorithm.factory.createAlgorithm(p.vrp);
+		informCalculationsEnds(p, algorithm.name, vra, run, solutions);
-			informRunStarts(indicators,vra);
-			StopWatch stopwatch = new StopWatch();
-			vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
-			Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
-			VehicleRoutingProblemSolution best = Solutions.bestOf(solutions);
-			informRunEnds(indicators,best);
-		}
-		System.out.println("finished runs of " + algorithm.name + " on " + p.name);
-		ResultContainer resultContainer = getResultContainer(indicators, p.name, algorithm.name);
-		return resultContainer;
 	}
 
-	private ResultContainer getResultContainer(List<Indicator> indicators, String instanceName, String algorithmName) {
+	private void informCalculationStarts(BenchmarkInstance p, String name, VehicleRoutingAlgorithm vra, int run) {
-		ResultContainer rc = new ResultContainer(instanceName, algorithmName);
+		for(CalculationListener l : listeners) l.calculationStarts(p, name, vra, run);
-		for(final Indicator i : indicators) {
-			rc.addAllResults(i.getResults());
-		}
-		return rc;
 	}
 
-	private void informRunEnds(List<Indicator> indicators,VehicleRoutingProblemSolution best) {
+	private void informCalculationsEnds(BenchmarkInstance p, String name, VehicleRoutingAlgorithm vra, int run,
-		for(Indicator i : indicators) i.runEnds(best);
+			Collection<VehicleRoutingProblemSolution> solutions) {
-	}
+		for(CalculationListener l : listeners) l.calculationEnds(p, name, vra, run, solutions);
-
-	private void informRunStarts(List<Indicator> indicators,VehicleRoutingAlgorithm vra) {
-		for(Indicator i : indicators) i.runStarts(vra);
-	}
-
-	private void informCalculationStarts(List<Indicator> indicators) {
-		for(Indicator i : indicators) i.calculationStarts();
-	}
-
-	private List<Indicator> createIndicators(VehicleRoutingProblem vrp) {
-		List<Indicator> indicators = new ArrayList<ComputationalLaboratory.Indicator>();
-		for(IndicatorFactory iFactory : indicatorFactories) indicators.add(iFactory.createIndicator(vrp));
-		return indicators;
-	}
-
-	private void write(Collection<ResultContainer> results) {
-		for(ResultWriter writer : writers){
-			writer.writeResults(results);
-		}
 	}
 
 }