mirror of
https://github.com/graphhopper/jsprit.git
synced 2020-01-24 07:45:05 +01:00
refined, simplified, accelerated and documented
analysis.toolbox.ComputationalLaboratory.java
This commit is contained in:
oblonski
parent
c3043a7431
commit
98920bba00
1 changed files with 264 additions and 128 deletions
|
|
@ -2,83 +2,198 @@ package jsprit.analysis.toolbox;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import jsprit.core.algorithm.VehicleRoutingAlgorithm;
|
||||
import jsprit.core.algorithm.VehicleRoutingAlgorithmFactory;
|
||||
import jsprit.core.algorithm.listener.VehicleRoutingAlgorithmListeners.Priority;
|
||||
import jsprit.core.problem.VehicleRoutingProblem;
|
||||
import jsprit.core.problem.solution.VehicleRoutingProblemSolution;
|
||||
import jsprit.core.util.BenchmarkInstance;
|
||||
import jsprit.core.util.Solutions;
|
||||
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
public class ComputationalLaboratory {
|
||||
|
||||
public abstract static interface Result {
|
||||
/**
|
||||
* Listener-interface to listen to calculation.
|
||||
*
|
||||
* <p>Note that calculations are run concurrently, i.e. a unique task that is distributed to an available thread is
|
||||
* {algorithm, instance, run}.
|
||||
*
|
||||
* @author schroeder
|
||||
*
|
||||
*/
|
||||
public static interface CalculationListener {
|
||||
|
||||
public abstract String getIndicatorName();
|
||||
public void calculationStarts(final BenchmarkInstance p, final String algorithmName, final VehicleRoutingAlgorithm algorithm, final int run);
|
||||
|
||||
public abstract double getResult();
|
||||
}
|
||||
|
||||
public abstract static interface Indicator {
|
||||
|
||||
public abstract void calculationStarts();
|
||||
|
||||
public abstract void runStarts(VehicleRoutingAlgorithm vra);
|
||||
|
||||
public abstract void runEnds(VehicleRoutingProblemSolution vrs);
|
||||
|
||||
public abstract Collection<Result> getResults();
|
||||
public void calculationEnds(final BenchmarkInstance p, final String algorithmName, final VehicleRoutingAlgorithm algorithm, final int run, final Collection<VehicleRoutingProblemSolution> solutions);
|
||||
|
||||
}
|
||||
|
||||
public static class ResultContainer {
|
||||
|
||||
private List<Result> results = new ArrayList<Result>();
|
||||
/**
|
||||
* Collects whatever indicators you require by algorithmName, instanceName, run and indicator.
|
||||
*
|
||||
* @author schroeder
|
||||
*
|
||||
*/
|
||||
public static class DataCollector {
|
||||
|
||||
public static class Key {
|
||||
private String instanceName;
|
||||
|
||||
private String algorithmName;
|
||||
private int run;
|
||||
private String indicatorName;
|
||||
|
||||
public ResultContainer(String instanceName, String algorithmName) {
|
||||
public Key(String instanceName, String algorithmName, int run,String indicatorName) {
|
||||
super();
|
||||
this.instanceName = instanceName;
|
||||
this.algorithmName = algorithmName;
|
||||
this.run = run;
|
||||
this.indicatorName = indicatorName;
|
||||
}
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime
|
||||
* result
|
||||
+ ((algorithmName == null) ? 0 : algorithmName
|
||||
.hashCode());
|
||||
result = prime
|
||||
* result
|
||||
+ ((indicatorName == null) ? 0 : indicatorName
|
||||
.hashCode());
|
||||
result = prime
|
||||
* result
|
||||
+ ((instanceName == null) ? 0 : instanceName.hashCode());
|
||||
result = prime * result + run;
|
||||
return result;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
Key other = (Key) obj;
|
||||
if (algorithmName == null) {
|
||||
if (other.algorithmName != null)
|
||||
return false;
|
||||
} else if (!algorithmName.equals(other.algorithmName))
|
||||
return false;
|
||||
if (indicatorName == null) {
|
||||
if (other.indicatorName != null)
|
||||
return false;
|
||||
} else if (!indicatorName.equals(other.indicatorName))
|
||||
return false;
|
||||
if (instanceName == null) {
|
||||
if (other.instanceName != null)
|
||||
return false;
|
||||
} else if (!instanceName.equals(other.instanceName))
|
||||
return false;
|
||||
if (run != other.run)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
public String getInstanceName() {
|
||||
return instanceName;
|
||||
}
|
||||
public String getAlgorithmName() {
|
||||
return algorithmName;
|
||||
}
|
||||
public int getRun() {
|
||||
return run;
|
||||
}
|
||||
public String getIndicatorName() {
|
||||
return indicatorName;
|
||||
}
|
||||
|
||||
public void addResult(Result result){
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
public List<Result> getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
public String getInstanceName() { return instanceName; }
|
||||
|
||||
public String getAlgorithmName() { return algorithmName; }
|
||||
|
||||
public void addAllResults(Collection<Result> results) {
|
||||
this.results.addAll(results);
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[algorithm="+algorithmName+"][instance="+instanceName+"][run="+run+"][indicator="+indicatorName+"]";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public abstract static interface IndicatorFactory {
|
||||
public abstract Indicator createIndicator(VehicleRoutingProblem vrp);
|
||||
private ConcurrentHashMap<Key, Double> data = new ConcurrentHashMap<ComputationalLaboratory.DataCollector.Key, Double>();
|
||||
|
||||
/**
|
||||
* Adds a single date by instanceName, algorithmName, run and indicatorName.
|
||||
* <p>If there is already an entry for this instance, algorithm, run and indicatorName, it is overwritten.
|
||||
*
|
||||
* @param instanceName
|
||||
* @param algorithmName
|
||||
* @param run
|
||||
* @param indicatorName
|
||||
* @param value
|
||||
*/
|
||||
public void addDate(String instanceName, String algorithmName, int run, String indicatorName, double value){
|
||||
Key key = new Key(instanceName,algorithmName,run,indicatorName);
|
||||
data.put(key, value);
|
||||
}
|
||||
|
||||
public static interface ResultWriter {
|
||||
public void writeResults(Collection<ResultContainer> results);
|
||||
/**
|
||||
* Returns a collections of indicator values representing the calculated values of individual runs.
|
||||
*
|
||||
* @param instanceName
|
||||
* @param algorithmName
|
||||
* @param indicator
|
||||
* @return
|
||||
*/
|
||||
public Collection<Double> getData(String instanceName, String algorithmName, String indicator){
|
||||
List<Double> values = new ArrayList<Double>();
|
||||
for(Key key : data.keySet()){
|
||||
if(key.getAlgorithmName().equals(algorithmName) && key.getInstanceName().equals(instanceName) && key.getIndicatorName().equals(indicator)){
|
||||
values.add(data.get(key));
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns indicator value.
|
||||
*
|
||||
* @param instanceName
|
||||
* @param algorithmName
|
||||
* @param run
|
||||
* @param indicator
|
||||
* @return
|
||||
*/
|
||||
public Double getDate(String instanceName, String algorithmName, int run, String indicator){
|
||||
return data.get(new Key(instanceName,algorithmName,run,indicator));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all keys that have been created. A key is a unique combination of algorithmName, instanceName, run and indicator.
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Set<Key> keySet(){
|
||||
return data.keySet();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns date associated to specified key.
|
||||
*
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public Double getData(Key key){
|
||||
return data.get(key);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static class Algorithm {
|
||||
|
||||
|
|
@ -98,46 +213,90 @@ public class ComputationalLaboratory {
|
|||
|
||||
private int runs = 1;
|
||||
|
||||
private Collection<ResultWriter> writers = new ArrayList<ResultWriter>();
|
||||
|
||||
private Collection<ResultContainer> results = new ArrayList<ResultContainer>();
|
||||
|
||||
private Collection<IndicatorFactory> indicatorFactories = new ArrayList<IndicatorFactory>();
|
||||
private Collection<CalculationListener> listeners = new ArrayList<ComputationalLaboratory.CalculationListener>();
|
||||
|
||||
private List<Algorithm> algorithms = new ArrayList<ComputationalLaboratory.Algorithm>();
|
||||
|
||||
private Set<String> algorithmNames = new HashSet<String>();
|
||||
|
||||
private Set<String> instanceNames = new HashSet<String>();
|
||||
|
||||
private int threads = Runtime.getRuntime().availableProcessors()+1;
|
||||
|
||||
public ComputationalLaboratory() {
|
||||
Logger.getRootLogger().setLevel(Level.ERROR);
|
||||
}
|
||||
|
||||
public void addResultWriter(ResultWriter writer){
|
||||
writers.add(writer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds algorithmFactory by name.
|
||||
*
|
||||
* @param name
|
||||
* @param factory
|
||||
* @throws IllegalStateException if there is already an algorithmFactory with the same name
|
||||
*/
|
||||
public void addAlgorithmFactory(String name, VehicleRoutingAlgorithmFactory factory){
|
||||
if(algorithmNames.contains(name)) throw new IllegalStateException("there is already a algorithmFactory with the same name (algorithmName="+name+"). unique names are required.");
|
||||
algorithms.add(new Algorithm(name,factory));
|
||||
algorithmNames.add(name);
|
||||
}
|
||||
|
||||
public void addIndicatorFactory(IndicatorFactory indicatorFactory){
|
||||
indicatorFactories.add(indicatorFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds instance by name.
|
||||
*
|
||||
* @param name
|
||||
* @param problem
|
||||
* @throws IllegalStateException if there is already an instance with the same name.
|
||||
*/
|
||||
public void addInstance(String name, VehicleRoutingProblem problem){
|
||||
if(benchmarkInstances.contains(name)) throw new IllegalStateException("there is already an instance with the same name (instanceName="+name+"). unique names are required.");
|
||||
benchmarkInstances.add(new BenchmarkInstance(name,problem,null,null));
|
||||
instanceNames.add(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds instance.
|
||||
*
|
||||
* @param name
|
||||
* @param problem
|
||||
* @throws IllegalStateException if there is already an instance with the same name.
|
||||
*/
|
||||
public void addInstance(BenchmarkInstance instance){
|
||||
if(benchmarkInstances.contains(instance.name)) throw new IllegalStateException("there is already an instance with the same name (instanceName="+instance.name+"). unique names are required.");
|
||||
benchmarkInstances.add(instance);
|
||||
instanceNames.add(instance.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds collection of instances.
|
||||
*
|
||||
* @param name
|
||||
* @param problem
|
||||
* @throws IllegalStateException if there is already an instance with the same name.
|
||||
*/
|
||||
public void addAllInstances(Collection<BenchmarkInstance> instances){
|
||||
benchmarkInstances.addAll(instances);
|
||||
for(BenchmarkInstance i : instances){
|
||||
addInstance(i);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds instance by name, and with best known results.
|
||||
*
|
||||
* @param name
|
||||
* @param problem
|
||||
* @throws IllegalStateException if there is already an instance with the same name.
|
||||
*/
|
||||
public void addInstance(String name, VehicleRoutingProblem problem, Double bestKnownResult, Double bestKnownVehicles){
|
||||
benchmarkInstances.add(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
|
||||
addInstance(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds listener to listen computational experiments.
|
||||
*
|
||||
* @param listener
|
||||
*/
|
||||
public void addListener(CalculationListener listener){
|
||||
listeners.add(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -150,6 +309,19 @@ public class ComputationalLaboratory {
|
|||
this.runs = runs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs experiments.
|
||||
*
|
||||
* <p>If nuThreads > 1 it runs them concurrently, i.e. individual runs are distributed to available threads. Therefore
|
||||
* a unique task is defined by its algorithmName, instanceName and its runNumber.
|
||||
* <p>If you have one algorithm called "myAlgorithm" and one instance called "myInstance", and you need to run "myAlgorithm" on "myInstance" three times
|
||||
* with three threads then "myAlgorithm","myInstance",run1 runs on the first thread, "myAlgorithm", "myInstance", run2 on the second etc.
|
||||
* <p>You can register whatever analysisTool you require by implementing and registering CalculationListener. Then your tool is informed just
|
||||
* before a calculation starts as well as just after a calculation has been finished.
|
||||
*
|
||||
* @see CalculationListener
|
||||
* @throws IllegalStateException if either no algorithm or no instance has been specified
|
||||
*/
|
||||
public void run(){
|
||||
if(algorithms.isEmpty()){
|
||||
throw new IllegalStateException("no algorithm specified. at least one algorithm needs to be specified.");
|
||||
|
|
@ -157,96 +329,60 @@ public class ComputationalLaboratory {
|
|||
if(benchmarkInstances.isEmpty()){
|
||||
throw new IllegalStateException("no instance specified. at least one instance needs to be specified.");
|
||||
}
|
||||
if(indicatorFactories.isEmpty()){
|
||||
throw new IllegalStateException("no indicator specified. at least one indicator needs to be specified.");
|
||||
}
|
||||
System.out.println("start benchmarking [nuAlgorithms="+algorithms.size()+"][nuInstances=" + benchmarkInstances.size() + "][runsPerInstance=" + runs + "]");
|
||||
double startTime = System.currentTimeMillis();
|
||||
ExecutorService executor = Executors.newFixedThreadPool(threads);
|
||||
List<Future<ResultContainer>> futures = new ArrayList<Future<ResultContainer>>();
|
||||
for(final Algorithm algorithm : algorithms){
|
||||
for(final BenchmarkInstance p : benchmarkInstances){
|
||||
|
||||
Future<ResultContainer> futureResult = executor.submit(new Callable<ResultContainer>(){
|
||||
for(int run=0;run<runs;run++){
|
||||
final int r = run;
|
||||
executor.submit(new Runnable(){
|
||||
|
||||
@Override
|
||||
public ResultContainer call() throws Exception {
|
||||
return runAlgoAndGetResult(p, algorithm);
|
||||
public void run() {
|
||||
runAlgorithm(p, algorithm, r+1);
|
||||
}
|
||||
|
||||
});
|
||||
futures.add(futureResult);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
for(Future<ResultContainer> f : futures){
|
||||
results.add(f.get());
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ExecutionException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
executor.shutdown();
|
||||
write(results);
|
||||
System.out.println("done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]");
|
||||
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
System.out.println("benchmarking done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]");
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets number of threads.
|
||||
* <p>By default: <code>nuThreads = Runtime.getRuntime().availableProcessors()+1</code>
|
||||
*
|
||||
* @param threads
|
||||
*/
|
||||
public void setThreads(int threads) {
|
||||
this.threads = threads;
|
||||
}
|
||||
|
||||
private ResultContainer runAlgoAndGetResult(BenchmarkInstance p, Algorithm algorithm) {
|
||||
System.out.println("run " + algorithm.name + " on " + p.name);
|
||||
List<Indicator> indicators = createIndicators(p.vrp);
|
||||
informCalculationStarts(indicators);
|
||||
for(int run=0;run<runs;run++){
|
||||
System.out.println("algorithmName=" + algorithm.name + "; instanceName=" + p.name + "; run=" + (run+1));
|
||||
private void runAlgorithm(BenchmarkInstance p, Algorithm algorithm, int run) {
|
||||
System.out.println("[algorithm=" + algorithm.name + "][instance="+p.name+"][run="+run+"][status=start]");
|
||||
VehicleRoutingAlgorithm vra = algorithm.factory.createAlgorithm(p.vrp);
|
||||
informRunStarts(indicators,vra);
|
||||
StopWatch stopwatch = new StopWatch();
|
||||
vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
|
||||
informCalculationStarts(p, algorithm.name, vra, run);
|
||||
Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
|
||||
VehicleRoutingProblemSolution best = Solutions.bestOf(solutions);
|
||||
informRunEnds(indicators,best);
|
||||
}
|
||||
System.out.println("finished runs of " + algorithm.name + " on " + p.name);
|
||||
ResultContainer resultContainer = getResultContainer(indicators, p.name, algorithm.name);
|
||||
return resultContainer;
|
||||
System.out.println("[algorithm=" + algorithm.name + "][instance="+p.name+"][run="+run+"][status=finished]");
|
||||
informCalculationsEnds(p, algorithm.name, vra, run, solutions);
|
||||
}
|
||||
|
||||
private ResultContainer getResultContainer(List<Indicator> indicators, String instanceName, String algorithmName) {
|
||||
ResultContainer rc = new ResultContainer(instanceName, algorithmName);
|
||||
for(final Indicator i : indicators) {
|
||||
rc.addAllResults(i.getResults());
|
||||
}
|
||||
return rc;
|
||||
private void informCalculationStarts(BenchmarkInstance p, String name, VehicleRoutingAlgorithm vra, int run) {
|
||||
for(CalculationListener l : listeners) l.calculationStarts(p, name, vra, run);
|
||||
}
|
||||
|
||||
private void informRunEnds(List<Indicator> indicators,VehicleRoutingProblemSolution best) {
|
||||
for(Indicator i : indicators) i.runEnds(best);
|
||||
}
|
||||
|
||||
private void informRunStarts(List<Indicator> indicators,VehicleRoutingAlgorithm vra) {
|
||||
for(Indicator i : indicators) i.runStarts(vra);
|
||||
}
|
||||
|
||||
private void informCalculationStarts(List<Indicator> indicators) {
|
||||
for(Indicator i : indicators) i.calculationStarts();
|
||||
}
|
||||
|
||||
private List<Indicator> createIndicators(VehicleRoutingProblem vrp) {
|
||||
List<Indicator> indicators = new ArrayList<ComputationalLaboratory.Indicator>();
|
||||
for(IndicatorFactory iFactory : indicatorFactories) indicators.add(iFactory.createIndicator(vrp));
|
||||
return indicators;
|
||||
}
|
||||
|
||||
private void write(Collection<ResultContainer> results) {
|
||||
for(ResultWriter writer : writers){
|
||||
writer.writeResults(results);
|
||||
}
|
||||
private void informCalculationsEnds(BenchmarkInstance p, String name, VehicleRoutingAlgorithm vra, int run,
|
||||
Collection<VehicleRoutingProblemSolution> solutions) {
|
||||
for(CalculationListener l : listeners) l.calculationEnds(p, name, vra, run, solutions);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue