trans/graphhopper-jsprit
1
0
Fork 0
mirror of https://github.com/graphhopper/jsprit.git synced 2020-01-24 07:45:05 +01:00
Code Issues Releases Wiki Activity

refine and improve benchmarking

Browse source
This commit is contained in:
oblonski 2013-07-09 14:01:41 +02:00
parent 70bb212848
commit 3d0594448e
6 changed files with 537 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

125
jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java
View file

@ -30,9 +30,12 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import util.BenchmarkInstance;
import util.BenchmarkResult;
import util.BenchmarkWriter;
import util.Solutions;
import algorithms.VehicleRoutingAlgorithms;
@ -43,45 +46,32 @@ import basics.algo.VehicleRoutingAlgorithmListeners.Priority;
public class ConcurrentBenchmarker {
public static class BenchmarkInstance {
public final String name;
public final VehicleRoutingProblem vrp;
public final Double bestKnown;
public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnown) {
super();
this.name = name;
this.vrp = vrp;
this.bestKnown = bestKnown;
}
public static interface Cost {
public double getCost(VehicleRoutingProblemSolution sol);
}
public static class BenchmarkResult {
public final double result;
public final double time;
public final int nuOfVehicles;
public final BenchmarkInstance instance;
public Double delta = null;
public BenchmarkResult(BenchmarkInstance p, double result, double time, int nuOfVehicles) {
super();
this.result = result;
this.time = time;
this.instance = p;
this.nuOfVehicles = nuOfVehicles;
}
void setBestKnownDelta(double delta){
this.delta = delta;
}
}
private String algorithmConfig;
private List<BenchmarkInstance> problems = new ArrayList<BenchmarkInstance>();
private List<BenchmarkInstance> benchmarkInstances = new ArrayList<BenchmarkInstance>();
private int runs = 1;
private Collection<BenchmarkWriter> writers = new ArrayList<BenchmarkWriter>();
private Collection<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
private Collection<BenchmarkResult> results = new ArrayList<BenchmarkResult>();
private Cost cost = new Cost(){
@Override
public double getCost(VehicleRoutingProblemSolution sol) {
return sol.getCost();
}
};
public void setCost(Cost cost){ this.cost = cost; }
public ConcurrentBenchmarker(String algorithmConfig) {
super();
@ -93,12 +83,20 @@ public class ConcurrentBenchmarker {
writers.add(writer);
}
public void addProblem(String name, VehicleRoutingProblem problem){
problems.add(new BenchmarkInstance(name,problem,null));
public void addInstance(String name, VehicleRoutingProblem problem){
benchmarkInstances.add(new BenchmarkInstance(name,problem,null,null));
}
public void addProblem(String name, VehicleRoutingProblem problem, double bestKnown){
problems.add(new BenchmarkInstance(name,problem,bestKnown));
public void addInstane(BenchmarkInstance instance){
benchmarkInstances.add(instance);
}
public void addAllInstances(Collection<BenchmarkInstance> instances){
benchmarkInstances.addAll(instances);
}
public void addInstance(String name, VehicleRoutingProblem problem, Double bestKnownResult, Double bestKnownVehicles){
benchmarkInstances.add(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
}
public void setNuOfRuns(int runs){
@ -106,13 +104,12 @@ public class ConcurrentBenchmarker {
}
public void run(){
System.out.println("start benchmarking [nuOfInstances=" + problems.size() + "]");
System.out.println("start benchmarking [nuOfInstances=" + benchmarkInstances.size() + "][runsPerInstance=" + runs + "]");
double startTime = System.currentTimeMillis();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()+1);
List<Future<BenchmarkResult>> futures = new ArrayList<Future<BenchmarkResult>>();
// List<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
for(final BenchmarkInstance p : problems){
for(int run=0;run<runs;run++){
for(final BenchmarkInstance p : benchmarkInstances){
Future<BenchmarkResult> futureResult = executor.submit(new Callable<BenchmarkResult>(){
@Override
@ -122,7 +119,7 @@ public class ConcurrentBenchmarker {
});
futures.add(futureResult);
}
}
try {
int count = 1;
@ -145,22 +142,30 @@ public class ConcurrentBenchmarker {
}
private BenchmarkResult runAlgoAndGetResult(BenchmarkInstance p) {
VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
StopWatch stopwatch = new StopWatch();
vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
BenchmarkResult result = new BenchmarkResult(p,best.getCost(),stopwatch.getCompTimeInSeconds(), best.getRoutes().size());
if(p.bestKnown != null) result.setBestKnownDelta((best.getCost()/p.bestKnown-1));
return result;
double[] vehicles = new double[runs];
double[] results = new double[runs];
double[] times = new double[runs];
for(int run=0;run<runs;run++){
VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
StopWatch stopwatch = new StopWatch();
vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
vehicles[run] = best.getRoutes().size();
results[run] = cost.getCost(best);
times[run] = stopwatch.getCompTimeInSeconds();
}
return new BenchmarkResult(p, runs, results, times, vehicles);
}
private void print(Collection<BenchmarkResult> results) {
double sumTime=0.0;
double sumResult=0.0;
for(BenchmarkResult r : results){
sumTime+=r.time;
sumResult+=r.result;
sumTime+=r.getTimesStats().getMean();
sumResult+=r.getResultStats().getMean();
// print(r);
}
System.out.println("[avgTime="+round(sumTime/(double)results.size(),2)+"][avgResult="+round(sumResult/(double)results.size(),2)+"]");
@ -170,9 +175,27 @@ public class ConcurrentBenchmarker {
}
private void print(BenchmarkResult r, int count) {
System.out.println("("+count+"/"+problems.size() +")"+ "\t[instance="+r.instance.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+
"][delta="+round(r.delta,3)+"][nuOfVehicles="+r.nuOfVehicles+"]");
Double avgDelta = null;
Double bestDelta = null;
Double worstDelta = null;
if(r.instance.bestKnownResult != null){
avgDelta = (r.getResultStats().getMean() / r.instance.bestKnownResult - 1) * 100;
bestDelta = (r.getResultStats().getMin() / r.instance.bestKnownResult - 1) * 100;
worstDelta = (r.getResultStats().getMax() / r.instance.bestKnownResult - 1) * 100;
}
System.out.println("("+count+"/"+benchmarkInstances.size() +")"+ "\t[instance="+r.instance.name+
"][avgTime="+round(r.getTimesStats().getMean(),2)+"]" +
"[Result=" + getString(r.getResultStats()) + "]" +
"[Vehicles=" + getString(r.getVehicleStats()) + "]" +
"[Delta[%]=" + getString(bestDelta,avgDelta,worstDelta) + "]");
}
private String getString(Double bestDelta, Double avgDelta,Double worstDelta) {
return "[best="+round(bestDelta,2)+"][avg="+round(avgDelta,2)+"][worst="+round(worstDelta,2)+"]";
}
private String getString(DescriptiveStatistics stats){
return "[best="+round(stats.getMin(),2)+"][avg="+round(stats.getMean(),2)+"][worst="+round(stats.getMax(),2)+"][stdDev=" + round(stats.getStandardDeviation(),2)+"]";
}
private Double round(Double value, int i) {