trans/graphhopper-jsprit
1
0
Fork 0
mirror of https://github.com/graphhopper/jsprit.git synced 2020-01-24 07:45:05 +01:00
Code Issues Releases Wiki Activity

refine and improve benchmarking

Browse source
This commit is contained in:
oblonski 2013-07-09 14:01:41 +02:00
parent 70bb212848
commit 3d0594448e
6 changed files with 537 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

125
jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java
View file

@ -30,9 +30,12 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import util.BenchmarkInstance;
import util.BenchmarkResult;
import util.BenchmarkWriter;
import util.Solutions;
import algorithms.VehicleRoutingAlgorithms;
@ -43,45 +46,32 @@ import basics.algo.VehicleRoutingAlgorithmListeners.Priority;
public class ConcurrentBenchmarker {
public static class BenchmarkInstance {
public final String name;
public final VehicleRoutingProblem vrp;
public final Double bestKnown;
public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnown) {
super();
this.name = name;
this.vrp = vrp;
this.bestKnown = bestKnown;
}
public static interface Cost {
public double getCost(VehicleRoutingProblemSolution sol);
}
public static class BenchmarkResult {
public final double result;
public final double time;
public final int nuOfVehicles;
public final BenchmarkInstance instance;
public Double delta = null;
public BenchmarkResult(BenchmarkInstance p, double result, double time, int nuOfVehicles) {
super();
this.result = result;
this.time = time;
this.instance = p;
this.nuOfVehicles = nuOfVehicles;
}
void setBestKnownDelta(double delta){
this.delta = delta;
}
}
private String algorithmConfig;
private List<BenchmarkInstance> problems = new ArrayList<BenchmarkInstance>();
private List<BenchmarkInstance> benchmarkInstances = new ArrayList<BenchmarkInstance>();
private int runs = 1;
private Collection<BenchmarkWriter> writers = new ArrayList<BenchmarkWriter>();
private Collection<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
private Collection<BenchmarkResult> results = new ArrayList<BenchmarkResult>();
private Cost cost = new Cost(){
@Override
public double getCost(VehicleRoutingProblemSolution sol) {
return sol.getCost();
}
};
public void setCost(Cost cost){ this.cost = cost; }
public ConcurrentBenchmarker(String algorithmConfig) {
super();
@ -93,12 +83,20 @@ public class ConcurrentBenchmarker {
writers.add(writer);
}
public void addProblem(String name, VehicleRoutingProblem problem){
problems.add(new BenchmarkInstance(name,problem,null));
public void addInstance(String name, VehicleRoutingProblem problem){
benchmarkInstances.add(new BenchmarkInstance(name,problem,null,null));
}
public void addProblem(String name, VehicleRoutingProblem problem, double bestKnown){
problems.add(new BenchmarkInstance(name,problem,bestKnown));
public void addInstane(BenchmarkInstance instance){
benchmarkInstances.add(instance);
}
public void addAllInstances(Collection<BenchmarkInstance> instances){
benchmarkInstances.addAll(instances);
}
public void addInstance(String name, VehicleRoutingProblem problem, Double bestKnownResult, Double bestKnownVehicles){
benchmarkInstances.add(new BenchmarkInstance(name,problem,bestKnownResult,bestKnownVehicles));
}
public void setNuOfRuns(int runs){
@ -106,13 +104,12 @@ public class ConcurrentBenchmarker {
}
public void run(){
System.out.println("start benchmarking [nuOfInstances=" + problems.size() + "]");
System.out.println("start benchmarking [nuOfInstances=" + benchmarkInstances.size() + "][runsPerInstance=" + runs + "]");
double startTime = System.currentTimeMillis();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()+1);
List<Future<BenchmarkResult>> futures = new ArrayList<Future<BenchmarkResult>>();
// List<BenchmarkResult> results = new ArrayList<ConcurrentBenchmarker.BenchmarkResult>();
for(final BenchmarkInstance p : problems){
for(int run=0;run<runs;run++){
for(final BenchmarkInstance p : benchmarkInstances){
Future<BenchmarkResult> futureResult = executor.submit(new Callable<BenchmarkResult>(){
@Override
@ -122,7 +119,7 @@ public class ConcurrentBenchmarker {
});
futures.add(futureResult);
}
}
try {
int count = 1;
@ -145,22 +142,30 @@ public class ConcurrentBenchmarker {
}
private BenchmarkResult runAlgoAndGetResult(BenchmarkInstance p) {
VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
StopWatch stopwatch = new StopWatch();
vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
BenchmarkResult result = new BenchmarkResult(p,best.getCost(),stopwatch.getCompTimeInSeconds(), best.getRoutes().size());
if(p.bestKnown != null) result.setBestKnownDelta((best.getCost()/p.bestKnown-1));
return result;
double[] vehicles = new double[runs];
double[] results = new double[runs];
double[] times = new double[runs];
for(int run=0;run<runs;run++){
VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig);
StopWatch stopwatch = new StopWatch();
vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH);
Collection<VehicleRoutingProblemSolution> solutions = vra.searchSolutions();
VehicleRoutingProblemSolution best = Solutions.getBest(solutions);
vehicles[run] = best.getRoutes().size();
results[run] = cost.getCost(best);
times[run] = stopwatch.getCompTimeInSeconds();
}
return new BenchmarkResult(p, runs, results, times, vehicles);
}
private void print(Collection<BenchmarkResult> results) {
double sumTime=0.0;
double sumResult=0.0;
for(BenchmarkResult r : results){
sumTime+=r.time;
sumResult+=r.result;
sumTime+=r.getTimesStats().getMean();
sumResult+=r.getResultStats().getMean();
// print(r);
}
System.out.println("[avgTime="+round(sumTime/(double)results.size(),2)+"][avgResult="+round(sumResult/(double)results.size(),2)+"]");
@ -170,9 +175,27 @@ public class ConcurrentBenchmarker {
}
private void print(BenchmarkResult r, int count) {
System.out.println("("+count+"/"+problems.size() +")"+ "\t[instance="+r.instance.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+
"][delta="+round(r.delta,3)+"][nuOfVehicles="+r.nuOfVehicles+"]");
Double avgDelta = null;
Double bestDelta = null;
Double worstDelta = null;
if(r.instance.bestKnownResult != null){
avgDelta = (r.getResultStats().getMean() / r.instance.bestKnownResult - 1) * 100;
bestDelta = (r.getResultStats().getMin() / r.instance.bestKnownResult - 1) * 100;
worstDelta = (r.getResultStats().getMax() / r.instance.bestKnownResult - 1) * 100;
}
System.out.println("("+count+"/"+benchmarkInstances.size() +")"+ "\t[instance="+r.instance.name+
"][avgTime="+round(r.getTimesStats().getMean(),2)+"]" +
"[Result=" + getString(r.getResultStats()) + "]" +
"[Vehicles=" + getString(r.getVehicleStats()) + "]" +
"[Delta[%]=" + getString(bestDelta,avgDelta,worstDelta) + "]");
}
private String getString(Double bestDelta, Double avgDelta,Double worstDelta) {
return "[best="+round(bestDelta,2)+"][avg="+round(avgDelta,2)+"][worst="+round(worstDelta,2)+"]";
}
private String getString(DescriptiveStatistics stats){
return "[best="+round(stats.getMin(),2)+"][avg="+round(stats.getMean(),2)+"][worst="+round(stats.getMax(),2)+"][stdDev=" + round(stats.getStandardDeviation(),2)+"]";
}
private Double round(Double value, int i) {

2
jsprit-analysis/src/main/java/util/BenchmarkWriter.java
View file

@ -23,8 +23,6 @@ package util;
import java.util.Collection;
import analysis.ConcurrentBenchmarker.BenchmarkResult;
public interface BenchmarkWriter {
public void write(Collection<BenchmarkResult> results);
}

183
jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java
View file

@ -27,7 +27,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import analysis.ConcurrentBenchmarker.BenchmarkResult;
import org.jfree.chart.renderer.xy.DeviationRenderer;
public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
@ -45,34 +45,170 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
writer.write(openTable() + newline());
//table head
writer.write(openRow() + newline());
writer.write(head("instance") + newline());
writer.write(head("compTime [sec]") + newline());
writer.write(head("result") + newline());
writer.write(head("&Delta; bestKnown [in %]") + newline());
writer.write(head("inst") + newline());
writer.write(head("runs") + newline());
writer.write(head("&Oslash; time [sec]") + newline());
writer.write(head("results",4));
writer.write(head("vehicles",4));
writer.write(head("res*") + newline());
writer.write(head("veh*") + newline());
writer.write(closeRow() + newline());
writer.write(openRow() + newline());
writer.write(head("") + newline());
writer.write(head("") + newline());
writer.write(head("") + newline());
writer.write(head("best") + newline());
writer.write(head("avg") + newline());
writer.write(head("worst") + newline());
writer.write(head("stdev") + newline());
writer.write(head("best") + newline());
writer.write(head("avg") + newline());
writer.write(head("worst") + newline());
writer.write(head("stdev") + newline());
writer.write(head("") + newline());
writer.write(head("") + newline());
writer.write(closeRow() + newline());
//data
double sum_time = 0.0;
double sum_result = 0.0;
double sum_delta = 0.0;
double sum_avg_time = 0.0;
double sum_best_result = 0.0;
double sum_avg_result = 0.0;
double sum_worst_result = 0.0;
double sum_dev_result = 0.0;
double sum_best_veh = 0.0;
double sum_avg_veh = 0.0;
double sum_worst_veh = 0.0;
double sum_dev_veh = 0.0;
Integer runs = null;
Double sum_res_star=null;
Double sum_veh_star=null;
for(BenchmarkResult result : results){
sum_time+=result.time;
sum_result+=result.result;
sum_delta+=result.delta;
if(runs==null) runs=result.runs;
writer.write(openRow() + newline());
writer.write(date(result.instance.name) + newline());
writer.write(date(Double.valueOf(round(result.time,2)).toString()) + newline());
writer.write(date(Double.valueOf(round(result.result,2)).toString()) + newline());
writer.write(date(Double.valueOf(round(result.delta*100.0,2)).toString()) + newline());
writer.write(date(Integer.valueOf(result.runs).toString()) + newline());
Double avg_time = round(result.getTimesStats().getMean(),2);
writer.write(date(Double.valueOf(avg_time).toString()) + newline());
//bestRes
Double best_result = round(result.getResultStats().getMin(),2);
writer.write(date(Double.valueOf(best_result).toString()) + newline());
//avgRes
Double avg_result = round(result.getResultStats().getMean(),2);
writer.write(date(Double.valueOf(avg_result).toString()) + newline());
//worstRes
Double worst_result = round(result.getResultStats().getMax(),2);
writer.write(date(Double.valueOf(worst_result).toString()) + newline());
//stdevRes
Double std_result = round(result.getResultStats().getStandardDeviation(),2);
writer.write(date(Double.valueOf(std_result).toString()) + newline());
//bestVeh
Double best_vehicle = round(result.getVehicleStats().getMin(),2);
writer.write(date(Double.valueOf(best_vehicle).toString()) + newline());
//avgVeh
Double avg_vehicle = round(result.getVehicleStats().getMean(),2);
writer.write(date(Double.valueOf(avg_vehicle).toString()) + newline());
//worstVeh
Double worst_vehicle = round(result.getVehicleStats().getMax(),2);
writer.write(date(Double.valueOf(worst_vehicle).toString()) + newline());
//stdevVeh
Double std_vehicle = round(result.getVehicleStats().getStandardDeviation(),2);
writer.write(date(Double.valueOf(std_vehicle).toString()) + newline());
//bestKnownRes
writer.write(date("" + result.instance.bestKnownResult + newline()));
//bestKnownVeh
writer.write(date("" + result.instance.bestKnownVehicles + newline()));
writer.write(closeRow() + newline());
sum_avg_time+=avg_time;
sum_best_result+=best_result;
sum_avg_result+=avg_result;
sum_worst_result+=worst_result;
sum_dev_result+=std_result;
sum_best_veh+=best_vehicle;
sum_avg_veh+=avg_vehicle;
sum_worst_veh+=worst_vehicle;
sum_dev_veh+=std_vehicle;
if(result.instance.bestKnownResult != null){
if(sum_res_star==null) sum_res_star=result.instance.bestKnownResult;
else sum_res_star+=result.instance.bestKnownResult;
}
if(result.instance.bestKnownVehicles != null){
if(sum_veh_star==null) sum_veh_star=result.instance.bestKnownVehicles;
else sum_veh_star+=result.instance.bestKnownVehicles;
}
}
writer.write(openRow() + newline());
writer.write(date("avg") + newline());
writer.write(date(Double.valueOf(round(sum_time/(double)results.size(),2)).toString()) + newline());
writer.write(date(Double.valueOf(round(sum_result/(double)results.size(),2)).toString()) + newline());
writer.write(date(Double.valueOf(round(sum_delta/(double)results.size()*100.0,2)).toString()) + newline());
writer.write(date("&Oslash;") + newline());
writer.write(date(""+runs) + newline());
Double average_time = round(sum_avg_time/(double)results.size(),2);
writer.write(date(Double.valueOf(average_time).toString()) + newline());
//bestRes
writer.write(date(Double.valueOf(round(sum_best_result/(double)results.size(),2)).toString()) + newline());
//avgRes
Double average_result = round(sum_avg_result/(double)results.size(),2);
writer.write(date(Double.valueOf(average_result).toString()) + newline());
//worstRes
writer.write(date(Double.valueOf(round(sum_worst_result/(double)results.size(),2)).toString()) + newline());
//stdevRes
writer.write(date(Double.valueOf(round(sum_dev_result/(double)results.size(),2)).toString()) + newline());
//bestVeh
writer.write(date(Double.valueOf(round(sum_best_veh/(double)results.size(),2)).toString()) + newline());
//avgVeh
Double average_vehicles = round(sum_avg_veh/(double)results.size(),2);
writer.write(date(Double.valueOf(average_vehicles).toString()) + newline());
//worstVeh
writer.write(date(Double.valueOf(round(sum_worst_veh/(double)results.size(),2)).toString()) + newline());
//stdevVeh
writer.write(date(Double.valueOf(round(sum_dev_veh/(double)results.size(),2)).toString()) + newline());
//bestKnownRes
Double delta_res = null;
if(sum_res_star != null){
writer.write(date(Double.valueOf(round(sum_res_star.doubleValue()/(double)results.size(),2)).toString()) + newline());
delta_res = (sum_avg_result/sum_res_star - 1)*100;
}
else writer.write(date("null") + newline());
//bestKnownVeh
Double delta_veh = null;
if(sum_veh_star != null){
writer.write(date(Double.valueOf(round(sum_veh_star.doubleValue()/(double)results.size(),2)).toString()) + newline());
delta_veh = (sum_avg_veh - sum_veh_star)/(double)results.size();
}
else writer.write(date("null") + newline());
writer.write(closeRow() + newline());
writer.write(closeTable() + newline());
writer.write("avg. percentage deviation to best-known result: " + round(delta_res,2) + newline() + newline());
writer.write("avg. absolute deviation to best-known vehicles: " + round(delta_veh,2) + newline());
writer.write(openTable() + newline());
writer.write(openRow() + newline());
writer.write(date("") + newline());
writer.write(date("") + newline());
writer.write(date("") + newline());
writer.write(date("") + newline());
writer.write(date(Double.valueOf(average_time).toString(),"align=\"right\"") + newline());
writer.write(date(Double.valueOf(average_result).toString(),"align=\"right\"") + newline());
writer.write(date(Double.valueOf(average_vehicles).toString(),"align=\"right\"") + newline());
if(delta_res != null){
writer.write(date(Double.valueOf(round(delta_res,2)).toString(),"align=\"right\"") + newline());
}else writer.write(date("n.a.") + newline());
if(delta_veh != null){
writer.write(date(Double.valueOf(round(delta_veh,2)).toString(),"align=\"right\"") + newline());
}else writer.write(date("n.a.") + newline());
writer.write(closeRow() + newline());
writer.write(closeTable() + newline());
writer.close();
} catch (IOException e) {
// TODO Auto-generated catch block
@ -81,7 +217,12 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
}
private double round(Double value, int i) {
private String head(String string, int i) {
return "<th colspan=\""+i+"\">"+string+"</th>";
}
private Double round(Double value, int i) {
if(value==null) return null;
long roundedVal = Math.round(value*Math.pow(10, i));
return (double)roundedVal/(double)(Math.pow(10, i));
}
@ -105,6 +246,10 @@ public class HtmlBenchmarkTableWriter implements BenchmarkWriter{
private String date(String date) {
return "<td>"+date+"</td>";
}
private String date(String date, String metaData) {
return "<td " + metaData + ">"+date+"</td>";
}
private String newline() {
return "\n";