From fa8d97b7e9e340cde8b61c84d8f5bafd8d2580bc Mon Sep 17 00:00:00 2001 From: oblonski <4sschroeder@gmail.com> Date: Mon, 1 Jul 2013 02:49:04 +0200 Subject: [PATCH] improve benchmarking --- .../java/analysis/ConcurrentBenchmarker.java | 99 +++++++++++++------ .../src/main/java/util/BenchmarkWriter.java | 9 ++ .../java/util/HtmlBenchmarkTableWriter.java | 98 ++++++++++++++++++ 3 files changed, 174 insertions(+), 32 deletions(-) create mode 100644 jsprit-analysis/src/main/java/util/BenchmarkWriter.java create mode 100644 jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java diff --git a/jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java b/jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java index 4ea3fbcd..bcaa0380 100644 --- a/jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java +++ b/jsprit-analysis/src/main/java/analysis/ConcurrentBenchmarker.java @@ -4,13 +4,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import util.BenchmarkWriter; import util.Solutions; import algorithms.VehicleRoutingAlgorithms; import basics.VehicleRoutingAlgorithm; @@ -19,12 +21,12 @@ import basics.VehicleRoutingProblemSolution; import basics.algo.VehicleRoutingAlgorithmListeners.Priority; public class ConcurrentBenchmarker { - - static class Problem { + + public static class BenchmarkInstance { public final String name; public final VehicleRoutingProblem vrp; public final Double bestKnown; - public Problem(String name, VehicleRoutingProblem vrp, Double bestKnown) { + public BenchmarkInstance(String name, VehicleRoutingProblem vrp, Double bestKnown) { super(); this.name = name; this.vrp = vrp; @@ -32,16 +34,16 @@ public class ConcurrentBenchmarker { } } - static class Result { + public static class BenchmarkResult { public final double result; public final double time; - public final Problem problem; + public final BenchmarkInstance instance; public Double delta = null; - public Result(Problem p, double result, double time) { + public BenchmarkResult(BenchmarkInstance p, double result, double time) { super(); this.result = result; this.time = time; - this.problem = p; + this.instance = p; } void setBestKnownDelta(double delta){ this.delta = delta; @@ -50,39 +52,62 @@ public class ConcurrentBenchmarker { private String algorithmConfig; - private List problems = new ArrayList(); + private List problems = new ArrayList(); + + private int runs = 1; + + private Collection writers = new ArrayList(); + + private Collection results = new ArrayList(); public ConcurrentBenchmarker(String algorithmConfig) { super(); this.algorithmConfig = algorithmConfig; + Logger.getRootLogger().setLevel(Level.ERROR); + } + + public void addBenchmarkWriter(BenchmarkWriter writer){ + writers.add(writer); } public void addProblem(String name, VehicleRoutingProblem problem){ - problems.add(new Problem(name,problem,null)); + problems.add(new BenchmarkInstance(name,problem,null)); } public void addProblem(String name, VehicleRoutingProblem problem, double bestKnown){ - problems.add(new Problem(name,problem,bestKnown)); + problems.add(new BenchmarkInstance(name,problem,bestKnown)); + } + + public void setNuOfRuns(int runs){ + this.runs = runs; } public void run(){ + System.out.println("start benchmarking [nuOfInstances=" + problems.size() + "]"); + double startTime = System.currentTimeMillis(); ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()+1); - List> futures = new ArrayList>(); - List results = new ArrayList(); - for(final Problem p : problems){ - Future futureResult = executor.submit(new Callable(){ + List> futures = new ArrayList>(); +// List results = new ArrayList(); + for(final BenchmarkInstance p : problems){ + for(int run=0;run futureResult = executor.submit(new Callable(){ - @Override - public Result call() throws Exception { - return runAlgoAndGetResult(p); - } - - }); - futures.add(futureResult); + @Override + public BenchmarkResult call() throws Exception { + return runAlgoAndGetResult(p); + } + + }); + futures.add(futureResult); + } } try { - for(Future f : futures){ + int count = 1; + for(Future f : futures){ + BenchmarkResult r = f.get(); + print(r,count); results.add(f.get()); + count++; } } catch (InterruptedException e) { // TODO Auto-generated catch block @@ -93,33 +118,43 @@ public class ConcurrentBenchmarker { } executor.shutdown(); print(results); + System.out.println("done [time="+(System.currentTimeMillis()-startTime)/1000 + "sec]"); } - private Result runAlgoAndGetResult(Problem p) { + private BenchmarkResult runAlgoAndGetResult(BenchmarkInstance p) { VehicleRoutingAlgorithm vra = VehicleRoutingAlgorithms.readAndCreateAlgorithm(p.vrp, algorithmConfig); StopWatch stopwatch = new StopWatch(); vra.getAlgorithmListeners().addListener(stopwatch,Priority.HIGH); Collection solutions = vra.searchSolutions(); VehicleRoutingProblemSolution best = Solutions.getBest(solutions); - Result result = new Result(p,best.getCost(),stopwatch.getCompTimeInSeconds()); + BenchmarkResult result = new BenchmarkResult(p,best.getCost(),stopwatch.getCompTimeInSeconds()); if(p.bestKnown != null) result.setBestKnownDelta((best.getCost()/p.bestKnown-1)); return result; } - private void print(List results) { - System.out.println("instance,time [in sec],result,delta [in percent to bestKnown]"); + private void print(Collection results) { double sumTime=0.0; double sumResult=0.0; - for(Result r : results){ + for(BenchmarkResult r : results){ sumTime+=r.time; sumResult+=r.result; - System.out.println("[instance="+r.problem.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+"][delta="+round(r.delta,3)+"]"); +// print(r); } System.out.println("[avgTime="+round(sumTime/(double)results.size(),2)+"][avgResult="+round(sumResult/(double)results.size(),2)+"]"); + for(BenchmarkWriter writer : writers){ + writer.write(results); + } } - private double round(Double delta, int i) { - long roundedVal = Math.round(delta*Math.pow(10, i)); + private void print(BenchmarkResult r, int count) { + System.out.println("("+count+"/"+problems.size() +")"+ "\t[instance="+r.instance.name+"][time="+round(r.time,2)+"][result="+round(r.result,2)+"][delta="+round(r.delta,3)+"]"); +// for(BenchmarkWriter writer : writers){ +// writer.write(r); +// } + } + + private double round(Double value, int i) { + long roundedVal = Math.round(value*Math.pow(10, i)); return (double)roundedVal/(double)(Math.pow(10, i)); } diff --git a/jsprit-analysis/src/main/java/util/BenchmarkWriter.java b/jsprit-analysis/src/main/java/util/BenchmarkWriter.java new file mode 100644 index 00000000..6ca10b3d --- /dev/null +++ b/jsprit-analysis/src/main/java/util/BenchmarkWriter.java @@ -0,0 +1,9 @@ +package util; + +import java.util.Collection; + +import analysis.ConcurrentBenchmarker.BenchmarkResult; + +public interface BenchmarkWriter { + public void write(Collection results); +} \ No newline at end of file diff --git a/jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java b/jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java new file mode 100644 index 00000000..b69ac67d --- /dev/null +++ b/jsprit-analysis/src/main/java/util/HtmlBenchmarkTableWriter.java @@ -0,0 +1,98 @@ +package util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collection; + +import analysis.ConcurrentBenchmarker.BenchmarkResult; + +public class HtmlBenchmarkTableWriter implements BenchmarkWriter{ + + private String filename; + + public HtmlBenchmarkTableWriter(String filename) { + this.filename = filename; + } + + @Override + public void write(Collection results) { + + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filename))); + writer.write(openTable() + newline()); + //table head + writer.write(openRow() + newline()); + writer.write(head("instance") + newline()); + writer.write(head("compTime [sec]") + newline()); + writer.write(head("result") + newline()); + writer.write(head("Δ bestKnown [in %]") + newline()); + writer.write(closeRow() + newline()); + //data + double sum_time = 0.0; + double sum_result = 0.0; + double sum_delta = 0.0; + for(BenchmarkResult result : results){ + sum_time+=result.time; + sum_result+=result.result; + sum_delta+=result.delta; + writer.write(openRow() + newline()); + writer.write(date(result.instance.name) + newline()); + writer.write(date(Double.valueOf(round(result.time,2)).toString()) + newline()); + writer.write(date(Double.valueOf(round(result.result,2)).toString()) + newline()); + writer.write(date(Double.valueOf(round(result.delta*100.0,2)).toString()) + newline()); + writer.write(closeRow() + newline()); + } + writer.write(openRow() + newline()); + writer.write(date("avg") + newline()); + writer.write(date(Double.valueOf(round(sum_time/(double)results.size(),2)).toString()) + newline()); + writer.write(date(Double.valueOf(round(sum_result/(double)results.size(),2)).toString()) + newline()); + writer.write(date(Double.valueOf(round(sum_delta/(double)results.size()*100.0,2)).toString()) + newline()); + writer.write(closeRow() + newline()); + + writer.write(closeTable() + newline()); + writer.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + private double round(Double value, int i) { + long roundedVal = Math.round(value*Math.pow(10, i)); + return (double)roundedVal/(double)(Math.pow(10, i)); + } + + private String head(String head) { + return ""+head+""; + } + + private String closeTable() { + return ""; + } + + private String openTable() { + return ""; + } + + private String closeRow() { + return ""; + } + + private String date(String date) { + return ""; + } + + private String newline() { + return "\n"; + } + + private String openRow() { + return ""; + } + + + +}
"+date+"