changeset 1356:f066ab6230cf

7901810: Group thread distribution is unstable between iterations
author shade
date Wed, 28 Sep 2016 21:57:04 +0200
parents e810ce09937a
children b35ea744ed81
files jmh-core-it/src/test/java/org/openjdk/jmh/it/threadparams/GroupThreadParamsStabilityTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threadparams/ThreadParamsStabilityTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupThreadStateHelperTimesTest.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java jmh-core/src/test/java/org/openjdk/jmh/runner/DistributeGroupsTest.java
diffstat 5 files changed, 496 insertions(+), 188 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threadparams/GroupThreadParamsStabilityTest.java	Wed Sep 28 21:57:04 2016 +0200
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.threadparams;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.All)
+@Warmup(iterations = 5,  time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(1)
+public class GroupThreadParamsStabilityTest {
+
+    @State(Scope.Thread)
+    public static class Data {
+        ThreadParams last;
+
+        void test(ThreadParams params) {
+            if (last == null) {
+                last = params;
+            } else {
+                // simple equality suffices
+                Assert.assertEquals(last, params);
+            }
+        }
+    }
+
+    @Group("T")
+    @Benchmark
+    public void test1(ThreadParams tp, Data d) {
+        d.test(tp);
+    }
+
+    @Group("T")
+    @Benchmark
+    public void test2(ThreadParams tp, Data d) {
+        d.test(tp);
+    }
+
+    @Test
+    public void test() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .shouldFailOnError(true)
+                    .build();
+            new Runner(opt).run();
+        }
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threadparams/ThreadParamsStabilityTest.java	Wed Sep 28 21:57:04 2016 +0200
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.threadparams;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.All)
+@Warmup(iterations = 5,  time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(1)
+public class ThreadParamsStabilityTest {
+
+    @State(Scope.Thread)
+    public static class Data {
+        ThreadParams last;
+
+        void test(ThreadParams params) {
+            if (last == null) {
+                last = params;
+            } else {
+                // simple equality suffices
+                Assert.assertEquals(last, params);
+            }
+        }
+    }
+
+    @Benchmark
+    public void test1(ThreadParams tp, Data d) {
+        d.test(tp);
+    }
+
+    @Test
+    public void test() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .shouldFailOnError(true)
+                    .build();
+            new Runner(opt).run();
+        }
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupThreadStateHelperTimesTest.java	Wed Sep 28 21:57:04 2016 +0200
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Tests if harness executes setup, run, and tearDown in the same workers.
+ */
+@BenchmarkMode(Mode.All)
+@Warmup(iterations = 5,  time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(1)
+@State(Scope.Benchmark)
+public class GroupThreadStateHelperTimesTest {
+
+    @State(Scope.Thread)
+    public static class State1 {
+        static final AtomicInteger setupTimes = new AtomicInteger();
+        static final AtomicInteger tearDownTimes = new AtomicInteger();
+
+        @Setup
+        public void setup() {
+            setupTimes.incrementAndGet();
+        }
+
+        @Setup
+        public void tearDown() {
+            tearDownTimes.incrementAndGet();
+        }
+    }
+
+    @State(Scope.Thread)
+    public static class State2 {
+        static final AtomicInteger setupTimes = new AtomicInteger();
+        static final AtomicInteger tearDownTimes = new AtomicInteger();
+
+        @Setup
+        public void setup() {
+            setupTimes.incrementAndGet();
+        }
+
+        @Setup
+        public void tearDown() {
+            tearDownTimes.incrementAndGet();
+        }
+    }
+
+    @Setup
+    public void reset() {
+        State1.setupTimes.set(0);
+        State1.tearDownTimes.set(0);
+        State2.setupTimes.set(0);
+        State2.tearDownTimes.set(0);
+    }
+
+    @TearDown
+    public void verify() {
+        Assert.assertEquals(1, State1.setupTimes.get());
+        Assert.assertEquals(1, State1.tearDownTimes.get());
+        Assert.assertEquals(1, State2.setupTimes.get());
+        Assert.assertEquals(1, State2.tearDownTimes.get());
+    }
+
+    @Benchmark
+    @Group("T")
+    public void test1(State1 state1) {
+        Fixtures.work();
+    }
+
+    @Benchmark
+    @Group("T")
+    public void test2(State2 state2) {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .shouldFailOnError(true)
+                    .build();
+            new Runner(opt).run();
+        }
+    }
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Tue Sep 20 20:14:30 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Wed Sep 28 21:57:04 2016 +0200
@@ -54,7 +54,7 @@
     private final ExecutorService executor;
 
     // (Aleksey) Forgive me, Father, for I have sinned.
-    private final ThreadLocal<Object> instances;
+    private final ThreadLocal<ThreadData> threadData;
 
     private final OutputFormat out;
     private final List<InternalProfiler> profilers;
@@ -72,11 +72,19 @@
         this.profilersRev = new ArrayList<InternalProfiler>(profilers);
         Collections.reverse(profilersRev);
 
-        this.instances = new ThreadLocal<Object>() {
+        final BlockingQueue<ThreadParams> tps = new ArrayBlockingQueue<ThreadParams>(executionParams.getThreads());
+        tps.addAll(distributeThreads(executionParams.getThreads(), executionParams.getThreadGroups()));
+
+        this.threadData = new ThreadLocal<ThreadData>() {
             @Override
-            protected Object initialValue() {
+            protected ThreadData initialValue() {
                 try {
-                    return clazz.newInstance();
+                    Object o = clazz.newInstance();
+                    ThreadParams t = tps.poll();
+                    if (t == null) {
+                        throw new IllegalStateException("Cannot get another thread params");
+                    }
+                    return new ThreadData(o, t);
                 } catch (InstantiationException e) {
                     throw new RuntimeException("Class " + clazz.getName() + " instantiation error ", e);
                 } catch (IllegalAccessException e) {
@@ -84,6 +92,7 @@
                 }
             }
         };
+
         this.out = out;
         try {
             this.executor = EXECUTOR_TYPE.createExecutor(executionParams.getThreads(), executionParams.getBenchmark());
@@ -92,8 +101,8 @@
         }
     }
 
-    static ThreadParams[] distributeThreads(int threads, int[] groups) {
-        ThreadParams[] result = new ThreadParams[threads];
+    static List<ThreadParams> distributeThreads(int threads, int[] groups) {
+        List<ThreadParams> result = new ArrayList<ThreadParams>();
         int totalGroupThreads = Utils.sum(groups);
         int totalGroups = (int) Math.ceil(1D * threads / totalGroupThreads);
         int totalSubgroups = groups.length;
@@ -113,13 +122,14 @@
                 currentSubgroupThread = 0;
             }
 
-            result[t] = new ThreadParams(
+            result.add(new ThreadParams(
                     t, threads,
                     currentGroup, totalGroups,
                     currentSubgroup, totalSubgroups,
                     currentGroupThread, totalGroupThreads,
                     currentSubgroupThread, groups[currentSubgroup]
-                  );
+                  )
+            );
 
             currentGroupThread++;
             currentSubgroupThread++;
@@ -318,10 +328,8 @@
 
         // preparing the worker runnables
         BenchmarkTask[] runners = new BenchmarkTask[numThreads];
-
-        ThreadParams[] threadParamses = distributeThreads(numThreads, benchmarkParams.getThreadGroups());
         for (int i = 0; i < runners.length; i++) {
-            runners[i] = new BenchmarkTask(control, threadParamses[i]);
+            runners[i] = new BenchmarkTask(control);
         }
 
         long waitDeadline = System.nanoTime() + benchmarkParams.getTimeout().convertTo(TimeUnit.NANOSECONDS);
@@ -424,14 +432,11 @@
      * Worker body.
      */
     class BenchmarkTask implements Callable<BenchmarkTaskResult> {
-
         private volatile Thread runner;
         private final InfraControl control;
-        private final ThreadParams threadParams;
 
-        BenchmarkTask(InfraControl control, ThreadParams threadParams) {
+        BenchmarkTask(InfraControl control) {
             this.control = control;
-            this.threadParams = threadParams;
         }
 
         @Override
@@ -441,7 +446,8 @@
                 runner = Thread.currentThread();
 
                 // go for the run
-                return (BenchmarkTaskResult) method.invoke(instances.get(), control, threadParams);
+                ThreadData td = threadData.get();
+                return (BenchmarkTaskResult) method.invoke(td.instance, control, td.params);
             } catch (Throwable e) {
                 // about to fail the iteration;
                 // compensate for missed sync-iteration latches, we don't care about that anymore
@@ -470,4 +476,27 @@
         }
     }
 
+    /**
+     * Handles thread-local data for each worker that should not change
+     * between the iterations.
+     */
+    private static class ThreadData {
+        /**
+         * Synthetic benchmark instance, which holds the benchmark metadata.
+         * Expected to be touched by a single thread only.
+         */
+        final Object instance;
+
+        /**
+         * Thread parameters. Among other things, holds the thread's place
+         * in group distribution, and thus should be the same for a given thread.
+         */
+        final ThreadParams params;
+
+        public ThreadData(Object instance, ThreadParams params) {
+            this.instance = instance;
+            this.params = params;
+        }
+    }
+
 }
--- a/jmh-core/src/test/java/org/openjdk/jmh/runner/DistributeGroupsTest.java	Tue Sep 20 20:14:30 2016 +0200
+++ b/jmh-core/src/test/java/org/openjdk/jmh/runner/DistributeGroupsTest.java	Wed Sep 28 21:57:04 2016 +0200
@@ -28,276 +28,278 @@
 import org.junit.Test;
 import org.openjdk.jmh.infra.ThreadParams;
 
+import java.util.List;
+
 public class DistributeGroupsTest {
 
     @Test
     public void test1() {
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(1, new int[]{1});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(1, new int[]{1});
 
-        Assert.assertEquals(1, controls.length);
+        Assert.assertEquals(1, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(1, controls[0].getThreadCount());
-        Assert.assertEquals(0, controls[0].getThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getThreadCount());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
 
         // one group with a single thread
-        Assert.assertEquals(1, controls[0].getGroupCount());
-        Assert.assertEquals(0, controls[0].getGroupIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupCount());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
 
         // the thread is alone in the group
-        Assert.assertEquals(1, controls[0].getGroupThreadCount());
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
 
         // no subgroups in the group
-        Assert.assertEquals(1, controls[0].getSubgroupCount());
-        Assert.assertEquals(0, controls[0].getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupIndex());
 
         // the thread is alone in the subgroup
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[0].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupThreadIndex());
     }
 
     @Test
     public void test2() {
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(2, new int[]{1});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(2, new int[]{1});
 
-        Assert.assertEquals(2, controls.length);
+        Assert.assertEquals(2, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(2, controls[0].getThreadCount());
-        Assert.assertEquals(2, controls[1].getThreadCount());
-        Assert.assertEquals(0, controls[0].getThreadIndex());
-        Assert.assertEquals(1, controls[1].getThreadIndex());
+        Assert.assertEquals(2, controls.get(0).getThreadCount());
+        Assert.assertEquals(2, controls.get(1).getThreadCount());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getThreadIndex());
 
         // two groups, each for a single thread
-        Assert.assertEquals(2, controls[0].getGroupCount());
-        Assert.assertEquals(2, controls[1].getGroupCount());
-        Assert.assertEquals(0, controls[0].getGroupIndex());
-        Assert.assertEquals(1, controls[1].getGroupIndex());
+        Assert.assertEquals(2, controls.get(0).getGroupCount());
+        Assert.assertEquals(2, controls.get(1).getGroupCount());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
+        Assert.assertEquals(1, controls.get(1).getGroupIndex());
 
         // each thread is alone in the group
-        Assert.assertEquals(1, controls[0].getGroupThreadCount());
-        Assert.assertEquals(1, controls[1].getGroupThreadCount());
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(1, controls.get(1).getGroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getGroupThreadIndex());
 
         // no subgroups in the group
-        Assert.assertEquals(1, controls[0].getSubgroupCount());
-        Assert.assertEquals(1, controls[1].getSubgroupCount());
-        Assert.assertEquals(0, controls[0].getSubgroupIndex());
-        Assert.assertEquals(0, controls[1].getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(1, controls.get(1).getSubgroupCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupIndex());
+        Assert.assertEquals(0, controls.get(1).getSubgroupIndex());
 
         // each thread is alone in the subgroup
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(1, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[0].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(1, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getSubgroupThreadIndex());
     }
 
     @Test
     public void test3() {
         // First "subgroup" is ignored
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(2, new int[]{0, 1});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(2, new int[]{0, 1});
 
-        Assert.assertEquals(2, controls.length);
+        Assert.assertEquals(2, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(2, controls[0].getThreadCount());
-        Assert.assertEquals(2, controls[1].getThreadCount());
-        Assert.assertEquals(0, controls[0].getThreadIndex());
-        Assert.assertEquals(1, controls[1].getThreadIndex());
+        Assert.assertEquals(2, controls.get(0).getThreadCount());
+        Assert.assertEquals(2, controls.get(1).getThreadCount());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getThreadIndex());
 
         // two groups, each for a single thread
-        Assert.assertEquals(2, controls[0].getGroupCount());
-        Assert.assertEquals(2, controls[1].getGroupCount());
-        Assert.assertEquals(0, controls[0].getGroupIndex());
-        Assert.assertEquals(1, controls[1].getGroupIndex());
+        Assert.assertEquals(2, controls.get(0).getGroupCount());
+        Assert.assertEquals(2, controls.get(1).getGroupCount());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
+        Assert.assertEquals(1, controls.get(1).getGroupIndex());
 
         // each thread is alone in the group
-        Assert.assertEquals(1, controls[0].getGroupThreadCount());
-        Assert.assertEquals(1, controls[1].getGroupThreadCount());
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(1, controls.get(1).getGroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getGroupThreadIndex());
 
         // two subgroups, but first is not populated
-        Assert.assertEquals(2, controls[0].getSubgroupCount());
-        Assert.assertEquals(2, controls[1].getSubgroupCount());
-        Assert.assertEquals(1, controls[0].getSubgroupIndex());
-        Assert.assertEquals(1, controls[1].getSubgroupIndex());
+        Assert.assertEquals(2, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupCount());
+        Assert.assertEquals(1, controls.get(0).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(1).getSubgroupIndex());
 
         // each thread is alone in the subgroup
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(1, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[0].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(1, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getSubgroupThreadIndex());
     }
 
     @Test
     public void test4() {
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(2, new int[]{1, 1});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(2, new int[]{1, 1});
 
-        Assert.assertEquals(2, controls.length);
+        Assert.assertEquals(2, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(2, controls[0].getThreadCount());
-        Assert.assertEquals(2, controls[1].getThreadCount());
-        Assert.assertEquals(0, controls[0].getThreadIndex());
-        Assert.assertEquals(1, controls[1].getThreadIndex());
+        Assert.assertEquals(2, controls.get(0).getThreadCount());
+        Assert.assertEquals(2, controls.get(1).getThreadCount());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getThreadIndex());
 
         // one group only
-        Assert.assertEquals(1, controls[0].getGroupCount());
-        Assert.assertEquals(1, controls[1].getGroupCount());
-        Assert.assertEquals(0, controls[0].getGroupIndex());
-        Assert.assertEquals(0, controls[1].getGroupIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupCount());
+        Assert.assertEquals(1, controls.get(1).getGroupCount());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
+        Assert.assertEquals(0, controls.get(1).getGroupIndex());
 
         // both threads share the group
-        Assert.assertEquals(2, controls[0].getGroupThreadCount());
-        Assert.assertEquals(2, controls[1].getGroupThreadCount());
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
-        Assert.assertEquals(1, controls[1].getGroupThreadIndex());
+        Assert.assertEquals(2, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(2, controls.get(1).getGroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getGroupThreadIndex());
 
         // two subgroups, threads in different subgroups
-        Assert.assertEquals(2, controls[0].getSubgroupCount());
-        Assert.assertEquals(2, controls[1].getSubgroupCount());
-        Assert.assertEquals(0, controls[0].getSubgroupIndex());
-        Assert.assertEquals(1, controls[1].getSubgroupIndex());
+        Assert.assertEquals(2, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(1).getSubgroupIndex());
 
         // each subgroup has a single thread
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(1, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[0].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(1, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getSubgroupThreadIndex());
 
     }
 
     @Test
     public void test5() {
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(3, new int[]{1, 2});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(3, new int[]{1, 2});
 
-        Assert.assertEquals(3, controls.length);
+        Assert.assertEquals(3, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(3, controls[0].getThreadCount());
-        Assert.assertEquals(3, controls[1].getThreadCount());
-        Assert.assertEquals(3, controls[2].getThreadCount());
-        Assert.assertEquals(0, controls[0].getThreadIndex());
-        Assert.assertEquals(1, controls[1].getThreadIndex());
-        Assert.assertEquals(2, controls[2].getThreadIndex());
+        Assert.assertEquals(3, controls.get(0).getThreadCount());
+        Assert.assertEquals(3, controls.get(1).getThreadCount());
+        Assert.assertEquals(3, controls.get(2).getThreadCount());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getThreadIndex());
+        Assert.assertEquals(2, controls.get(2).getThreadIndex());
 
         // one group only
-        Assert.assertEquals(1, controls[0].getGroupCount());
-        Assert.assertEquals(1, controls[1].getGroupCount());
-        Assert.assertEquals(1, controls[2].getGroupCount());
-        Assert.assertEquals(0, controls[0].getGroupIndex());
-        Assert.assertEquals(0, controls[1].getGroupIndex());
-        Assert.assertEquals(0, controls[2].getGroupIndex());
+        Assert.assertEquals(1, controls.get(0).getGroupCount());
+        Assert.assertEquals(1, controls.get(1).getGroupCount());
+        Assert.assertEquals(1, controls.get(2).getGroupCount());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
+        Assert.assertEquals(0, controls.get(1).getGroupIndex());
+        Assert.assertEquals(0, controls.get(2).getGroupIndex());
 
         // all threads share the group
-        Assert.assertEquals(3, controls[0].getGroupThreadCount());
-        Assert.assertEquals(3, controls[1].getGroupThreadCount());
-        Assert.assertEquals(3, controls[2].getGroupThreadCount());
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
-        Assert.assertEquals(1, controls[1].getGroupThreadIndex());
-        Assert.assertEquals(2, controls[2].getGroupThreadIndex());
+        Assert.assertEquals(3, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(1).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(2).getGroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getGroupThreadIndex());
+        Assert.assertEquals(2, controls.get(2).getGroupThreadIndex());
 
         // two subgroups, first thread in distinct subgroup
-        Assert.assertEquals(2, controls[0].getSubgroupCount());
-        Assert.assertEquals(2, controls[1].getSubgroupCount());
-        Assert.assertEquals(2, controls[2].getSubgroupCount());
-        Assert.assertEquals(0, controls[0].getSubgroupIndex());
-        Assert.assertEquals(1, controls[1].getSubgroupIndex());
-        Assert.assertEquals(1, controls[2].getSubgroupIndex());
+        Assert.assertEquals(2, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(2).getSubgroupCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(1).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(2).getSubgroupIndex());
 
         // first subgroup has a single thread, second has two threads
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[2].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[0].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[1].getSubgroupThreadIndex());
-        Assert.assertEquals(1, controls[2].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(2).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(0).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(1).getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(2).getSubgroupThreadIndex());
 
     }
 
     @Test
     public void test6() {
-        ThreadParams[] controls = BenchmarkHandler.distributeThreads(6, new int[]{1, 2});
+        List<ThreadParams> controls = BenchmarkHandler.distributeThreads(6, new int[]{1, 2});
 
-        Assert.assertEquals(6, controls.length);
+        Assert.assertEquals(6, controls.size());
 
         // threads agree on thread count, and enumerated
-        Assert.assertEquals(6, controls[0].getThreadCount());
-        Assert.assertEquals(6, controls[1].getThreadCount());
-        Assert.assertEquals(6, controls[2].getThreadCount());
-        Assert.assertEquals(6, controls[3].getThreadCount());
-        Assert.assertEquals(6, controls[4].getThreadCount());
-        Assert.assertEquals(6, controls[5].getThreadCount());
+        Assert.assertEquals(6, controls.get(0).getThreadCount());
+        Assert.assertEquals(6, controls.get(1).getThreadCount());
+        Assert.assertEquals(6, controls.get(2).getThreadCount());
+        Assert.assertEquals(6, controls.get(3).getThreadCount());
+        Assert.assertEquals(6, controls.get(4).getThreadCount());
+        Assert.assertEquals(6, controls.get(5).getThreadCount());
 
-        Assert.assertEquals(0, controls[0].getThreadIndex());
-        Assert.assertEquals(1, controls[1].getThreadIndex());
-        Assert.assertEquals(2, controls[2].getThreadIndex());
-        Assert.assertEquals(3, controls[3].getThreadIndex());
-        Assert.assertEquals(4, controls[4].getThreadIndex());
-        Assert.assertEquals(5, controls[5].getThreadIndex());
+        Assert.assertEquals(0, controls.get(0).getThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getThreadIndex());
+        Assert.assertEquals(2, controls.get(2).getThreadIndex());
+        Assert.assertEquals(3, controls.get(3).getThreadIndex());
+        Assert.assertEquals(4, controls.get(4).getThreadIndex());
+        Assert.assertEquals(5, controls.get(5).getThreadIndex());
 
         // two groups
-        Assert.assertEquals(2, controls[0].getGroupCount());
-        Assert.assertEquals(2, controls[1].getGroupCount());
-        Assert.assertEquals(2, controls[2].getGroupCount());
-        Assert.assertEquals(2, controls[3].getGroupCount());
-        Assert.assertEquals(2, controls[4].getGroupCount());
-        Assert.assertEquals(2, controls[5].getGroupCount());
+        Assert.assertEquals(2, controls.get(0).getGroupCount());
+        Assert.assertEquals(2, controls.get(1).getGroupCount());
+        Assert.assertEquals(2, controls.get(2).getGroupCount());
+        Assert.assertEquals(2, controls.get(3).getGroupCount());
+        Assert.assertEquals(2, controls.get(4).getGroupCount());
+        Assert.assertEquals(2, controls.get(5).getGroupCount());
 
-        Assert.assertEquals(0, controls[0].getGroupIndex());
-        Assert.assertEquals(0, controls[1].getGroupIndex());
-        Assert.assertEquals(0, controls[2].getGroupIndex());
-        Assert.assertEquals(1, controls[3].getGroupIndex());
-        Assert.assertEquals(1, controls[4].getGroupIndex());
-        Assert.assertEquals(1, controls[5].getGroupIndex());
+        Assert.assertEquals(0, controls.get(0).getGroupIndex());
+        Assert.assertEquals(0, controls.get(1).getGroupIndex());
+        Assert.assertEquals(0, controls.get(2).getGroupIndex());
+        Assert.assertEquals(1, controls.get(3).getGroupIndex());
+        Assert.assertEquals(1, controls.get(4).getGroupIndex());
+        Assert.assertEquals(1, controls.get(5).getGroupIndex());
 
         // two groups, three threads each
-        Assert.assertEquals(3, controls[0].getGroupThreadCount());
-        Assert.assertEquals(3, controls[1].getGroupThreadCount());
-        Assert.assertEquals(3, controls[2].getGroupThreadCount());
-        Assert.assertEquals(3, controls[3].getGroupThreadCount());
-        Assert.assertEquals(3, controls[4].getGroupThreadCount());
-        Assert.assertEquals(3, controls[5].getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(0).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(1).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(2).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(3).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(4).getGroupThreadCount());
+        Assert.assertEquals(3, controls.get(5).getGroupThreadCount());
 
-        Assert.assertEquals(0, controls[0].getGroupThreadIndex());
-        Assert.assertEquals(1, controls[1].getGroupThreadIndex());
-        Assert.assertEquals(2, controls[2].getGroupThreadIndex());
-        Assert.assertEquals(0, controls[3].getGroupThreadIndex());
-        Assert.assertEquals(1, controls[4].getGroupThreadIndex());
-        Assert.assertEquals(2, controls[5].getGroupThreadIndex());
+        Assert.assertEquals(0, controls.get(0).getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(1).getGroupThreadIndex());
+        Assert.assertEquals(2, controls.get(2).getGroupThreadIndex());
+        Assert.assertEquals(0, controls.get(3).getGroupThreadIndex());
+        Assert.assertEquals(1, controls.get(4).getGroupThreadIndex());
+        Assert.assertEquals(2, controls.get(5).getGroupThreadIndex());
 
         // two subgroups: first subgroup has a single thread, second has two threads
-        Assert.assertEquals(2, controls[0].getSubgroupCount());
-        Assert.assertEquals(2, controls[1].getSubgroupCount());
-        Assert.assertEquals(2, controls[2].getSubgroupCount());
-        Assert.assertEquals(2, controls[3].getSubgroupCount());
-        Assert.assertEquals(2, controls[4].getSubgroupCount());
-        Assert.assertEquals(2, controls[5].getSubgroupCount());
+        Assert.assertEquals(2, controls.get(0).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(2).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(3).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(4).getSubgroupCount());
+        Assert.assertEquals(2, controls.get(5).getSubgroupCount());
 
-        Assert.assertEquals(0, controls[0].getSubgroupIndex());
-        Assert.assertEquals(1, controls[1].getSubgroupIndex());
-        Assert.assertEquals(1, controls[2].getSubgroupIndex());
-        Assert.assertEquals(0, controls[3].getSubgroupIndex());
-        Assert.assertEquals(1, controls[4].getSubgroupIndex());
-        Assert.assertEquals(1, controls[5].getSubgroupIndex());
+        Assert.assertEquals(0, controls.get(0).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(1).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(2).getSubgroupIndex());
+        Assert.assertEquals(0, controls.get(3).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(4).getSubgroupIndex());
+        Assert.assertEquals(1, controls.get(5).getSubgroupIndex());
 
         // first subgroup has a single thread, second has two threads
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[2].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[3].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[4].getSubgroupThreadIndex());
-        Assert.assertEquals(1, controls[5].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(2).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(3).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(4).getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(5).getSubgroupThreadIndex());
 
-        Assert.assertEquals(1, controls[0].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[1].getSubgroupThreadCount());
-        Assert.assertEquals(2, controls[2].getSubgroupThreadCount());
-        Assert.assertEquals(0, controls[3].getSubgroupThreadIndex());
-        Assert.assertEquals(0, controls[4].getSubgroupThreadIndex());
-        Assert.assertEquals(1, controls[5].getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(0).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(1).getSubgroupThreadCount());
+        Assert.assertEquals(2, controls.get(2).getSubgroupThreadCount());
+        Assert.assertEquals(0, controls.get(3).getSubgroupThreadIndex());
+        Assert.assertEquals(0, controls.get(4).getSubgroupThreadIndex());
+        Assert.assertEquals(1, controls.get(5).getSubgroupThreadIndex());
 
     }