Report per operation timings in the multi editor and perspective tests

tests/org.eclipse.ui.tests.performance/src/org/eclipse/ui/tests/performance/OpenMultipleEditorTest.java

@@ -55,8 +55,12 @@
 @RunWith(Parameterized.class)
 public class OpenMultipleEditorTest {
 
-	/** Number of files the {@link UIPerformanceTestRule} creates per extension. */
-	private static final int EDITOR_COUNT = 100;
+	/**
+	 * Stays below the REUSE_EDITORS threshold, which defaults to 99. Above it the
+	 * workbench recycles the oldest editor, so further opens would measure a reuse
+	 * instead of an open.
+	 */
+	private static final int EDITOR_COUNT = 90;
 
 	/**
 	 * How many opens at each end of the batch are reported. The ones in between
@@ -134,6 +138,8 @@ private void openAndCloseAll(IWorkbenchPage page, Timings timings) throws PartIn
 				}
 			}
 		}
+		assertEquals("Not all editors stayed open, so an open measured a reuse", EDITOR_COUNT,
+				page.getEditorReferences().length);
 
 		if (closeAll) {
 			long before = System.nanoTime();

tests/org.eclipse.ui.tests.performance/src/org/eclipse/ui/tests/performance/PerspectiveSwitchTest.java

@@ -17,15 +17,19 @@
 import static org.eclipse.ui.tests.harness.util.UITestUtil.processEvents;
 import static org.eclipse.ui.tests.performance.UIPerformanceTestRule.getTestProject;
 import static org.eclipse.ui.tests.performance.UIPerformanceTestUtil.exercise;
+import static org.eclipse.ui.tests.performance.UIPerformanceTestUtil.reportTimings;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.runtime.CoreException;
-import org.eclipse.test.performance.PerformanceTestCaseJunit4;
 import org.eclipse.ui.IPerspectiveDescriptor;
 import org.eclipse.ui.IPerspectiveRegistry;
 import org.eclipse.ui.IWorkbenchPage;
@@ -42,10 +46,23 @@
 import org.junit.runners.Parameterized.Parameters;
 
 /**
- * Test perspective switching.
+ * Measures switching back and forth between two perspectives, and reports each
+ * switch direction separately.
  */
 @RunWith(Parameterized.class)
-public class PerspectiveSwitchTest extends PerformanceTestCaseJunit4 {
+public class PerspectiveSwitchTest {
+
+	/**
+	 * Enough switches to get the participating parts loaded and compiled. Below
+	 * roughly this many, the reported times are dominated by JIT warm-up.
+	 */
+	private static final int WARMUP_PAIRS = 5;
+
+	private static final int MIN_PAIRS = 5;
+
+	private static final int MAX_PAIRS = 50;
+
+	private static final int MAX_MEASURE_TIME_MS = 12000;
 
 	@ClassRule
 	public static final UIPerformanceTestRule uiPerformanceTestRule = new UIPerformanceTestRule();
@@ -80,28 +97,18 @@ public PerspectiveSwitchTest(String id1, String id2, String activeEditor) {
 		this.activeEditor = activeEditor;
 	}
 
-	/**
-	 * Test perspective switching performance.
-	 */
 	@Test
 	public void test() throws CoreException, WorkbenchException {
 		// Get the two perspectives to switch between.
 		final IPerspectiveRegistry registry = WorkbenchPlugin.getDefault().getPerspectiveRegistry();
 		final IPerspectiveDescriptor perspective1 = registry.findPerspectiveWithId(id1);
 		final IPerspectiveDescriptor perspective2 = registry.findPerspectiveWithId(id2);
 
-		// Don't fail if we reference an unknown perspective ID. This can be
-		// a normal occurrance since the test suites reference JDT perspectives, which
-		// might not exist. Just skip the test.
-		if (perspective1 == null) {
-			System.out.println("Unknown perspective ID: " + id1);
-			return;
-		}
-
-		if (perspective2 == null) {
-			System.out.println("Unknown perspective ID: " + id2);
-			return;
-		}
+		// The parameters reference JDT perspectives, which are not part of every target
+		// platform. Skip visibly rather than reporting a pass for something that was
+		// never measured.
+		assumeTrue("Perspective not available: " + id1, perspective1 != null);
+		assumeTrue("Perspective not available: " + id2, perspective2 != null);
 
 		// Open the two perspectives and the file, in a new window.
 		// Do this outside the loop so as not to include
@@ -111,25 +118,43 @@ public void test() throws CoreException, WorkbenchException {
 		assertNotNull(page);
 		page.setPerspective(perspective2);
 
-		// IFile aFile = getProject().getFile("1." +
-		// EditorPerformanceSuite.EDITOR_FILE_EXTENSIONS[0]);
 		IFile aFile = getTestProject().getFile(activeEditor);
-		assertTrue(aFile.exists());
+		assertTrue("Missing test file " + activeEditor + ", the fixture does not create it", aFile.exists());
 
 		IDE.openEditor(page, aFile, true);
 
+		// Class loading and JIT warm-up would otherwise end up in the reported times.
+		for (int i = 0; i < WARMUP_PAIRS; i++) {
+			switchTo(page, perspective1, null);
+			switchTo(page, perspective2, null);
+		}
+		EditorTestHelper.calmDown(500, 30000, 500);
+
+		List<Long> toFirst = new ArrayList<>();
+		List<Long> toSecond = new ArrayList<>();
+
 		exercise(() -> {
-			processEvents();
-
-			startMeasuring();
-			page.setPerspective(perspective1);
-			processEvents();
-			page.setPerspective(perspective2);
-			processEvents();
-			stopMeasuring();
-		});
-
-		commitMeasurements();
-		assertPerformance();
+			switchTo(page, perspective1, toFirst);
+			switchTo(page, perspective2, toSecond);
+		}, MIN_PAIRS, MAX_PAIRS, MAX_MEASURE_TIME_MS);
+
+		reportTimings("PerspectiveSwitch to [" + id1 + "]", toFirst);
+		reportTimings("PerspectiveSwitch to [" + id2 + "]", toSecond);
+	}
+
+	/**
+	 * Switches to the given perspective, recording the time when the given list is
+	 * not {@code null}.
+	 */
+	private static void switchTo(IWorkbenchPage page, IPerspectiveDescriptor perspective, List<Long> times) {
+		long before = System.nanoTime();
+		page.setPerspective(perspective);
+		processEvents();
+		long after = System.nanoTime();
+		assertEquals("Wrong perspective active", perspective, page.getPerspective());
+
+		if (times != null) {
+			times.add(after - before);
+		}
 	}
 }