dart/editor/tools/plugins/com.google.dart.tools.tests.buildbot_test/src/com/google/dart/tools/tests/buildbot/runner/AbstractTestRunner.java

/*
 * Copyright (c) 2013, the Dart project authors.
 * 
 * Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 * 
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.dart.tools.tests.buildbot.runner;

import com.google.dart.tools.core.test.IgnoreLoggedErrors;

import junit.framework.AssertionFailedError;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestResult;
import junit.framework.TestSuite;

import org.junit.Ignore;

import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * An abstract class to execute JUnit tests. Subclasses must implement the five abstract methods.
 * The call order is:
 * <p>
 * testsStarted()
 * <p>
 * testStarted()
 * <p>
 * testPassed() or testFailed()
 * <p>
 * testsFinished()
 * <p>
 */
public abstract class AbstractTestRunner {
  /**
   * A TestCase and elapsed time tuple. This is used to store the elapsed time information for slow
   * tests.
   */
  public static class TestTime implements Comparable<TestTime> {
    private TestCase test;
    private long elapsedTime;

    public TestTime(TestCase test, long elapsedTime) {
      this.test = test;
      this.elapsedTime = elapsedTime;
    }

    @Override
    public int compareTo(TestTime other) {
      if (other.elapsedTime == elapsedTime) {
        return 0;
      } else if (other.elapsedTime > elapsedTime) {
        return 1;
      } else {
        return -1;
      }
    }

    @Override
    public String toString() {
      return formatDouble(elapsedTime / 1000.0) + " sec, " + getTestId(test);
    }
  }

  public static String getTestId(TestCase test) {
    return test.getClass().getName() + "." + test.getName();
  }

  private static String formatDouble(double d) {
    NumberFormat nf = new DecimalFormat();
    nf.setMaximumFractionDigits(2);
    nf.setMinimumFractionDigits(2);
    return nf.format(d);
  }

  private Test mainTest;
  private static final long ONE_SEC_MILLIS = 1000;

  protected static <T extends Annotation> boolean hasAnnotation(TestCase test,
      Class<? extends T> annotationClass) {
    try {
      Method m = test.getClass().getMethod(test.getName());
      T a = m.getAnnotation(annotationClass);
      return a != null;
    } catch (Throwable e) {
    }
    return false;
  }

  private static boolean hasAnnotationIgnored(TestCase test) {
    return hasAnnotation(test, Ignore.class);
  }

  private boolean currentTestLoggedError;

  public AbstractTestRunner(Test test) {
    this.mainTest = test;
  }

  /**
   * Run the tests. Return true if they succeed; false if they fail.
   * 
   * @return true if the tests succeed and false if they fail
   */
  public final boolean runTests() {
    List<TestCase> tests = filterTests(flattenTests(mainTest));

    testsStarted(tests);

    long totalStartTime = System.currentTimeMillis();

    List<TestResult> failures = new ArrayList<TestResult>();
    List<TestTime> slowTests = new ArrayList<TestTime>();

    for (TestCase test : tests) {
      currentTestLoggedError = false;

      testStarted(test);

      long startTime = System.nanoTime();
      TestResult result = test.run();
      long elapsedTimeMS = (System.nanoTime() - startTime) / (1000 * 1000);

      if (currentTestLoggedError && result.wasSuccessful()
          && !hasAnnotation(test, IgnoreLoggedErrors.class)) {
        result.addFailure(test, new AssertionFailedError("IStatus.ERROR written to eclipse log"));
      }

      if (isTestFailureExpected(test) && result.wasSuccessful()) {
        result.addFailure(test, new AssertionFailedError("test passed but was expected to fail"));
      } else if (isTestFailureExpected(test) && !result.wasSuccessful()) {
        // This test failed and was expected to fail. We clear out the test results.
        result = new TestResult();
      }

      if (result.wasSuccessful()) {
        testPassed(test);

        if (elapsedTimeMS >= ONE_SEC_MILLIS) {
          slowTests.add(new TestTime(test, elapsedTimeMS));
        }
      } else {
        testFailed(test, result);
        failures.add(result);
      }
    }

    long totalTestTime = System.currentTimeMillis() - totalStartTime;

    testsFinished(tests, failures, slowTests, totalTestTime);

    return failures.size() == 0;
  }

  public final void setStatusFile(String filePath) {
    // TODO(devoncarew): read and use the status file

  }

  protected void markCurrentTestLoggedError() {
    this.currentTestLoggedError = true;
  }

  protected boolean filterTest(TestCase test) {
    if (hasAnnotationIgnored(test)) {
      return true;
    }

    return false;
  }

  protected boolean isTestFailureExpected(TestCase test) {
    String testId = getTestId(test);

    return testId.indexOf(".fail_") != -1;
  }

  protected abstract void testFailed(TestCase test, TestResult result);

  protected abstract void testPassed(TestCase test);

  protected abstract void testsFinished(List<TestCase> allTests, List<TestResult> failures,
      List<TestTime> slowTests, long totalTime);

  protected abstract void testsStarted(List<TestCase> tests);

  protected abstract void testStarted(TestCase test);

  private List<TestCase> filterTests(List<TestCase> tests) {
    List<TestCase> copy = new ArrayList<TestCase>();

    for (TestCase test : tests) {
      if (!filterTest(test)) {
        copy.add(test);
      }
    }

    return copy;
  }

  private void flatten(Test test, List<TestCase> tests) {
    if (test instanceof TestCase) {
      tests.add((TestCase) test);
    } else if (test instanceof TestSuite) {
      TestSuite suite = (TestSuite) test;

      for (Test child : Collections.list(suite.tests())) {
        flatten(child, tests);
      }
    } else {
      System.out.println("Test instance not recognized: " + test);
    }
  }

  private List<TestCase> flattenTests(Test mainTest) {
    List<TestCase> tests = new ArrayList<TestCase>();
    flatten(mainTest, tests);
    return tests;
  }

}