android-7.0.0_r1/s

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.cts.tradefed.testtype;

import com.android.ddmlib.IShellOutputReceiver;
import com.android.ddmlib.Log;
import com.android.ddmlib.MultiLineReceiver;
import com.android.ddmlib.testrunner.ITestRunListener;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.testtype.testdefs.XmlDefsTest;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * TODO: Merge change to add package prefix to tradefed's GTestResultParser.
 *       Otherwise this file should be the same as TradeFed's version.
 *
 * Parses the 'raw output mode' results of native tests using GTest that run from shell, and informs
 * a ITestRunListener of the results.
 * <p>Sample format of output expected:
 *
 * <pre>
 * [==========] Running 15 tests from 1 test case.
 * [----------] Global test environment set-up.
 * [----------] 15 tests from MessageTest
 * [ RUN      ] MessageTest.DefaultConstructor
 * [       OK ] MessageTest.DefaultConstructor (1 ms)
 * [ RUN      ] MessageTest.CopyConstructor
 * external/gtest/test/gtest-message_test.cc:67: Failure
 * Value of: 5
 * Expected: 2
 * external/gtest/test/gtest-message_test.cc:68: Failure
 * Value of: 1 == 1
 * Actual: true
 * Expected: false
 * [  FAILED  ] MessageTest.CopyConstructor (2 ms)
 *  ...
 * [ RUN      ] MessageTest.DoesNotTakeUpMuchStackSpace
 * [       OK ] MessageTest.DoesNotTakeUpMuchStackSpace (0 ms)
 * [----------] 15 tests from MessageTest (26 ms total)
 *
 * [----------] Global test environment tear-down
 * [==========] 15 tests from 1 test case ran. (26 ms total)
 * [  PASSED  ] 6 tests.
 * [  FAILED  ] 9 tests, listed below:
 * [  FAILED  ] MessageTest.CopyConstructor
 * [  FAILED  ] MessageTest.ConstructsFromCString
 * [  FAILED  ] MessageTest.StreamsCString
 * [  FAILED  ] MessageTest.StreamsNullCString
 * [  FAILED  ] MessageTest.StreamsString
 * [  FAILED  ] MessageTest.StreamsStringWithEmbeddedNUL
 * [  FAILED  ] MessageTest.StreamsNULChar
 * [  FAILED  ] MessageTest.StreamsInt
 * [  FAILED  ] MessageTest.StreamsBasicIoManip
 * 9 FAILED TESTS
 * </pre>
 *
 * <p>where the following tags are used to signal certain events:
 * <pre>
 * [==========]: the first occurrence indicates a new run started, including the number of tests
 *                  to be expected in this run
 * [ RUN      ]: indicates a new test has started to run; a series of zero or more lines may
 *                  follow a test start, and will be captured in case of a test failure or error
 * [       OK ]: the preceding test has completed successfully, optionally including the time it
 *                  took to run (in ms)
 * [  FAILED  ]: the preceding test has failed, optionally including the time it took to run (in ms)
 * [==========]: the preceding test run has completed, optionally including the time it took to run
 *                  (in ms)
 * </pre>
 *
 * All other lines are ignored.
 */
public class GeeTestResultParser extends MultiLineReceiver {
    private static final String LOG_TAG = "GTestResultParser";

    // Variables to keep track of state
    private TestResult mCurrentTestResult = null;
    private int mNumTestsRun = 0;
    private int mNumTestsExpected = 0;
    private long mTotalRunTime = 0;
    private boolean mTestInProgress = false;
    private boolean mTestRunInProgress = false;
    private final String mTestRunId;
    private final Collection<ITestRunListener> mTestListeners;

    /** Fake adding a package prefix if the test listener needs it. */
    private String mFakePackagePrefix = "";

    /** True if start of test has already been reported to listener. */
    private boolean mTestRunStartReported = false;

    /** True if current test run has been canceled by user. */
    private boolean mIsCancelled = false;

    private String mCoverageTarget = null;

    /**
     * Test result data
     */
    private static class TestResult {
        private String mTestName = null;
        private String mTestClass = null;
        private StringBuilder mStackTrace = null;
        @SuppressWarnings("unused")
        private Long mRunTime = null;

        /** Returns whether expected values have been parsed
         *
         * @return true if all expected values have been parsed
         */
        boolean isComplete() {
            return mTestName != null && mTestClass != null;
        }

        /** Returns whether there is currently a stack trace
         *
         * @return true if there is currently a stack trace, false otherwise
         */
        boolean hasStackTrace() {
            return mStackTrace != null;
        }

        /**
         * Returns the stack trace of the current test.
         *
         * @return a String representation of the current test's stack trace; if there is not
         * a current stack trace, it returns an error string. Use {@link TestResult#hasStackTrace}
         * if you need to know whether there is a stack trace.
         */
        String getTrace() {
            if (hasStackTrace()) {
                return mStackTrace.toString();
            } else {
                Log.e(LOG_TAG, "Could not find stack trace for failed test");
                return new Throwable("Unknown failure").toString();
            }
        }

        /** Provides a more user readable string for TestResult, if possible */
        @Override
        public String toString() {
            StringBuilder output = new StringBuilder();
            if (mTestClass != null ) {
                output.append(mTestClass);
                output.append('#');
            }
            if (mTestName != null) {
                output.append(mTestName);
            }
            if (output.length() > 0) {
                return output.toString();
            }
            return "unknown result";
        }
    }

    /** Internal helper struct to store parsed test info. */
    private static class ParsedTestInfo {
        String mTestName = null;
        String mTestClassName = null;
        String mTestRunTime = null;

        public ParsedTestInfo(String testName, String testClassName, String testRunTime) {
            mTestName = testName;
            mTestClassName = testClassName;
            mTestRunTime = testRunTime;
        }
    }

    /** Prefixes used to demarcate and identify output. */
    private static class Prefixes {
        @SuppressWarnings("unused")
        private static final String INFORMATIONAL_MARKER = "[----------]";
        private static final String START_TEST_RUN_MARKER = "[==========] Running";
        private static final String TEST_RUN_MARKER = "[==========]";
        private static final String START_TEST_MARKER = "[ RUN      ]";
        private static final String OK_TEST_MARKER = "[       OK ]";
        private static final String FAILED_TEST_MARKER = "[  FAILED  ]";
    }

    /**
     * Creates the GTestResultParser.
     *
     * @param testRunId the test run id to provide to
     *            {@link ITestRunListener#testRunStarted(String, int)}
     * @param listeners informed of test results as the tests are executing
     */
    public GeeTestResultParser(String testRunId, Collection<ITestRunListener> listeners) {
        mTestRunId = testRunId;
        mTestListeners = new ArrayList<ITestRunListener>(listeners);
    }

    /**
     * Creates the GTestResultParser for a single listener.
     *
     * @param testRunId the test run id to provide to
     *            {@link ITestRunListener#testRunStarted(String, int)}
     * @param listener informed of test results as the tests are executing
     */
    public GeeTestResultParser(String testRunId, ITestRunListener listener) {
        mTestRunId = testRunId;
        mTestListeners = new ArrayList<ITestRunListener>(1);
        mTestListeners.add(listener);
    }

    /**
     * Package prefix to be added to test names when they are reported like
     * "android.nativemedia." You may need to add the dot if you need it.
     */
    public void setFakePackagePrefix(String prefix) {
        mFakePackagePrefix = prefix;
    }

    /**
     * Returns the current TestResult for test in progress, or a new default one.
     *
     * @return The TestResult for the current test run
     */
    private TestResult getCurrentTestResult() {
        if (mCurrentTestResult == null) {
            mCurrentTestResult = new TestResult();
        }
        return mCurrentTestResult;
    }


    /**
     * Clears out the current TestResult.
     */
    private void clearCurrentTestResult() {
        mCurrentTestResult = null;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void processNewLines(String[] lines) {
        for (String line : lines) {
            parse(line);
        }
    }

    /**
     * Parse an individual output line.
     *
     * @param line  Text output line
     */
    private void parse(String line) {
        String message = null;

        if (mTestRunInProgress || line.startsWith(Prefixes.TEST_RUN_MARKER)) {
            if (line.startsWith(Prefixes.START_TEST_MARKER)) {
                // Individual test started
                message = line.substring(Prefixes.START_TEST_MARKER.length()).trim();
                processTestStartedTag(message);
            }
            else if (line.startsWith(Prefixes.OK_TEST_MARKER)) {
                // Individual test completed successfully
                message = line.substring(Prefixes.OK_TEST_MARKER.length()).trim();
                processOKTag(message);
                clearCurrentTestResult();
            }
            else if (line.startsWith(Prefixes.FAILED_TEST_MARKER)) {
                // Individual test completed with failure
                message = line.substring(Prefixes.FAILED_TEST_MARKER.length()).trim();
                processFailedTag(message);
                clearCurrentTestResult();
            }
            else if (line.startsWith(Prefixes.START_TEST_RUN_MARKER)) {
                // Test run started
                // Make sure to leave the "Running" in the string
                message = line.substring(Prefixes.TEST_RUN_MARKER.length()).trim();
                processRunStartedTag(message);
            }
            else if (line.startsWith(Prefixes.TEST_RUN_MARKER)) {
                // Test run ended
                // This is for the end of the test suite run, so make sure this else-if is after the
                // check for START_TEST_SUITE_MARKER
                message = line.substring(Prefixes.TEST_RUN_MARKER.length()).trim();
                processRunCompletedTag(message);
            }
            else if (testInProgress()) {
                // Note this does not handle the case of an error outside an actual test run
                appendTestOutputLine(line);
            }
        }
    }

    /**
     * Returns true if test run canceled.
     *
     * @see IShellOutputReceiver#isCancelled()
     */
    @Override
    public boolean isCancelled() {
        return mIsCancelled;
    }

    /**
     * Requests cancellation of test run.
     */
    public void cancel() {
        mIsCancelled = true;
    }

    /**
     * Returns whether we're in the middle of running a test.
     *
     * @return True if a test was started, false otherwise
     */
    private boolean testInProgress() {
        return mTestInProgress;
    }

    /**
     * Set state to indicate we've started running a test.
     *
     */
    private void setTestStarted() {
        mTestInProgress = true;
    }

    /**
     * Set state to indicate we've started running a test.
     *
     */
    private void setTestEnded() {
        mTestInProgress = false;
    }

    /**
     * Reports the start of a test run, and the total test count, if it has not been previously
     * reported.
     */
    private void reportTestRunStarted() {
        // if start test run not reported yet
        if (!mTestRunStartReported) {
            for (ITestRunListener listener : mTestListeners) {
                listener.testRunStarted(mTestRunId, mNumTestsExpected);
            }
            mTestRunStartReported = true;
        }
    }

    /**
     * Reports the end of a test run, and resets that test
     */
    private void reportTestRunEnded() {
        for (ITestRunListener listener : mTestListeners) {
            listener.testRunEnded(mTotalRunTime, getRunMetrics());
        }
        mTestRunStartReported = false;
    }

    /**
     * Create the run metrics {@link Map} to report.
     *
     * @return a {@link Map} of run metrics data
     */
    private Map<String, String> getRunMetrics() {
        Map<String, String> metricsMap = new HashMap<String, String>();
        if (mCoverageTarget != null) {
            metricsMap.put(XmlDefsTest.COVERAGE_TARGET_KEY, mCoverageTarget);
        }
        return metricsMap;
    }

    /**
     * Parse the test identifier (class and test name), and optional time info.
     *
     * @param identifier Raw identifier of the form classname.testname, with an optional time
     *          element in the format of (XX ms) at the end
     * @return A ParsedTestInfo representing the parsed info from the identifier string.
     *
     *          If no time tag was detected, then the third element in the array (time_in_ms) will
     *          be null. If the line failed to parse properly (eg: could not determine name of
     *          test/class) then an "UNKNOWN" string value will be returned for the classname and
     *          testname. This method guarantees a string will always be returned for the class and
     *          test names (but not for the time value).
     */
    private ParsedTestInfo parseTestIdentifier(String identifier) {
        ParsedTestInfo returnInfo = new ParsedTestInfo("UNKNOWN_CLASS", "UNKNOWN_TEST", null);

        Pattern timePattern = Pattern.compile(".*(\\((\\d+) ms\\))");  // eg: (XX ms)
        Matcher time = timePattern.matcher(identifier);

        // Try to find a time
        if (time.find()) {
            String timeString = time.group(2);  // the "XX" in "(XX ms)"
            String discardPortion = time.group(1);  // everything after the test class/name
            identifier = identifier.substring(0, identifier.lastIndexOf(discardPortion)).trim();
            returnInfo.mTestRunTime = timeString;
        }

        String[] testId = identifier.split("\\.");
        if (testId.length < 2) {
            Log.e(LOG_TAG, "Could not detect the test class and test name, received: " +
                    identifier);
        }
        else {
            returnInfo.mTestClassName = testId[0];
            returnInfo.mTestName = testId[1];
        }
        return returnInfo;
    }

    /**
     * Parses and stores the test identifier (class and test name).
     *
     * @param identifier Raw identifier
     */
    private void processRunStartedTag(String identifier) {
        // eg: (Running XX tests from 1 test case.)
        Pattern numTestsPattern = Pattern.compile("Running (\\d+) test[s]? from .*");
        Matcher numTests = numTestsPattern.matcher(identifier);

        // Try to find number of tests
        if (numTests.find()) {
            try {
                mNumTestsExpected = Integer.parseInt(numTests.group(1));
            }
            catch (NumberFormatException e) {
                Log.e(LOG_TAG, "Unable to determine number of tests expected, received: " +
                        numTests.group(1));
            }
        }
        if (mNumTestsExpected > 0) {
          reportTestRunStarted();
          mNumTestsRun = 0;
          mTestRunInProgress = true;
        }
    }

    /**
     * Processes and informs listener when we encounter a tag indicating that a test suite is done.
     *
     * @param identifier Raw log output from the suite ended tag
     */
    private void processRunCompletedTag(String identifier) {
        Pattern timePattern = Pattern.compile(".*\\((\\d+) ms total\\)");  // eg: (XX ms total)
        Matcher time = timePattern.matcher(identifier);

        // Try to find the total run time
        if (time.find()) {
            try {
                mTotalRunTime = Long.parseLong(time.group(1));
            }
            catch (NumberFormatException e) {
                Log.e(LOG_TAG, "Unable to determine the total running time, received: " +
                        time.group(1));
            }
        }
        reportTestRunEnded();
        mTestRunInProgress = false;
    }

    /**
     * Processes and informs listener when we encounter a tag indicating that a test has started.
     *
     * @param identifier Raw log output of the form classname.testname, with an optional time (x ms)
     */
    private void processTestStartedTag(String identifier) {
        ParsedTestInfo parsedResults = parseTestIdentifier(identifier);
        TestResult testResult = getCurrentTestResult();
        testResult.mTestClass = parsedResults.mTestClassName;
        testResult.mTestName = parsedResults.mTestName;

        TestIdentifier testId = new TestIdentifier(mFakePackagePrefix + testResult.mTestClass,
                testResult.mTestName);

        for (ITestRunListener listener : mTestListeners) {
            listener.testStarted(testId);
        }
        setTestStarted();
    }

    /**
     * Helper method to do the work necessary when a test has ended.
     *
     * @param identifier Raw log output of the form "classname.testname" with an optional (XX ms)
     *          at the end indicating the running time.
     * @param testPassed Indicates whether the test passed or failed (set to true if passed, false
     *          if failed)
     */
    private void doTestEnded(String identifier, boolean testPassed) {
        ParsedTestInfo parsedResults = parseTestIdentifier(identifier);
        TestResult testResult = getCurrentTestResult();
        TestIdentifier testId = new TestIdentifier(mFakePackagePrefix + testResult.mTestClass,
                testResult.mTestName);

        // Error - trying to end a test when one isn't in progress
        if (!testInProgress()) {
            Log.e(LOG_TAG, "Test currently not in progress when trying to end test: " + identifier);
            return;
        }

        // Save the run time for this test if one exists
        if (parsedResults.mTestRunTime != null) {
            try {
                testResult.mRunTime = new Long(parsedResults.mTestRunTime);
            }
            catch (NumberFormatException e) {
                Log.e(LOG_TAG, "Test run time value is invalid, received: " +
                        parsedResults.mTestRunTime);
            }
        }

        // Check that the test result is for the same test/class we're expecting it to be for
        boolean encounteredUnexpectedTest = false;
        if (!testResult.isComplete()) {
            Log.e(LOG_TAG, "No test/class name is currently recorded as running!");
        }
        else {
            if (testResult.mTestClass.compareTo(parsedResults.mTestClassName) != 0) {
                Log.e(LOG_TAG, "Name for current test class does not match class we started " +
                        "with, expected: " + testResult.mTestClass + " but got: " +
                        parsedResults.mTestClassName);
                encounteredUnexpectedTest = true;
            }
            if (testResult.mTestName.compareTo(parsedResults.mTestName) != 0) {
                Log.e(LOG_TAG, "Name for current test does not match test we started with," +
                        "expected: " + testResult.mTestName + " bug got: " +
                        parsedResults.mTestName);
                encounteredUnexpectedTest = true;
            }
        }

        if (encounteredUnexpectedTest) {
            // If the test name of the result changed from what we started with, report that
            // the last known test failed, regardless of whether we received a pass or fail tag.
            for (ITestRunListener listener : mTestListeners) {
                listener.testFailed(testId, mCurrentTestResult.getTrace());
            }
        }
        else if (!testPassed) {  // test failed
            for (ITestRunListener listener : mTestListeners) {
                listener.testFailed(testId, mCurrentTestResult.getTrace());
            }
        }
        // For all cases (pass or fail), we ultimately need to report test has ended
        Map <String, String> emptyMap = Collections.emptyMap();
        for (ITestRunListener listener : mTestListeners) {
            // @TODO: Add reporting of test run time to ITestRunListener
            listener.testEnded(testId, emptyMap);
        }

        setTestEnded();
        ++mNumTestsRun;
    }

    /**
     * Processes and informs listener when we encounter the OK tag.
     *
     * @param identifier Raw log output of the form "classname.testname" with an optional (XX ms)
     *          at the end indicating the running time.
     */
    private void processOKTag(String identifier) {
        doTestEnded(identifier, true);
    }

    /**
     * Processes and informs listener when we encounter the FAILED tag.
     *
     * @param identifier Raw log output of the form "classname.testname" with an optional (XX ms)
     *          at the end indicating the running time.
     */
    private void processFailedTag(String identifier) {
        doTestEnded(identifier, false);
    }

    /**
     * Appends the test output to the current TestResult.
     *
     * @param line Raw test result line of output.
     */
    private void appendTestOutputLine(String line) {
        TestResult testResult = getCurrentTestResult();
        if (testResult.mStackTrace == null) {
            testResult.mStackTrace = new StringBuilder();
        }
        else {
            testResult.mStackTrace.append("\r\n");
        }
        testResult.mStackTrace.append(line);
    }

    /**
     * Process an instrumentation run failure
     *
     * @param errorMsg The message to output about the nature of the error
     */
    private void handleTestRunFailed(String errorMsg) {
        errorMsg = (errorMsg == null ? "Unknown error" : errorMsg);
        Log.i(LOG_TAG, String.format("Test run failed: %s", errorMsg));
        String testRunStackTrace = "";

        // Report that the last known test failed
        if ((mCurrentTestResult != null) && (mCurrentTestResult.isComplete())) {
            // current test results are cleared out after every complete test run,
            // if it's not null, assume the last test caused this and report as a test failure
            TestIdentifier testId = new TestIdentifier(mCurrentTestResult.mTestClass,
                    mCurrentTestResult.mTestName);

            // If there was any stack trace during the test run, append it to the "test failed"
            // error message so we have an idea of what caused the crash/failure.
            Map<String, String> emptyMap = Collections.emptyMap();
            if (mCurrentTestResult.hasStackTrace()) {
                testRunStackTrace = mCurrentTestResult.getTrace();
            }
            for (ITestRunListener listener : mTestListeners) {
                listener.testFailed(testId, "No test results.\r\n" + testRunStackTrace);
                listener.testEnded(testId, emptyMap);
            }
            clearCurrentTestResult();
        }
        // Report the test run failed
        for (ITestRunListener listener : mTestListeners) {
            listener.testRunFailed(errorMsg);
            listener.testRunEnded(mTotalRunTime, getRunMetrics());
        }
    }

    /**
     * Called by parent when adb session is complete.
     */
    @Override
    public void done() {
        super.done();
        if (mNumTestsExpected > mNumTestsRun) {
            handleTestRunFailed(String.format("Test run incomplete. Expected %d tests, received %d",
                    mNumTestsExpected, mNumTestsRun));
        }
        else if (mTestRunInProgress) {
            handleTestRunFailed("No test results");
        }
    }

    /**
     * Sets the coverage target for this test.
     * <p/>
     * Will be sent as a metric to test listeners.
     *
     * @param coverageTarget the coverage target
     */
    public void setCoverageTarget(String coverageTarget) {
        mCoverageTarget = coverageTarget;
    }
}