chrome/test/chromedriver/test/test_expectations

# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Test expectation list for WebDriver Java acceptance tests.

It is evaluated through Python.
"""

import fnmatch


_NEGATIVE_FILTER = [
    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2532
    'SlowLoadingPageTest.testRefreshShouldBlockUntilPageLoads',
    'PageLoadingTest.testShouldTimeoutIfAPageTakesTooLongToRefresh',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2653
    'ReferrerTest.navigationWhenProxyInterceptsASpecificUrl',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2854
    'ChromeOptionsFunctionalTest.canSetAcceptInsecureCerts',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2835
    'CookieImplementationTest.deleteAllCookies',
    'CookieImplementationTest.canHandleHttpOnlyCookie',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2858
    'ElementFindingTest.testFindingMultipleElementsByInvalidClassNameShouldThrow',
    'ElementFindingTest.testFindingMultipleElementsByCompoundClassNameShouldThrow',
    'ElementFindingTest.testFindingMultipleElementsByEmptyIdShouldReturnEmptyList',
    'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
    'PositionAndSizeTest.testShouldGetCoordinatesInViewPortOfAnElementInAFrame',
    'PositionAndSizeTest.testShouldGetCoordinatesInViewPortOfAnElementInANestedFrame',
    'DragAndDropTest.testDragAndDropElementWithOffsetInScrolledDiv',
    'DragAndDropTest.canDragAnElementNotVisibleInTheCurrentViewportDueToAParentOverflow',
    'LocationContextTest.testShouldSetAndGetLatitude',
    'LocationContextTest.testShouldSetAndGetLongitude',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1960
    'CombinedInputActionsTest.testChordControlCutAndPaste',
]

_READY_TO_RUN_FILTER = [
    # marked as not yet implemented with chrome but already works
    'ProxySettingTest.canUsePACThatOnlyProxiesCertainHosts',
]

_OS_NEGATIVE_FILTER = {}
_OS_NEGATIVE_FILTER['win'] = [
]
_OS_NEGATIVE_FILTER['linux'] = [
]
_OS_NEGATIVE_FILTER['mac'] = [
    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2663
    # Test 'WindowTest.testSetsTheSizeOfTheCurrentWindowFromIframe' passes while it is run
    # standalone and fails in a test suit run
    'WindowTest.testSetsTheSizeOfTheCurrentWindowFromIframe',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2856
    'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
    'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
]

_OS_NEGATIVE_FILTER['android:chrome'] = [
    'ChromeOptionsFunctionalTest.*',
    'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',

    'ClickTest.testShouldOnlyFollowHrefOnce',
    'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
    'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
    'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
    'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
    'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
    'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
    'PageLoadingTest.testShouldFollowMetaRedirects',
    'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
    'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
    'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
    'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
    'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
    'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
    'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
    'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
    'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
    'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',

    # Tests require multiple sessions, not supported on Android'
    'CorrectEventFiringTest.testShouldFireFocusEventInNonTopmostWindow',
    'PerformanceLoggingTest.testLogsSingleHttpCommand',
    'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
    'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
    'ProxySettingTest.*',
    'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFireInNonTopmostWindow',
    'ReferrerTest.navigationWhenProxyInterceptsASpecificUrl',
    'ReferrerTest.crossDomainHistoryNavigationWhenProxyInterceptsHostRequests',
    'ReferrerTest.crossDomainHistoryNavigationWithAProxiedHost',

    # Tests require proxy server can't be run on Android
    'ReferrerTest.crossDomainHistoryNavigationWithADirectProxy',
    'ReferrerTest.basicHistoryNavigationWithADirectProxy',
    'ReferrerTest.crossDomainHistoryNavigationWithoutAProxy',
    'ReferrerTest.basicHistoryNavigationWithoutAProxy',

    # Test with file upload not supported on Android
    'UploadTest.testUploadingWithHiddenFileInput',

    # This tests are failing because of Keys.CONTROL/SHIFT/.. are not being supported for drop down items.
    'CombinedInputActionsTest.testControlClickingOnMultiSelectionList',
    'CombinedInputActionsTest.testShiftClickingOnMultiSelectionList',
    'CombinedInputActionsTest.testPlainClickingOnMultiSelectionList',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
    'ImplicitWaitTest.testShouldImplicitlyWaitForASingleElement',

    # Alerts are not yet supported on Android.
    'AlertsTest.*',
    'UnexpectedAlertBehaviorTest.*',

    # http://crbug.com/156390
    'DragAndDropTest.*',

    # Scrolling touch events are not supported.
    'TouchScrollTest.*',

    # These tests start multiple sessions, which is not supported on a single
    # Android device.
    "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
    "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
    "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
    "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
    # Test is written using local files; doesn't work on Android.
    'UploadTest.testFileUploading',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2861
    'PageLoadingTest.testShouldTimeoutIfAPageTakesTooLongToLoadAfterClick',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=604
    'I18nTest.testShouldBeAbleToActivateIMEEngine',

    # https://bugs.chromium.org/p/chromium/issues/detail?id=418590
    'ClearTest.testContentEditableAreaShouldClear',
    'ClickScrollingTest.testClickingOnAnchorScrollsPage',
    'JavascriptEnabledDriverTest.testChangeEventIsFiredAppropriatelyWhenFocusIsLost',
    'TypingTest.testShouldBeAbleToTypeIntoEmptyContentEditableElement',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=922
    'CorrectEventFiringTest.testShouldEmitOnClickEventsWhenSelectingElements',
    'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFire',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1176
    'ChromeDriverTests.clientLogShouldBeEnabledByDefault',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1005
    'WindowTest.*',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1478
    'WindowSwitchingTest.*',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1878
    'CombinedInputActionsTest.testSelectingMultipleItems',
    'CombinedInputActionsTest.testHoldingDownShiftKeyWhileClicking',
    'BasicMouseInterfaceTest.testDoubleClick',
    'BasicMouseInterfaceTest.testContextClick',
    'CorrectEventFiringTest.testShouldFireMouseUpEventWhenClicking',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1960
    'CorrectEventFiringTest.testShouldNotThrowIfEventHandlerThrows',
    'CorrectEventFiringTest.testShouldFireMouseOverEventWhenClicking',
    'CorrectEventFiringTest.testShouldFireMouseDownEventWhenClicking',
    'CorrectEventFiringTest.testShouldFireTwoClickEventsWhenClickingOnALabel',
    'CorrectEventFiringTest.testShouldIssueClickEvents',
    'CorrectEventFiringTest.testShouldFireMouseMoveEventWhenClicking',
    'BasicKeyboardInterfaceTest.testSendingKeysWithShiftPressed',
    'BasicKeyboardInterfaceTest.testBasicKeyboardInputOnActiveElement',
    'BasicKeyboardInterfaceTest.testSendingKeyUp',
    'BasicKeyboardInterfaceTest.testSendingKeyDownOnly',
    'BasicMouseInterfaceTest.testMoveAndClick',
    'BasicMouseInterfaceTest.testShouldClickElementInIFrame',
    'CombinedInputActionsTest.testCanClickOnLinks',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
    'ImplicitWaitTest.testShouldRetainImplicitlyWaitFromTheReturnedWebDriverOfFrameSwitchTo',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2594
    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
    'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
    'TakesScreenshotTest.testShouldCaptureScreenshotOfCurrentViewport',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2601
    'BasicKeyboardInterfaceTest.testSelectionSelectBySymbol',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2611
    'CorrectEventFiringTest.testClickAnElementThatDisappear',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2697
    'CorrectEventFiringTest.testShouldEmitClickEventWhenClickingOnATextInputElement',

    # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2860
    'UploadTest.testUploadingWithInvisibleFileInput',
]

_OS_NEGATIVE_FILTER['android:chrome_stable'] = (
    _OS_NEGATIVE_FILTER['android:chrome'] + [
    ]
)
_OS_NEGATIVE_FILTER['android:chrome_beta'] = (
    _OS_NEGATIVE_FILTER['android:chrome'] + [
    ]
)
_OS_NEGATIVE_FILTER['android:chromium'] = (
    _OS_NEGATIVE_FILTER['android:chrome'] + [
    ]
)
_OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
    _OS_NEGATIVE_FILTER['android:chrome'] + [
        # Not applicable. ChromeDriverWebViewShell doesn't support tabs.
        'WindowSwitchingTest.*',
        'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',

        # Not applicable. ChromeDriverWebViewShell doesn't support local storage
        'LocalStorageTest.*',

        # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2330
        'VisibilityTest.testShouldModifyTheVisibilityOfAnElementDynamically',

        # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2595
        'TakesScreenshotTest.*',

        # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
        'ImplicitWaitTest.testShouldRetainImplicitlyWaitFromTheReturnedWebDriverOfFrameSwitchTo',

        # https://bugs.chromium.org/p/chromedriver/issues/detail?id=2899
        'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
        'ClickTest.testCanClickOnALinkThatOverflowsAndFollowIt',
    ]
)


def _GetNegativeFilter():
  return _NEGATIVE_FILTER

def _GetReadyToRunFilter():
  return _READY_TO_RUN_FILTER

def GetDisabledTestMatchers(operating_system):
  """Returns the list of disabled test matchers for the specific configuration.

  Args:
    operating_system: The operating system, one of 'linux', 'mac', 'win', or
        'android'.

  Returns:
    List of disabled test matchers, which may contain '*' wildcards.
  """
  return (_OS_NEGATIVE_FILTER[operating_system] +
          _GetNegativeFilter())[:]

def GetReadyToRunTestMatchers():
  return (_GetReadyToRunFilter())[:]

def ApplyJavaTestFilter(operating_system, tests):
  """Applies the test filter to the given list of tests.

  Args:
    operating_system: The operating system, one of 'linux', 'mac', 'win', or
        'android'.
    test: list of test names to filter.

  Returns:
    Set of passed test names.
  """
  filters = (_OS_NEGATIVE_FILTER[operating_system] +
             _GetNegativeFilter())
  passed = set(tests)
  for f in filters:
    passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
  return passed