blob: e027a96f2cc01a370c2d95cab67ae8cbf9b503ac [file] [log] [blame]
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Test expectation list for WebDriver Java acceptance tests.
It is evaluated through Python.
"""
import fnmatch
_NEGATIVE_FILTER = [
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2532
'SlowLoadingPageTest.testRefreshShouldBlockUntilPageLoads',
'PageLoadingTest.testShouldTimeoutIfAPageTakesTooLongToRefresh',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2653
'ReferrerTest.navigationWhenProxyInterceptsASpecificUrl',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2661
'WindowTest.testSetsTheSizeOfTheCurrentWindowFromIframe',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2854
'ChromeOptionsFunctionalTest.canSetAcceptInsecureCerts',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2835
'CookieImplementationTest.deleteAllCookies',
'CookieImplementationTest.canHandleHttpOnlyCookie',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2858
'ElementFindingTest.testFindingMultipleElementsByInvalidClassNameShouldThrow',
'ElementFindingTest.testFindingMultipleElementsByCompoundClassNameShouldThrow',
'ElementFindingTest.testFindingMultipleElementsByEmptyIdShouldReturnEmptyList',
'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
'PositionAndSizeTest.testShouldGetCoordinatesInViewPortOfAnElementInAFrame',
'PositionAndSizeTest.testShouldGetCoordinatesInViewPortOfAnElementInANestedFrame',
'DragAndDropTest.testDragAndDropElementWithOffsetInScrolledDiv',
'DragAndDropTest.canDragAnElementNotVisibleInTheCurrentViewportDueToAParentOverflow',
'LocationContextTest.testShouldSetAndGetLatitude',
'LocationContextTest.testShouldSetAndGetLongitude',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1960
'CombinedInputActionsTest.testChordControlCutAndPaste',
]
_READY_TO_RUN_FILTER = [
# marked as not yet implemented with chrome but already works
'ProxySettingTest.canUsePACThatOnlyProxiesCertainHosts',
]
_OS_NEGATIVE_FILTER = {}
_OS_NEGATIVE_FILTER['win'] = [
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2855
'WindowTest.canFullscreenTheWindow',
'WindowTest.canFullscreenTheWindowFromIframe',
'WindowTest.testCanMaximizeTheWindowFromIframe',
'WindowTest.testCanMaximizeTheWindow',
'BasicMouseInterfaceTest.testMovingMouseByRelativeOffset',
]
_OS_NEGATIVE_FILTER['linux'] = [
]
_OS_NEGATIVE_FILTER['mac'] = [
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2663
'WindowTest.canFullscreenTheWindow',
'WindowTest.testSetsTheSizeOfTheCurrentWindowFromIframe',
'WindowTest.testSetsTheSizeOfTheCurrentWindowFromFrame',
'WindowTest.canFullscreenTheWindowFromFrame',
'WindowTest.canFullscreenTheWindowFromIframe',
'WindowTest.testSetsTheSizeOfTheCurrentWindow',
'WindowTest.testCanMaximizeTheWindowFromFrame',
'WindowTest.testCanMaximizeTheWindowFromIframe',
'WindowTest.testCanMaximizeTheWindow',
'BasicMouseInterfaceTest.testMovingMouseByRelativeOffset',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2856
'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
]
_OS_NEGATIVE_FILTER['android:chrome'] = [
'ChromeOptionsFunctionalTest.*',
'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
'ClickTest.testShouldOnlyFollowHrefOnce',
'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
'PageLoadingTest.testShouldFollowMetaRedirects',
'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
# Tests require multiple sessions, not supported on Android'
'CorrectEventFiringTest.testShouldFireFocusEventInNonTopmostWindow',
'PerformanceLoggingTest.testLogsSingleHttpCommand',
'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
'ProxySettingTest.*',
'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFireInNonTopmostWindow',
'ReferrerTest.navigationWhenProxyInterceptsASpecificUrl',
'ReferrerTest.crossDomainHistoryNavigationWhenProxyInterceptsHostRequests',
'ReferrerTest.crossDomainHistoryNavigationWithAProxiedHost',
# Tests require proxy server can't be run on Android
'ReferrerTest.crossDomainHistoryNavigationWithADirectProxy',
'ReferrerTest.basicHistoryNavigationWithADirectProxy',
'ReferrerTest.crossDomainHistoryNavigationWithoutAProxy',
'ReferrerTest.basicHistoryNavigationWithoutAProxy',
# Test with file upload not supported on Android
'UploadTest.testUploadingWithHiddenFileInput',
# This tests are failing because of Keys.CONTROL/SHIFT/.. are not being supported for drop down items.
'CombinedInputActionsTest.testControlClickingOnMultiSelectionList',
'CombinedInputActionsTest.testShiftClickingOnMultiSelectionList',
'CombinedInputActionsTest.testPlainClickingOnMultiSelectionList',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
'ImplicitWaitTest.testShouldImplicitlyWaitForASingleElement',
# Alerts are not yet supported on Android.
'AlertsTest.*',
'UnexpectedAlertBehaviorTest.*',
# http://crbug.com/156390
'DragAndDropTest.*',
# Scrolling touch events are not supported.
'TouchScrollTest.*',
# These tests start multiple sessions, which is not supported on a single
# Android device.
"AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
"GetLogsTest.turningOffLogShouldMeanNoLogMessages",
"SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
"SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
# Test is written using local files; doesn't work on Android.
'UploadTest.testFileUploading',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2861
'PageLoadingTest.testShouldTimeoutIfAPageTakesTooLongToLoadAfterClick',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=604
'I18nTest.testShouldBeAbleToActivateIMEEngine',
# https://bugs.chromium.org/p/chromium/issues/detail?id=418590
'ClearTest.testContentEditableAreaShouldClear',
'ClickScrollingTest.testClickingOnAnchorScrollsPage',
'JavascriptEnabledDriverTest.testChangeEventIsFiredAppropriatelyWhenFocusIsLost',
'TypingTest.testShouldBeAbleToTypeIntoEmptyContentEditableElement',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=922
'CorrectEventFiringTest.testShouldEmitOnClickEventsWhenSelectingElements',
'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFire',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1176
'ChromeDriverTests.clientLogShouldBeEnabledByDefault',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1119
'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1005
'WindowTest.*',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1478
'WindowSwitchingTest.*',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1878
'CombinedInputActionsTest.testSelectingMultipleItems',
'CombinedInputActionsTest.testHoldingDownShiftKeyWhileClicking',
'BasicMouseInterfaceTest.testDoubleClick',
'BasicMouseInterfaceTest.testContextClick',
'CorrectEventFiringTest.testShouldFireMouseUpEventWhenClicking',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=1960
'CorrectEventFiringTest.testShouldNotThrowIfEventHandlerThrows',
'CorrectEventFiringTest.testShouldFireMouseOverEventWhenClicking',
'CorrectEventFiringTest.testShouldFireMouseDownEventWhenClicking',
'CorrectEventFiringTest.testShouldFireTwoClickEventsWhenClickingOnALabel',
'CorrectEventFiringTest.testShouldIssueClickEvents',
'CorrectEventFiringTest.testShouldFireMouseMoveEventWhenClicking',
'BasicKeyboardInterfaceTest.testSendingKeysWithShiftPressed',
'BasicKeyboardInterfaceTest.testBasicKeyboardInputOnActiveElement',
'BasicKeyboardInterfaceTest.testSendingKeyUp',
'BasicKeyboardInterfaceTest.testSendingKeyDownOnly',
'BasicMouseInterfaceTest.testMoveAndClick',
'BasicMouseInterfaceTest.testShouldClickElementInIFrame',
'CombinedInputActionsTest.testCanClickOnLinks',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
'ImplicitWaitTest.testShouldRetainImplicitlyWaitFromTheReturnedWebDriverOfFrameSwitchTo',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2594
'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
'TakesScreenshotTest.testShouldCaptureScreenshotOfCurrentViewport',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2601
'BasicKeyboardInterfaceTest.testSelectionSelectBySymbol',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2611
'CorrectEventFiringTest.testClickAnElementThatDisappear',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2697
'CorrectEventFiringTest.testShouldEmitClickEventWhenClickingOnATextInputElement',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2860
'UploadTest.testUploadingWithInvisibleFileInput',
]
_OS_NEGATIVE_FILTER['android:chrome_stable'] = (
_OS_NEGATIVE_FILTER['android:chrome'] + [
]
)
_OS_NEGATIVE_FILTER['android:chrome_beta'] = (
_OS_NEGATIVE_FILTER['android:chrome'] + [
]
)
_OS_NEGATIVE_FILTER['android:chromium'] = (
_OS_NEGATIVE_FILTER['android:chrome'] + [
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2580
'LocalStorageTest.*',
'LocationContextTest.*',
'SessionStorageTest.*',
]
)
_OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
_OS_NEGATIVE_FILTER['android:chrome'] + [
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2330
'VisibilityTest.testShouldModifyTheVisibilityOfAnElementDynamically',
# Not applicable on ChromeDriverWebViewShell (doesn't support tabs).
'WindowSwitchingTest.*',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2595
'TakesScreenshotTest.*',
# https://bugs.chromium.org/p/chromedriver/issues/detail?id=2376
'ImplicitWaitTest.testShouldRetainImplicitlyWaitFromTheReturnedWebDriverOfFrameSwitchTo',
]
)
def _GetNegativeFilter():
return _NEGATIVE_FILTER
def _GetReadyToRunFilter():
return _READY_TO_RUN_FILTER
def GetDisabledTestMatchers(operating_system):
"""Returns the list of disabled test matchers for the specific configuration.
Args:
operating_system: The operating system, one of 'linux', 'mac', 'win', or
'android'.
Returns:
List of disabled test matchers, which may contain '*' wildcards.
"""
return (_OS_NEGATIVE_FILTER[operating_system] +
_GetNegativeFilter())[:]
def GetReadyToRunTestMatchers():
return (_GetReadyToRunFilter())[:]
def ApplyJavaTestFilter(operating_system, tests):
"""Applies the test filter to the given list of tests.
Args:
operating_system: The operating system, one of 'linux', 'mac', 'win', or
'android'.
test: list of test names to filter.
Returns:
Set of passed test names.
"""
filters = (_OS_NEGATIVE_FILTER[operating_system] +
_GetNegativeFilter())
passed = set(tests)
for f in filters:
passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
return passed