| #!/usr/bin/env vpython3 |
| # Copyright 2019 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unittests for xcode_log_parser.py.""" |
| |
| import json |
| import mock |
| import os |
| import unittest |
| |
| from test_result_util import TestStatus |
| import test_runner |
| import test_runner_test |
| import xcode_log_parser |
| import constants |
| |
| |
| OUTPUT_PATH = '/tmp/attempt_0' |
| XCRESULT_PATH = '/tmp/attempt_0.xcresult' |
| XCODE11_DICT = { |
| 'path': '/Users/user1/Xcode.app', |
| 'version': '11.0', |
| 'build': '11M336w', |
| } |
| # A sample of json result when executing xcresulttool on .xcresult dir without |
| # --id. Some unused keys and values were removed. |
| XCRESULT_ROOT = """ |
| { |
| "_type" : { |
| "_name" : "ActionsInvocationRecord" |
| }, |
| "actions" : { |
| "_values" : [ |
| { |
| "actionResult" : { |
| "_type" : { |
| "_name" : "ActionResult" |
| }, |
| "diagnosticsRef" : { |
| "id" : { |
| "_value" : "DIAGNOSTICS_REF_ID" |
| } |
| }, |
| "logRef" : { |
| "id" : { |
| "_value" : "0~6jr1GkZxoWVzWfcUNA5feff3l7g8fPHJ1rqKetCBa3QXhCGY74PnEuRwzktleMTFounMfCdDpSr1hRfhUGIUEQ==" |
| } |
| }, |
| "testsRef" : { |
| "id" : { |
| "_value" : "0~iRbOkDnmtKVIvHSV2jkeuNcg4RDTUaCLZV7KijyxdCqvhqtp08MKxl0MwjBAPpjmruoI7qNHzBR1RJQAlANNHA==" |
| } |
| }, |
| "metrics" : { |
| "_type" : { |
| "_name" : "ResultMetrics" |
| }, |
| "testsCount" : { |
| "_type" : { |
| "_name" : "Int" |
| }, |
| "_value" : "2" |
| }, |
| "testsFailedCount" : { |
| "_type" : { |
| "_name" : "Int" |
| }, |
| "_value" : "2" |
| } |
| } |
| } |
| } |
| ] |
| }, |
| "issues" : { |
| "testFailureSummaries" : { |
| "_values" : [ |
| { |
| "documentLocationInCreatingWorkspace" : { |
| "url" : { |
| "_value" : "file:\/\/\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm#CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130" |
| } |
| }, |
| "message" : { |
| "_value": "Fail. Screenshots: {\\n\\"Failure\\": \\"path.png\\"\\n}" |
| }, |
| "testCaseName" : { |
| "_value": "-[PageStateTestCase testZeroContentOffsetAfterLoad]" |
| } |
| } |
| ] |
| } |
| }, |
| "metrics" : { |
| "testsCount" : { |
| "_value" : "2" |
| }, |
| "testsFailedCount" : { |
| "_value" : "1" |
| } |
| } |
| }""" |
| |
| XCRESULT_MISSING_ACTIONRESULT_METRICS = b""" |
| { |
| "_type" : { |
| "_name" : "ActionsInvocationRecord" |
| }, |
| "actions" : { |
| "_values" : [ |
| { |
| "actionResult" : { |
| "metrics" : { |
| "_type" : { |
| "_name" : "ResultMetrics" |
| }, |
| "errorCount" : { |
| "_type" : { |
| "_name" : "Int" |
| }, |
| "_value" : "1" |
| } |
| } |
| } |
| } |
| ] |
| }, |
| "metrics" : { |
| "errorCount" : { |
| "_type" : { |
| "_name" : "Int" |
| }, |
| "_value" : "1" |
| }, |
| "testsCount" : { |
| "_type" : { |
| "_name" : "Int" |
| }, |
| "_value" : "30" |
| } |
| } |
| }""" |
| |
| REF_ID = b""" |
| { |
| "actions": { |
| "_values": [{ |
| "actionResult": { |
| "testsRef": { |
| "id": { |
| "_value": "REF_ID" |
| } |
| } |
| } |
| }] |
| } |
| }""" |
| |
| # A sample of json result when executing xcresulttool on .xcresult dir with |
| # "testsRef" as --id input. Some unused keys and values were removed. |
| TESTS_REF = """ |
| { |
| "summaries": { |
| "_values": [{ |
| "testableSummaries": { |
| "_type": { |
| "_name": "Array" |
| }, |
| "_values": [{ |
| "tests": { |
| "_type": { |
| "_name": "Array" |
| }, |
| "_values": [{ |
| "identifier" : { |
| "_value" : "All tests" |
| }, |
| "name" : { |
| "_value" : "All tests" |
| }, |
| "subtests": { |
| "_values": [{ |
| "identifier" : { |
| "_value" : "ios_web_shell_eg2tests_module.xctest" |
| }, |
| "name" : { |
| "_value" : "ios_web_shell_eg2tests_module.xctest" |
| }, |
| "subtests": { |
| "_values": [{ |
| "identifier" : { |
| "_value" : "PageStateTestCase" |
| }, |
| "name" : { |
| "_value" : "PageStateTestCase" |
| }, |
| "subtests": { |
| "_values": [{ |
| "testStatus": { |
| "_value": "Success" |
| }, |
| "duration" : { |
| "_type" : { |
| "_name" : "Double" |
| }, |
| "_value" : "35.38412606716156" |
| }, |
| "identifier": { |
| "_value": "PageStateTestCase/testMethod1" |
| }, |
| "name": { |
| "_value": "testMethod1" |
| } |
| }, |
| { |
| "summaryRef": { |
| "id": { |
| "_value": "0~7Q_uAuUSJtx9gtHM08psXFm3g_xiTTg5bpdoDO88nMXo_iMwQTXpqlrlMe5AtkYmnZ7Ux5uEgAe83kJBfoIckw==" |
| } |
| }, |
| "testStatus": { |
| "_value": "Failure" |
| }, |
| "identifier": { |
| "_value": "PageStateTestCase\/testZeroContentOffsetAfterLoad" |
| }, |
| "name": { |
| "_value": "testZeroContentOffsetAfterLoad" |
| } |
| }, |
| { |
| "testStatus": { |
| "_value": "Expected Failure" |
| }, |
| "duration" : { |
| "_type" : { |
| "_name" : "Double" |
| }, |
| "_value" : "28.988606716156" |
| }, |
| "identifier": { |
| "_value": "PageStateTestCase/testMethod2" |
| }, |
| "name": { |
| "_value": "testMethod2" |
| } |
| }, |
| { |
| "testStatus": { |
| "_value": "Skipped" |
| }, |
| "duration" : { |
| "_type" : { |
| "_name" : "Double" |
| }, |
| "_value" : "0.0606716156" |
| }, |
| "identifier": { |
| "_value": "PageStateTestCase/testMethod3" |
| }, |
| "name": { |
| "_value": "testMethod3" |
| } |
| }] |
| } |
| }] |
| } |
| }] |
| } |
| }] |
| } |
| }] |
| } |
| }] |
| } |
| } |
| """ |
| |
| # A sample of json result when executing xcresulttool on .xcresult dir with |
| # a single test summaryRef id value as --id input. Some unused keys and values |
| # were removed. |
| SINGLE_TEST_SUMMARY_REF = """ |
| { |
| "_type" : { |
| "_name" : "ActionTestSummary", |
| "_supertype" : { |
| "_name" : "ActionTestSummaryIdentifiableObject", |
| "_supertype" : { |
| "_name" : "ActionAbstractTestSummary" |
| } |
| } |
| }, |
| "activitySummaries" : { |
| "_values" : [ |
| { |
| "attachments" : { |
| "_values" : [ |
| { |
| "filename" : { |
| "_value" : "Screenshot_25659115-F3E4-47AE-AA34-551C94333D7E.jpg" |
| }, |
| "payloadRef" : { |
| "id" : { |
| "_value" : "SCREENSHOT_REF_ID_1" |
| } |
| } |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Start Test at 2020-10-19 14:12:58.111" |
| } |
| }, |
| { |
| "subactivities" : { |
| "_values" : [ |
| { |
| "attachments" : { |
| "_values" : [ |
| { |
| "filename" : { |
| "_value" : "Screenshot_23D95D0E-8B97-4F99-BE3C-A46EDE5999D7.jpg" |
| }, |
| "payloadRef" : { |
| "id" : { |
| "_value" : "SCREENSHOT_REF_ID_2" |
| } |
| } |
| } |
| ] |
| }, |
| "subactivities" : { |
| "_values" : [ |
| { |
| "subactivities" : { |
| "_values" : [ |
| { |
| "attachments" : { |
| "_values" : [ |
| { |
| "filename" : { |
| "_value" : "Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash" |
| }, |
| "payloadRef" : { |
| "id" : { |
| "_value" : "CRASH_REF_ID_IN_ACTIVITY_SUMMARIES" |
| } |
| } |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Wait for org.chromium.ios-web-shell-eg2tests to idle" |
| } |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Activate org.chromium.ios-web-shell-eg2tests" |
| } |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Open org.chromium.ios-web-shell-eg2tests" |
| } |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Set Up" |
| } |
| }, |
| { |
| "title" : { |
| "_value" : "Find the Target Application 'org.chromium.ios-web-shell-eg2tests'" |
| } |
| }, |
| { |
| "attachments" : { |
| "_values" : [ |
| { |
| "filename" : { |
| "_value" : "Screenshot_278BA84B-2196-4CCD-9D31-2C07DDDC9DFC.jpg" |
| }, |
| "payloadRef" : { |
| "id" : { |
| "_value" : "SCREENSHOT_REF_ID_3" |
| } |
| } |
| |
| } |
| ] |
| }, |
| "title" : { |
| "_value" : "Uncaught Exception at page_state_egtest.mm:131: \\nCannot scroll, the..." |
| } |
| }, |
| { |
| "title" : { |
| "_value" : "Uncaught Exception: Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)" |
| } |
| }, |
| { |
| "title" : { |
| "_value" : "Tear Down" |
| } |
| } |
| ] |
| }, |
| "failureSummaries" : { |
| "_values" : [ |
| { |
| "attachments" : { |
| "_values" : [ |
| { |
| "filename" : { |
| "_value" : "kXCTAttachmentLegacyScreenImageData_1_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg" |
| }, |
| "payloadRef" : { |
| "id" : { |
| "_value" : "SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES" |
| } |
| } |
| } |
| ] |
| }, |
| "fileName" : { |
| "_value" : "\/..\/..\/ios\/web\/shell\/test\/page_state_egtest.mm" |
| }, |
| "lineNumber" : { |
| "_value" : "131" |
| }, |
| "message" : { |
| "_value" : "Some logs." |
| } |
| }, |
| { |
| "message" : { |
| "_value" : "Immediately halt execution of testcase (EarlGreyInternalTestInterruptException)" |
| } |
| } |
| ] |
| }, |
| "identifier" : { |
| "_value" : "PageStateTestCase\/testZeroContentOffsetAfterLoad" |
| }, |
| "name" : { |
| "_value" : "testZeroContentOffsetAfterLoad" |
| }, |
| "testStatus" : { |
| "_value" : "Failure" |
| } |
| }""" |
| |
| APP_SIDE_FAILURE_LOG = """Will attempt to recover by breaking constraint |
| <NSLayoutConstraint:0x1000444e0e0 H:[UIView:0x100031b0fc0]-(4)-[UIView:0x100031b1180] (active)> |
| |
| Make a symbolic breakpoint at UIViewAlertForUnsatisfiableConstraints to catch this in the debugger. |
| The methods in the UIConstraintBasedLayoutDebugging category on UIView listed in <UIKitCore/UIView.h> may also be helpful. |
| [1009/111922.254778:WARNING:base_earl_grey_test_case_app_interface.mm(21)] ********************************* |
| Starting test: -[SmokeTestCase testOpenTab] |
| 2023-10-09 09:25:00.318076-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] Unable to simultaneously satisfy constraints. |
| Probably at least one of the constraints in the following list is one you don't want. |
| Try this: |
| (1) look at each constraint and try to figure out which you don't expect; |
| (2) find the code that added the unwanted constraint or constraints and fix it. |
| ( |
| "<NSLayoutConstraint:0x12003d1a6e0 H:|-(0)-[UIView:0x12002478a80] (active, names: '|':UIView:0x120029d4380 )>", |
| "<NSLayoutConstraint:0x12003d1a680 UIView:0x12002478a80.trailing == UIView:0x120029d4380.trailing (active)>", |
| "<NSLayoutConstraint:0x12003d1a440 H:|-(8)-[UIView:0x1200247d080] (active, names: '|':UIView:0x12002478a80 )>", |
| "<NSLayoutConstraint:0x12003d18340 H:[UIView:0x1200247d080]-(4)-[UIView:0x1200247ec80] (active)>", |
| "<NSLayoutConstraint:0x12003d182e0 UIView:0x1200247ec80.trailing == UIView:0x12002478a80.trailing - 8 (active)>", |
| "<NSLayoutConstraint:0x12003d42a00 'UIView-Encapsulated-Layout-Width' UIView:0x120029d4380.width == 0 (active)>" |
| ) |
| |
| Will attempt to recover by breaking constraint |
| <NSLayoutConstraint:0x12003d18340 H:[UIView:0x1200247d080]-(4)-[UIView:0x1200247ec80] (active)> |
| |
| Make a symbolic breakpoint at UIViewAlertForUnsatisfiableConstraints to catch this in the debugger. |
| The methods in the UIConstraintBasedLayoutDebugging category on UIView listed in <UIKitCore/UIView.h> may also be helpful. |
| 2023-10-09 09:25:01.480402-0700 ios_chrome_eg2tests[56690:4719583] [unspecified] container_create_or_lookup_app_group_path_by_app_group_identifier: client is not entitled |
| 2023-10-09 09:25:02.383910-0700 ios_chrome_eg2tests[56690:4719754] [VoiceShortcutClient] -[VCVoiceShortcutClient unsafeSetupXPCConnection]_block_invoke Client connection to VCVoiceShortcut XPC server interrupted |
| 2023-10-09 09:25:02.385353-0700 ios_chrome_eg2tests[56690:4719754] [Intents] -[INVoiceShortcutCenter getAllVoiceShortcutsWithCompletion:]_block_invoke Error from -getVoiceShortcutsWithCompletion: Error Domain=NSCocoaErrorDomain Code=4097 "Couldn’t communicate with a helper application." |
| [1009/092503.896821:ERROR:loopback_server.cc(907)] Loopback sync cannot read the persistent state file (/Users/chrome-bot/Library/Developer/CoreSimulator/Devices/82CF3734-9FF2-4C1B-920C-B3345C0CA891/data/Containers/Data/Application/F2938797-9668-4622-947A-895503B62BCD/tmp/.org.chromium.ost.chrome.unittests.dev.lxaMyc/profile.pb) with error FILE_ERROR_NOT_FOUND |
| 2023-10-09 09:25:03.964248-0700 ios_chrome_eg2tests[56690:4719583] [unspecified] container_create_or_lookup_app_group_path_by_app_group_identifier: client is not entitled |
| 2023-10-09 09:25:06.074170-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] Unable to simultaneously satisfy constraints. |
| Probably at least one of the constraints in the following list is one you don't want. |
| Try this: |
| (1) look at each constraint and try to figure out which you don't expect; |
| (2) find the code that added the unwanted constraint or constraints and fix it. |
| ( |
| "<NSLayoutConstraint:0x12003d2a8c0 H:|-(0)-[UIView:0x1200235e300] (active, names: '|':UIView:0x12002a10fc0 )>", |
| "<NSLayoutConstraint:0x12003d20360 UIView:0x1200235e300.trailing == UIView:0x12002a10fc0.trailing (active)>", |
| "<NSLayoutConstraint:0x12003d21d40 H:|-(8)-[UIView:0x1200235f640] (active, names: '|':UIView:0x1200235e300 )>", |
| "<NSLayoutConstraint:0x12003d24b60 UIView:0x1200235f640.width == 16 (active)>", |
| "<NSLayoutConstraint:0x12003d9c200 H:[UIView:0x1200235f640]-(4)-[UIView:0x1200235c380] (active)>", |
| "<NSLayoutConstraint:0x12003d98a20 UIView:0x1200235c380.trailing == UIView:0x1200235e300.trailing - 8 (active)>", |
| "<NSLayoutConstraint:0x12001326f60 'UIView-Encapsulated-Layout-Width' UIView:0x12002a10fc0.width == 0 (active)>" |
| ) |
| |
| Will attempt to recover by breaking constraint |
| <NSLayoutConstraint:0x12003d24b60 UIView:0x1200235f640.width == 16 (active)> |
| |
| Make a symbolic breakpoint at UIViewAlertForUnsatisfiableConstraints to catch this in the debugger. |
| The methods in the UIConstraintBasedLayoutDebugging category on UIView listed in <UIKitCore/UIView.h> may also be helpful. |
| 2023-10-09 09:25:06.075364-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] Unable to simultaneously satisfy constraints. |
| Probably at least one of the constraints in the following list is one you don't want. |
| Try this: |
| (1) look at each constraint and try to figure out which you don't expect; |
| (2) find the code that added the unwanted constraint or constraints and fix it. |
| ( |
| "<NSLayoutConstraint:0x12003d2a8c0 H:|-(0)-[UIView:0x1200235e300] (active, names: '|':UIView:0x12002a10fc0 )>", |
| "<NSLayoutConstraint:0x12003d20360 UIView:0x1200235e300.trailing == UIView:0x12002a10fc0.trailing (active)>", |
| "<NSLayoutConstraint:0x12003d21d40 H:|-(8)-[UIView:0x1200235f640] (active, names: '|':UIView:0x1200235e300 )>", |
| "<NSLayoutConstraint:0x12003d9c200 H:[UIView:0x1200235f640]-(4)-[UIView:0x1200235c380] (active)>", |
| "<NSLayoutConstraint:0x12003d98a20 UIView:0x1200235c380.trailing == UIView:0x1200235e300.trailing - 8 (active)>", |
| "<NSLayoutConstraint:0x12001326f60 'UIView-Encapsulated-Layout-Width' UIView:0x12002a10fc0.width == 0 (active)>" |
| ) |
| |
| Will attempt to recover by breaking constraint |
| <NSLayoutConstraint:0x12003d9c200 H:[UIView:0x1200235f640]-(4)-[UIView:0x1200235c380] (active)> |
| |
| Make a symbolic breakpoint at UIViewAlertForUnsatisfiableConstraints to catch this in the debugger. |
| The methods in the UIConstraintBasedLayoutDebugging category on UIView listed in <UIKitCore/UIView.h> may also be helpful. |
| [1009/111926.491858:FATAL:chrome_earl_grey_app_interface.mm(147)] Check failed: NO. |
| 0 ios_chrome_eg2testsMain 0x000000012a31b174 base::debug::CollectStackTrace(void const**, unsigned long) + 48 |
| 1 ios_chrome_eg2testsMain 0x000000012a2ed878 base::debug::StackTrace::StackTrace(unsigned long) + 92 |
| 2 ios_chrome_eg2testsMain 0x000000012a2ed910 base::debug::StackTrace::StackTrace(unsigned long) + 36 |
| 3 ios_chrome_eg2testsMain 0x000000012a2ed8dc base::debug::StackTrace::StackTrace() + 40 |
| 4 ios_chrome_eg2testsMain 0x000000012a03d7e0 logging::LogMessage::~LogMessage() + 204 |
| 5 ios_chrome_eg2testsMain 0x000000012a03e748 logging::LogMessage::~LogMessage() + 28 |
| 6 ios_chrome_eg2testsMain 0x000000012a03e774 logging::LogMessage::~LogMessage() + 28 |
| 7 ios_chrome_eg2testsMain 0x000000012a00bba8 logging::CheckError::~CheckError() + 112 |
| 8 ios_chrome_eg2testsMain 0x000000012a00bc08 logging::CheckError::~CheckError() + 28 |
| 9 ios_chrome_eg2testsMain 0x0000000126745008 +[ChromeEarlGreyAppInterface crashApp] + 104 |
| more of the stack trace and crash report logs... |
| |
| Standard output and standard error from com.google.chrome.unittests.dev with process ID 1358 beginning at 2023-10-09 15:19:36 +0000 |
| |
| 2023-10-09 11:19:37.449520-0400 ios_chrome_eg2tests[1358:24891823] [User Defaults] Not updating lastKnownShmemState in CFPrefsPlistSource<0x6000030083f0> (Domain: com.apple.keyboard.preferences.plist, User: kCFPreferencesCurrentUser, ByHost: No, Container: (null), Contents Need Refresh: Yes): 0 -> 323 |
| 2023-10-09 11:19:37.449594-0400 ios_chrome_eg2tests[1358:24891823] [User Defaults] Source was stale because shmem was null: CFPrefsPlistSource<0x6000030083f0> (Domain: com.apple.keyboard.preferences.plist, User: kCFPreferencesCurrentUser, ByHost: No, Container: (null), Contents Need Refresh: Yes) |
| |
| """ |
| |
| APP_SIDE_FAILURE_LOG_EXPECTED = f"""App crashed and disconnected. |
| Showing logs from application under test. For complete logs see attempt_0_simulator#0_StandardOutputAndStandardError-com.google.chrome.unittests.dev.txt in CAS outputs, which can be found in the swarming task of the shard this test ran on. |
| |
| Starting test: -[SmokeTestCase testOpenTab] |
| 2023-10-09 09:25:00.318076-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] {constants.LAYOUT_CONSTRAINT_MSG}. |
| 2023-10-09 09:25:01.480402-0700 ios_chrome_eg2tests[56690:4719583] [unspecified] container_create_or_lookup_app_group_path_by_app_group_identifier: client is not entitled |
| 2023-10-09 09:25:02.383910-0700 ios_chrome_eg2tests[56690:4719754] [VoiceShortcutClient] -[VCVoiceShortcutClient unsafeSetupXPCConnection]_block_invoke Client connection to VCVoiceShortcut XPC server interrupted |
| 2023-10-09 09:25:02.385353-0700 ios_chrome_eg2tests[56690:4719754] [Intents] -[INVoiceShortcutCenter getAllVoiceShortcutsWithCompletion:]_block_invoke Error from -getVoiceShortcutsWithCompletion: Error Domain=NSCocoaErrorDomain Code=4097 "Couldn’t communicate with a helper application." |
| [1009/092503.896821:ERROR:loopback_server.cc(907)] Loopback sync cannot read the persistent state file (/Users/chrome-bot/Library/Developer/CoreSimulator/Devices/82CF3734-9FF2-4C1B-920C-B3345C0CA891/data/Containers/Data/Application/F2938797-9668-4622-947A-895503B62BCD/tmp/.org.chromium.ost.chrome.unittests.dev.lxaMyc/profile.pb) with error FILE_ERROR_NOT_FOUND |
| 2023-10-09 09:25:03.964248-0700 ios_chrome_eg2tests[56690:4719583] [unspecified] container_create_or_lookup_app_group_path_by_app_group_identifier: client is not entitled |
| 2023-10-09 09:25:06.074170-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] {constants.LAYOUT_CONSTRAINT_MSG}. |
| 2023-10-09 09:25:06.075364-0700 ios_chrome_eg2tests[56690:4719583] [LayoutConstraints] {constants.LAYOUT_CONSTRAINT_MSG}. |
| [1009/111926.491858:FATAL:chrome_earl_grey_app_interface.mm(147)] Check failed: NO. |
| 0 ios_chrome_eg2testsMain 0x000000012a31b174 base::debug::CollectStackTrace(void const**, unsigned long) + 48 |
| 1 ios_chrome_eg2testsMain 0x000000012a2ed878 base::debug::StackTrace::StackTrace(unsigned long) + 92 |
| 2 ios_chrome_eg2testsMain 0x000000012a2ed910 base::debug::StackTrace::StackTrace(unsigned long) + 36 |
| 3 ios_chrome_eg2testsMain 0x000000012a2ed8dc base::debug::StackTrace::StackTrace() + 40 |
| 4 ios_chrome_eg2testsMain 0x000000012a03d7e0 logging::LogMessage::~LogMessage() + 204 |
| 5 ios_chrome_eg2testsMain 0x000000012a03e748 logging::LogMessage::~LogMessage() + 28 |
| 6 ios_chrome_eg2testsMain 0x000000012a03e774 logging::LogMessage::~LogMessage() + 28 |
| 7 ios_chrome_eg2testsMain 0x000000012a00bba8 logging::CheckError::~CheckError() + 112 |
| 8 ios_chrome_eg2testsMain 0x000000012a00bc08 logging::CheckError::~CheckError() + 28 |
| 9 ios_chrome_eg2testsMain 0x0000000126745008 +[ChromeEarlGreyAppInterface crashApp] + 104 |
| more of the stack trace and crash report logs... |
| |
| |
| """ |
| |
| |
| def _xcresulttool_get_side_effect(xcresult_path, ref_id=None): |
| """Side effect for _xcresulttool_get in XcodeLogParser tested.""" |
| if ref_id is None: |
| return XCRESULT_ROOT |
| if ref_id == 'testsRef': |
| return TESTS_REF |
| # Other situation in use cases of xcode_log_parser is asking for single test |
| # summary ref. |
| return SINGLE_TEST_SUMMARY_REF |
| |
| |
| class UtilMethodsTest(test_runner_test.TestCase): |
| """Test case for utility methods not related with Parser class.""" |
| |
| def setUp(self): |
| self.summary_xcode16_with_parallel = { |
| 'tests': { |
| '_values': ['TestSuite1', 'TestSuite2'] |
| } |
| } |
| |
| # Example test summary when running xcode version lower than 16. |
| # It could also be when running xcode version 16 without xcode |
| # parallelization enabled. |
| self.summary_pre_xcode16 = { |
| 'tests': { |
| '_values': [{ |
| 'subtests': { |
| '_values': [{ |
| 'subtests': { |
| '_values': ['TestSuite1', 'TestSuite2'] |
| } |
| }] |
| } |
| }] |
| } |
| } |
| |
| def testParseTestsForInterruptedRun(self): |
| test_output = """ |
| Test case '-[DownloadManagerTestCase testVisibleFileNameAndOpenInDownloads]' passed on 'Clone 2 of iPhone X 15.0 test simulator - ios_chrome_ui_eg2tests_module-Runner (34498)' (20.715 seconds) |
| Test case '-[SyncFakeServerTestCase testSyncDownloadBookmark]' passed on 'Clone 1 of iPhone X 15.0 test simulator - ios_chrome_ui_eg2tests_module-Runner (34249)' (14.880 seconds) |
| Random lines |
| t = 53.90s Tear Down |
| Test Case '-[LinkToTextTestCase testGenerateLinkForSimpleText]' failed (55.316 seconds). |
| t = nans Suite Tear Down |
| Test Suite 'LinkToTextTestCase' failed at 2021-06-15 07:13:17.406. |
| Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.338) seconds |
| Test Suite 'ios_chrome_ui_eg2tests_module.xctest' failed at 2021-06-15 07:13:17.407. |
| Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.340) seconds |
| Test Suite 'Selected tests' failed at 2021-06-15 07:13:17.408. |
| Executed 1 test, with 6 failures (6 unexpected) in 55.316 (55.342) seconds |
| """ |
| test_output_list = test_output.split('\n') |
| expected_passed = set([ |
| 'DownloadManagerTestCase/testVisibleFileNameAndOpenInDownloads', |
| 'SyncFakeServerTestCase/testSyncDownloadBookmark' |
| ]) |
| expected_failed = set(['LinkToTextTestCase/testGenerateLinkForSimpleText']) |
| expected_failed_message = 'Test failed in interrupted(timedout) run.' |
| |
| results = xcode_log_parser.parse_passed_failed_tests_for_interrupted_run( |
| test_output_list) |
| self.assertEqual(results.expected_tests(), expected_passed) |
| self.assertEqual(results.unexpected_tests(), expected_failed) |
| for result in results.test_results: |
| if result.name == 'LinkToTextTestCase/testGenerateLinkForSimpleText': |
| self.assertEqual(result.test_log, expected_failed_message) |
| |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def test_xcode16_parallel(self, mock_xcode_version): |
| mock_xcode_version.return_value = True |
| result = xcode_log_parser.get_test_suites( |
| self.summary_xcode16_with_parallel, True) |
| self.assertEqual(result, ['TestSuite1', 'TestSuite2']) |
| |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def test_xcode16_not_parallel(self, mock_xcode_version): |
| mock_xcode_version.return_value = True |
| result = xcode_log_parser.get_test_suites(self.summary_pre_xcode16, False) |
| self.assertEqual(result, ['TestSuite1', 'TestSuite2']) |
| |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def test_pre_xcode16_parallel(self, mock_xcode_version): |
| mock_xcode_version.return_value = False |
| result = xcode_log_parser.get_test_suites(self.summary_pre_xcode16, True) |
| self.assertEqual(result, ['TestSuite1', 'TestSuite2']) |
| |
| |
| class XcodeLogParserTest(test_runner_test.TestCase): |
| """Test case to test XcodeLogParser.""" |
| |
| def setUp(self): |
| super(XcodeLogParserTest, self).setUp() |
| self.mock(test_runner, 'get_current_xcode_info', lambda: XCODE11_DICT) |
| |
| @mock.patch('subprocess.check_output', autospec=True) |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def testXcresulttoolGetRoot(self, mock_xcode_version, mock_process): |
| mock_xcode_version.return_value = False |
| mock_process.return_value = b'%JSON%' |
| xcode_log_parser.XcodeLogParser()._xcresulttool_get('xcresult_path') |
| self.assertTrue( |
| os.path.join(XCODE11_DICT['path'], 'usr', 'bin') in os.environ['PATH']) |
| self.assertEqual( |
| ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'], |
| mock_process.mock_calls[0][1][0]) |
| |
| @mock.patch('subprocess.check_output', autospec=True) |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def testXcresulttoolGetRef(self, mock_xcode_version, mock_process): |
| mock_xcode_version.return_value = False |
| mock_process.side_effect = [REF_ID, b'JSON'] |
| xcode_log_parser.XcodeLogParser()._xcresulttool_get('xcresult_path', |
| 'testsRef') |
| self.assertEqual( |
| ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'], |
| mock_process.mock_calls[0][1][0]) |
| self.assertEqual([ |
| 'xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path', |
| '--id', 'REF_ID'], mock_process.mock_calls[1][1][0]) |
| |
| def testXcresulttoolListFailedTests(self): |
| failure_message = ( |
| 'file:///../../ios/web/shell/test/page_state_egtest.mm#' |
| 'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130\n' |
| 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}') |
| expected = set(['PageStateTestCase/testZeroContentOffsetAfterLoad']) |
| results = xcode_log_parser.XcodeLogParser()._list_of_failed_tests( |
| json.loads(XCRESULT_ROOT)) |
| self.assertEqual(expected, results.failed_tests()) |
| log = results.test_results[0].test_log |
| self.assertEqual(log, failure_message) |
| |
| def testXcresulttoolListFailedTestsExclude(self): |
| excluded = set(['PageStateTestCase/testZeroContentOffsetAfterLoad']) |
| results = xcode_log_parser.XcodeLogParser()._list_of_failed_tests( |
| json.loads(XCRESULT_ROOT), excluded=excluded) |
| self.assertEqual(set([]), results.all_test_names()) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser._export_data') |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| def testGetTestStatuses(self, mock_xcresult, mock_export): |
| mock_xcresult.side_effect = _xcresulttool_get_side_effect |
| # self.assertEqual(test_result.test_log, lo |
| expected_failure_log = ( |
| 'Logs from "failureSummaries" in .xcresult:\n' |
| 'file: /../../ios/web/shell/test/page_state_egtest.mm, line: 131\n' |
| 'Some logs.\n' |
| 'file: , line: \n' |
| 'Immediately halt execution of testcase ' |
| '(EarlGreyInternalTestInterruptException)\n') |
| expected_expected_tests = set([ |
| 'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2', |
| 'PageStateTestCase/testMethod3' |
| ]) |
| results = xcode_log_parser.XcodeLogParser()._get_test_statuses( |
| OUTPUT_PATH, False) |
| self.assertEqual(expected_expected_tests, results.expected_tests()) |
| seen_failed_test = False |
| for test_result in results.test_results: |
| if test_result.name == 'PageStateTestCase/testZeroContentOffsetAfterLoad': |
| seen_failed_test = True |
| self.assertEqual(test_result.test_log, expected_failure_log) |
| self.assertEqual(test_result.duration, None) |
| crash_file_name = ( |
| 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_' |
| 'Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash' |
| ) |
| jpeg_file_name = ( |
| 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad' |
| '_kXCTAttachmentLegacyScreenImageData_1' |
| '_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg') |
| self.assertDictEqual( |
| { |
| crash_file_name: '/tmp/%s' % crash_file_name, |
| jpeg_file_name: '/tmp/%s' % jpeg_file_name, |
| }, test_result.attachments) |
| if test_result.name == 'PageStateTestCase/testMethod1': |
| self.assertEqual(test_result.duration, 35384) |
| if test_result.name == 'PageStateTestCase/testMethod2': |
| self.assertEqual(test_result.duration, 28988) |
| if test_result.name == 'PageStateTestCase/testMethod3': |
| self.assertEqual(test_result.duration, 60) |
| |
| self.assertTrue(seen_failed_test) |
| |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('xcode_log_parser.XcodeLogParser._extract_artifacts_for_test') |
| @mock.patch('xcode_log_parser.XcodeLogParser.export_diagnostic_data') |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| def testCollectTestTesults(self, mock_root, mock_exist_file, *args): |
| expected_passed = set([ |
| 'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2', |
| 'PageStateTestCase/testMethod3' |
| ]) |
| expected_failed = set(['PageStateTestCase/testZeroContentOffsetAfterLoad']) |
| |
| mock_root.side_effect = _xcresulttool_get_side_effect |
| mock_exist_file.return_value = True |
| results = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, []) |
| |
| # Length ensures no duplicate results from |_get_test_statuses| and |
| # |_list_of_failed_tests|. |
| self.assertEqual(len(results.test_results), 4) |
| self.assertEqual(expected_passed, results.expected_tests()) |
| self.assertEqual(expected_failed, results.unexpected_tests()) |
| # Ensure format. |
| for test in results.test_results: |
| self.assertTrue(isinstance(test.name, str)) |
| if test.status == TestStatus.FAIL: |
| self.assertTrue(isinstance(test.test_log, str)) |
| |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('xcode_log_parser.XcodeLogParser.copy_artifacts') |
| @mock.patch('xcode_log_parser.XcodeLogParser.export_diagnostic_data') |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file, *args): |
| metrics_json = '{"actions": {}}' |
| mock_root.return_value = metrics_json |
| mock_exist_file.return_value = True |
| results = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, []) |
| self.assertTrue(results.crashed) |
| self.assertEqual(results.crash_message, '0 tests executed!') |
| self.assertEqual(len(results.all_test_names()), 0) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser._list_of_failed_tests') |
| @mock.patch('xcode_log_parser.XcodeLogParser._get_test_statuses') |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('xcode_log_parser.XcodeLogParser.copy_artifacts') |
| @mock.patch('xcode_log_parser.XcodeLogParser.export_diagnostic_data') |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| def testFallbackOnRootMetrics(self, mock_root, mock_exist_file, *args): |
| mock_root.return_value = XCRESULT_MISSING_ACTIONRESULT_METRICS |
| mock_exist_file.return_value = True |
| results = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, []) |
| self.assertTrue(results.crashed != True) |
| self.assertNotEqual(results.crash_message, '0 tests executed!') |
| |
| @mock.patch('os.path.exists', autospec=True) |
| def testCollectTestsDidNotRun(self, mock_exist_file): |
| mock_exist_file.return_value = False |
| results = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, []) |
| self.assertTrue(results.crashed) |
| self.assertEqual(results.crash_message, |
| '/tmp/attempt_0 with staging data does not exist.\n') |
| self.assertEqual(len(results.all_test_names()), 0) |
| |
| @mock.patch('os.path.exists', autospec=True) |
| def testCollectTestsInterruptedRun(self, mock_exist_file): |
| mock_exist_file.side_effect = [True, False] |
| results = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, []) |
| self.assertTrue(results.crashed) |
| self.assertEqual( |
| results.crash_message, |
| '/tmp/attempt_0.xcresult with test results does not exist.\n') |
| self.assertEqual(len(results.all_test_names()), 0) |
| |
| @mock.patch('subprocess.check_output', autospec=True) |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def testCopyScreenshots(self, mock_xcode_version, mock_xcresulttool_get, |
| mock_path_exists, mock_process): |
| mock_xcode_version.return_value = False |
| mock_path_exists.return_value = True |
| mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect |
| xcode_log_parser.XcodeLogParser().copy_artifacts(OUTPUT_PATH) |
| mock_process.assert_any_call([ |
| 'xcresulttool', 'export', '--type', 'file', '--id', |
| 'SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES', '--path', XCRESULT_PATH, |
| '--output-path', |
| '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad' |
| '_kXCTAttachmentLegacyScreenImageData_1' |
| '_6CED1FE5-96CA-47EA-9852-6FADED687262.jpeg' |
| ]) |
| mock_process.assert_any_call([ |
| 'xcresulttool', 'export', '--type', 'file', '--id', |
| 'CRASH_REF_ID_IN_ACTIVITY_SUMMARIES', '--path', XCRESULT_PATH, |
| '--output-path', |
| '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad' |
| '_Crash_3F0A2B1C-7ADA-436E-A54C-D4C39B8411F8.crash' |
| ]) |
| # Ensures screenshots in activitySummaries are not copied. |
| self.assertEqual(2, mock_process.call_count) |
| |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('subprocess.check_output', autospec=True) |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def testExportDiagnosticData(self, mock_xcode_version, mock_xcresulttool_get, |
| mock_path_exists, mock_process, _): |
| mock_xcode_version.return_value = False |
| mock_path_exists.return_value = True |
| mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect |
| xcode_log_parser.XcodeLogParser.export_diagnostic_data(OUTPUT_PATH) |
| mock_process.assert_called_with([ |
| 'xcresulttool', 'export', '--type', 'directory', '--id', |
| 'DIAGNOSTICS_REF_ID', '--path', XCRESULT_PATH, '--output-path', |
| '/tmp/attempt_0.xcresult_diagnostic' |
| ]) |
| |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('shutil.copy') |
| @mock.patch('subprocess.check_output', autospec=True) |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| @mock.patch('xcode_util.using_xcode_16_or_higher') |
| def testStdoutCopiedInExportDiagnosticData(self, mock_xcode_version, |
| mock_xcresulttool_get, |
| mock_path_exists, mock_process, |
| mock_copy, _): |
| mock_xcode_version.return_value = False |
| output_path_in_test = 'test_data/attempt_0' |
| xcresult_path_in_test = 'test_data/attempt_0.xcresult' |
| mock_path_exists.return_value = True |
| mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect |
| xcode_log_parser.XcodeLogParser.export_diagnostic_data( |
| output_path_in_test) |
| # os.walk() walks folders in unknown sequence. Use try-except blocks to |
| # assert that any of the 2 assertions is true. |
| try: |
| mock_copy.assert_any_call( |
| 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID1/StandardOutputAndStandardError.txt', |
| 'test_data/attempt_0/../attempt_0_simulator#1_StandardOutputAndStandardError.txt' |
| ) |
| except AssertionError: |
| mock_copy.assert_any_call( |
| 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID1/StandardOutputAndStandardError.txt', |
| 'test_data/attempt_0/../attempt_0_simulator#0_StandardOutputAndStandardError.txt' |
| ) |
| try: |
| mock_copy.assert_any_call( |
| 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID2/StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt', |
| 'test_data/attempt_0/../attempt_0_simulator#1_StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt' |
| ) |
| except AssertionError: |
| mock_copy.assert_any_call( |
| 'test_data/attempt_0.xcresult_diagnostic/test_module-UUID/test_module-UUID2/StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt', |
| 'test_data/attempt_0/../attempt_0_simulator#0_StandardOutputAndStandardError-org.chromium.gtest.ios-chrome-eg2tests.txt' |
| ) |
| |
| @mock.patch('os.path.exists', autospec=True) |
| def testCollectTestResults_interruptedTests(self, mock_path_exists): |
| mock_path_exists.side_effect = [True, False] |
| output = [ |
| '[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.', |
| '[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.', |
| '[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.', |
| '** BUILD INTERRUPTED **', |
| ] |
| not_found_message = ['%s with test results does not exist.' % XCRESULT_PATH] |
| res = xcode_log_parser.XcodeLogParser().collect_test_results( |
| OUTPUT_PATH, output) |
| self.assertTrue(res.crashed) |
| self.assertEqual('\n'.join(not_found_message + output), res.crash_message) |
| self.assertEqual( |
| set(['TestCase1/method1', 'TestCase2/method1']), res.expected_tests()) |
| |
| @mock.patch('file_util.zip_and_remove_folder') |
| @mock.patch('xcode_log_parser.XcodeLogParser._extract_artifacts_for_test') |
| @mock.patch('xcode_log_parser.XcodeLogParser.export_diagnostic_data') |
| @mock.patch('os.path.exists', autospec=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser._xcresulttool_get') |
| @mock.patch('xcode_log_parser.XcodeLogParser._list_of_failed_tests') |
| def testArtifactsDiagnosticLogsExportedInCollectTestTesults( |
| self, mock_get_failed_tests, mock_root, mock_exist_file, |
| mock_export_diagnostic_data, mock_extract_artifacts, mock_zip): |
| mock_root.side_effect = _xcresulttool_get_side_effect |
| mock_exist_file.return_value = True |
| xcode_log_parser.XcodeLogParser().collect_test_results(OUTPUT_PATH, []) |
| mock_export_diagnostic_data.assert_called_with(OUTPUT_PATH) |
| mock_extract_artifacts.assert_called() |
| |
| @mock.patch('os.listdir') |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=APP_SIDE_FAILURE_LOG)) |
| def testLogAppSideFailureReason(self, mock_listdir): |
| test_name = 'SmokeTestCase/testOpenTab' |
| mock_listdir.return_value = [ |
| 'run_1696864672.xctestrun', 'attempt_0.xcresult.zip', |
| 'attempt_0.xcresult_diagnostic.zip', |
| 'attempt_0_simulator#0_StandardOutputAndStandardError-com.google.chrome.unittests.dev.txt', |
| 'attempt_0_simulator#0_StandardOutputAndStandardError.txt', |
| ] |
| app_side_failure_message = \ |
| xcode_log_parser.XcodeLogParser()._get_app_side_failure( |
| test_name, OUTPUT_PATH) |
| self.assertEqual(app_side_failure_message, APP_SIDE_FAILURE_LOG_EXPECTED) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
