| #!/usr/bin/env python |
| # Copyright 2015 gRPC authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Run interop (cross-language) tests in parallel.""" |
| |
| from __future__ import print_function |
| |
| import argparse |
| import atexit |
| import itertools |
| import json |
| import multiprocessing |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| import uuid |
| import six |
| import traceback |
| |
| import python_utils.dockerjob as dockerjob |
| import python_utils.jobset as jobset |
| import python_utils.report_utils as report_utils |
| # It's ok to not import because this is only necessary to upload results to BQ. |
| try: |
| from python_utils.upload_test_results import upload_interop_results_to_bq |
| except ImportError as e: |
| print(e) |
| |
| # Docker doesn't clean up after itself, so we do it on exit. |
| atexit.register(lambda: subprocess.call(['stty', 'echo'])) |
| |
| ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
| os.chdir(ROOT) |
| |
| _DEFAULT_SERVER_PORT = 8080 |
| |
| _SKIP_CLIENT_COMPRESSION = [ |
| 'client_compressed_unary', 'client_compressed_streaming' |
| ] |
| |
| _SKIP_SERVER_COMPRESSION = [ |
| 'server_compressed_unary', 'server_compressed_streaming' |
| ] |
| |
| _SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION |
| |
| _SKIP_ADVANCED = [ |
| 'status_code_and_message', 'custom_metadata', 'unimplemented_method', |
| 'unimplemented_service' |
| ] |
| |
| _SKIP_SPECIAL_STATUS_MESSAGE = ['special_status_message'] |
| |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE = 'google_default_credentials' |
| |
| _SKIP_GOOGLE_DEFAULT_CREDS = [ |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE, |
| ] |
| |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = 'compute_engine_channel_credentials' |
| |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, |
| ] |
| |
| _TEST_TIMEOUT = 3 * 60 |
| |
| # disable this test on core-based languages, |
| # see https://github.com/grpc/grpc/issues/9779 |
| _SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] |
| |
| # report suffix "sponge_log.xml" is important for reports to get picked up by internal CI |
| _DOCKER_BUILD_XML_REPORT = 'interop_docker_build/sponge_log.xml' |
| _TESTS_XML_REPORT = 'interop_test/sponge_log.xml' |
| |
| |
| class CXXLanguage: |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.http2_cwd = None |
| self.safename = 'cxx' |
| |
| def client_cmd(self, args): |
| return ['bins/opt/interop_client'] + args |
| |
| def client_cmd_http2interop(self, args): |
| return ['bins/opt/http2_client'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['bins/opt/interop_server'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return [] |
| |
| def __str__(self): |
| return 'c++' |
| |
| |
| class CSharpLanguage: |
| |
| def __init__(self): |
| self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/net45' |
| self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/net45' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['mono', 'Grpc.IntegrationTesting.Server.exe'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_SERVER_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'csharp' |
| |
| |
| class CSharpCoreCLRLanguage: |
| |
| def __init__(self): |
| self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/netcoreapp2.1' |
| self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/netcoreapp2.1' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Client.dll'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_SERVER_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'csharpcoreclr' |
| |
| |
| class AspNetCoreLanguage: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-dotnet/testassets/InteropTestsClient/bin/Debug/netcoreapp3.0' |
| self.server_cwd = '../grpc-dotnet/testassets/InteropTestsWebsite/bin/Debug/netcoreapp3.0' |
| self.safename = str(self) |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def client_cmd(self, args): |
| return ['dotnet', 'exec', 'InteropTestsClient.dll'] + args |
| |
| def server_cmd(self, args): |
| return ['dotnet', 'exec', 'InteropTestsWebsite.dll'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| ['compute_engine_creds'] + \ |
| ['jwt_token_creds'] + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'aspnetcore' |
| |
| |
| class DartLanguage: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-dart/interop' |
| self.server_cwd = '../grpc-dart/interop' |
| self.http2_cwd = '../grpc-dart/interop' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['dart', 'bin/client.dart'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['dart', 'bin/server.dart'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE |
| |
| def __str__(self): |
| return 'dart' |
| |
| |
| class JavaLanguage: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-java' |
| self.server_cwd = '../grpc-java' |
| self.http2_cwd = '../grpc-java' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['./run-test-client.sh'] + args |
| |
| def client_cmd_http2interop(self, args): |
| return [ |
| './interop-testing/build/install/grpc-interop-testing/bin/http2-client' |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['./run-test-server.sh'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return [] |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'java' |
| |
| |
| class JavaOkHttpClient: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-java' |
| self.safename = 'java' |
| |
| def client_cmd(self, args): |
| return ['./run-test-client.sh', '--use_okhttp=true'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE |
| |
| def __str__(self): |
| return 'javaokhttp' |
| |
| |
| class GoLanguage: |
| |
| def __init__(self): |
| # TODO: this relies on running inside docker |
| self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' |
| self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' |
| self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['go', 'run', 'client.go'] + args |
| |
| def client_cmd_http2interop(self, args): |
| return ['go', 'run', 'negative_http2_client.go'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ['go', 'run', 'server.go'] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'go' |
| |
| |
| class Http2Server: |
| """Represents the HTTP/2 Interop Test server |
| |
| This pretends to be a language in order to be built and run, but really it |
| isn't. |
| """ |
| |
| def __init__(self): |
| self.server_cwd = None |
| self.safename = str(self) |
| |
| def server_cmd(self, args): |
| return ['python test/http2_test/http2_test_server.py'] |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _TEST_CASES + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _TEST_CASES |
| |
| def __str__(self): |
| return 'http2' |
| |
| |
| class Http2Client: |
| """Represents the HTTP/2 Interop Test |
| |
| This pretends to be a language in order to be built and run, but really it |
| isn't. |
| """ |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['tools/http2_interop/http2_interop.test', '-test.v'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _TEST_CASES + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _TEST_CASES |
| |
| def __str__(self): |
| return 'http2' |
| |
| |
| class NodeLanguage: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-node' |
| self.server_cwd = '../grpc-node' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
| 'node', '--require', './test/fixtures/native_native', |
| 'test/interop/interop_client.js' |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
| 'node', '--require', './test/fixtures/native_native', |
| 'test/interop/interop_server.js' |
| ] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'node' |
| |
| |
| class NodePureJSLanguage: |
| |
| def __init__(self): |
| self.client_cwd = '../grpc-node' |
| self.server_cwd = '../grpc-node' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
| 'node', '--require', './test/fixtures/js_js', |
| 'test/interop/interop_client.js' |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return [] |
| |
| def __str__(self): |
| return 'nodepurejs' |
| |
| |
| class PHPLanguage: |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['src/php/bin/interop_client.sh'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return [] |
| |
| def __str__(self): |
| return 'php' |
| |
| |
| class PHP7Language: |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ['src/php/bin/interop_client.sh'] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return [] |
| |
| def __str__(self): |
| return 'php7' |
| |
| |
| class ObjcLanguage: |
| |
| def __init__(self): |
| self.client_cwd = 'src/objective-c/tests' |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| # from args, extract the server port and craft xcodebuild command out of it |
| for arg in args: |
| port = re.search('--server_port=(\d+)', arg) |
| if port: |
| portnum = port.group(1) |
| cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test' % portnum |
| return [cmdline] |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| # ObjC test runs all cases with the same command. It ignores the testcase |
| # cmdline argument. Here we return all but one test cases as unimplemented, |
| # and depend upon ObjC test's behavior that it runs all cases even when |
| # we tell it to run just one. |
| return _TEST_CASES[1:] + \ |
| _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'objc' |
| |
| |
| class RubyLanguage: |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| 'tools/run_tests/interop/with_rvm.sh', 'ruby', |
| 'src/ruby/pb/test/client.rb' |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| 'tools/run_tests/interop/with_rvm.sh', 'ruby', |
| 'src/ruby/pb/test/server.rb' |
| ] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_SERVER_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_SPECIAL_STATUS_MESSAGE + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'ruby' |
| |
| |
| class PythonLanguage: |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.http2_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', |
| 'run_interop', '--client', '--args="{}"'.format(' '.join(args)) |
| ] |
| |
| def client_cmd_http2interop(self, args): |
| return [ |
| 'py37_native/bin/python', |
| 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', |
| 'run_interop', '--server', '--args="{}"'.format(' '.join(args)) |
| ] |
| |
| def global_env(self): |
| return { |
| 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), |
| 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) |
| } |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION + \ |
| _SKIP_DATA_FRAME_PADDING + \ |
| _SKIP_GOOGLE_DEFAULT_CREDS + \ |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return 'python' |
| |
| |
| _LANGUAGES = { |
| 'c++': CXXLanguage(), |
| 'csharp': CSharpLanguage(), |
| 'csharpcoreclr': CSharpCoreCLRLanguage(), |
| 'aspnetcore': AspNetCoreLanguage(), |
| 'dart': DartLanguage(), |
| 'go': GoLanguage(), |
| 'java': JavaLanguage(), |
| 'javaokhttp': JavaOkHttpClient(), |
| 'node': NodeLanguage(), |
| 'nodepurejs': NodePureJSLanguage(), |
| 'php': PHPLanguage(), |
| 'php7': PHP7Language(), |
| 'objc': ObjcLanguage(), |
| 'ruby': RubyLanguage(), |
| 'python': PythonLanguage(), |
| } |
| |
| # languages supported as cloud_to_cloud servers |
| _SERVERS = [ |
| 'c++', 'node', 'csharp', 'csharpcoreclr', 'aspnetcore', 'java', 'go', |
| 'ruby', 'python', 'dart' |
| ] |
| |
| _TEST_CASES = [ |
| 'large_unary', 'empty_unary', 'ping_pong', 'empty_stream', |
| 'client_streaming', 'server_streaming', 'cancel_after_begin', |
| 'cancel_after_first_response', 'timeout_on_sleeping_server', |
| 'custom_metadata', 'status_code_and_message', 'unimplemented_method', |
| 'client_compressed_unary', 'server_compressed_unary', |
| 'client_compressed_streaming', 'server_compressed_streaming', |
| 'unimplemented_service', 'special_status_message' |
| ] |
| |
| _AUTH_TEST_CASES = [ |
| 'compute_engine_creds', |
| 'jwt_token_creds', |
| 'oauth2_auth_token', |
| 'per_rpc_creds', |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE, |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, |
| ] |
| |
| _HTTP2_TEST_CASES = ['tls', 'framing'] |
| |
| _HTTP2_SERVER_TEST_CASES = [ |
| 'rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', |
| 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test' |
| ] |
| |
| _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { |
| 'data_frame_padding': 'large_unary', |
| 'no_df_padding_sanity_test': 'large_unary' |
| } |
| |
| _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys( |
| ) |
| |
| _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ |
| 'java', 'go', 'python', 'c++' |
| ] |
| |
| _LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] |
| |
| _SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] |
| |
| _TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] |
| |
| _CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ |
| 'tls', 'google_default_credentials', 'compute_engine_channel_creds' |
| ] |
| |
| DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' |
| |
| |
| def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): |
| """Wraps given cmdline array to create 'docker run' cmdline from it.""" |
| docker_cmdline = ['docker', 'run', '-i', '--rm=true'] |
| |
| # turn environ into -e docker args |
| if environ: |
| for k, v in environ.items(): |
| docker_cmdline += ['-e', '%s=%s' % (k, v)] |
| |
| # set working directory |
| workdir = DOCKER_WORKDIR_ROOT |
| if cwd: |
| workdir = os.path.join(workdir, cwd) |
| docker_cmdline += ['-w', workdir] |
| |
| docker_cmdline += docker_args + [image] + cmdline |
| return docker_cmdline |
| |
| |
| def manual_cmdline(docker_cmdline, docker_image): |
| """Returns docker cmdline adjusted for manual invocation.""" |
| print_cmdline = [] |
| for item in docker_cmdline: |
| if item.startswith('--name='): |
| continue |
| if item == docker_image: |
| item = "$docker_image" |
| item = item.replace('"', '\\"') |
| # add quotes when necessary |
| if any(character.isspace() for character in item): |
| item = "\"%s\"" % item |
| print_cmdline.append(item) |
| return ' '.join(print_cmdline) |
| |
| |
| def write_cmdlog_maybe(cmdlog, filename): |
| """Returns docker cmdline adjusted for manual invocation.""" |
| if cmdlog: |
| with open(filename, 'w') as logfile: |
| logfile.write('#!/bin/bash\n') |
| logfile.write('# DO NOT MODIFY\n') |
| logfile.write( |
| '# This file is generated by run_interop_tests.py/create_testcases.sh\n' |
| ) |
| logfile.writelines("%s\n" % line for line in cmdlog) |
| print('Command log written to file %s' % filename) |
| |
| |
| def bash_cmdline(cmdline): |
| """Creates bash -c cmdline from args list.""" |
| # Use login shell: |
| # * makes error messages clearer if executables are missing |
| return ['bash', '-c', ' '.join(cmdline)] |
| |
| |
| def compute_engine_creds_required(language, test_case): |
| """Returns True if given test requires access to compute engine creds.""" |
| language = str(language) |
| if test_case == 'compute_engine_creds': |
| return True |
| if test_case == 'oauth2_auth_token' and language == 'c++': |
| # C++ oauth2 test uses GCE creds because C++ only supports JWT |
| return True |
| return False |
| |
| |
| def auth_options(language, test_case, google_default_creds_use_key_file, |
| service_account_key_file, default_service_account): |
| """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" |
| |
| language = str(language) |
| cmdargs = [] |
| env = {} |
| |
| oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' |
| key_file_arg = '--service_account_key_file=%s' % service_account_key_file |
| default_account_arg = '--default_service_account=%s' % default_service_account |
| |
| if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: |
| if language in [ |
| 'csharp', 'csharpcoreclr', 'aspnetcore', 'node', 'php', 'php7', |
| 'python', 'ruby', 'nodepurejs' |
| ]: |
| env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file |
| else: |
| cmdargs += [key_file_arg] |
| |
| if test_case in ['per_rpc_creds', 'oauth2_auth_token']: |
| cmdargs += [oauth_scope_arg] |
| |
| if test_case == 'oauth2_auth_token' and language == 'c++': |
| # C++ oauth2 test uses GCE creds and thus needs to know the default account |
| cmdargs += [default_account_arg] |
| |
| if test_case == 'compute_engine_creds': |
| cmdargs += [oauth_scope_arg, default_account_arg] |
| |
| if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: |
| if google_default_creds_use_key_file: |
| env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file |
| cmdargs += [default_account_arg] |
| |
| if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: |
| cmdargs += [default_account_arg] |
| |
| return (cmdargs, env) |
| |
| |
| def _job_kill_handler(job): |
| if job._spec.container_name: |
| dockerjob.docker_kill(job._spec.container_name) |
| # When the job times out and we decide to kill it, |
| # we need to wait a before restarting the job |
| # to prevent "container name already in use" error. |
| # TODO(jtattermusch): figure out a cleaner way to this. |
| time.sleep(2) |
| |
| |
| def cloud_to_prod_jobspec(language, |
| test_case, |
| server_host_nickname, |
| server_host, |
| google_default_creds_use_key_file, |
| docker_image=None, |
| auth=False, |
| manual_cmd_log=None, |
| service_account_key_file=None, |
| default_service_account=None, |
| transport_security='tls'): |
| """Creates jobspec for cloud-to-prod interop test""" |
| container_name = None |
| cmdargs = [ |
| '--server_host=%s' % server_host, '--server_port=443', |
| '--test_case=%s' % test_case |
| ] |
| if transport_security == 'tls': |
| transport_security_options = ['--use_tls=true'] |
| elif transport_security == 'google_default_credentials' and str( |
| language) in ['c++', 'go', 'java', 'javaokhttp']: |
| transport_security_options = [ |
| '--custom_credentials_type=google_default_credentials' |
| ] |
| elif transport_security == 'compute_engine_channel_creds' and str( |
| language) in ['go', 'java', 'javaokhttp']: |
| transport_security_options = [ |
| '--custom_credentials_type=compute_engine_channel_creds' |
| ] |
| else: |
| print( |
| 'Invalid transport security option %s in cloud_to_prod_jobspec. Lang: %s' |
| % (str(language), transport_security)) |
| sys.exit(1) |
| cmdargs = cmdargs + transport_security_options |
| environ = dict(language.cloud_to_prod_env(), **language.global_env()) |
| if auth: |
| auth_cmdargs, auth_env = auth_options( |
| language, test_case, google_default_creds_use_key_file, |
| service_account_key_file, default_service_account) |
| cmdargs += auth_cmdargs |
| environ.update(auth_env) |
| cmdline = bash_cmdline(language.client_cmd(cmdargs)) |
| cwd = language.client_cwd |
| |
| if docker_image: |
| container_name = dockerjob.random_name( |
| 'interop_client_%s' % language.safename) |
| cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| cwd=cwd, |
| environ=environ, |
| docker_args=['--net=host', |
| '--name=%s' % container_name]) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image) |
| manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
| cwd = None |
| environ = None |
| |
| suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' |
| test_job = jobset.JobSpec( |
| cmdline=cmdline, |
| cwd=cwd, |
| environ=environ, |
| shortname='%s:%s:%s:%s:%s' % |
| (suite_name, language, server_host_nickname, test_case, |
| transport_security), |
| timeout_seconds=_TEST_TIMEOUT, |
| flake_retries=4 if args.allow_flakes else 0, |
| timeout_retries=2 if args.allow_flakes else 0, |
| kill_handler=_job_kill_handler) |
| if docker_image: |
| test_job.container_name = container_name |
| return test_job |
| |
| |
| def cloud_to_cloud_jobspec(language, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=None, |
| transport_security='tls', |
| manual_cmd_log=None): |
| """Creates jobspec for cloud-to-cloud interop test""" |
| interop_only_options = [ |
| '--server_host_override=foo.test.google.fr', |
| '--use_test_ca=true', |
| ] |
| if transport_security == 'tls': |
| interop_only_options += ['--use_tls=true'] |
| elif transport_security == 'alts': |
| interop_only_options += ['--use_tls=false', '--use_alts=true'] |
| elif transport_security == 'insecure': |
| interop_only_options += ['--use_tls=false'] |
| else: |
| print('Invalid transport security option %s in cloud_to_cloud_jobspec.' |
| % transport_security) |
| sys.exit(1) |
| |
| client_test_case = test_case |
| if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ |
| test_case] |
| if client_test_case in language.unimplemented_test_cases(): |
| print('asking client %s to run unimplemented test case %s' % |
| (repr(language), client_test_case)) |
| sys.exit(1) |
| |
| common_options = [ |
| '--test_case=%s' % client_test_case, |
| '--server_host=%s' % server_host, |
| '--server_port=%s' % server_port, |
| ] |
| |
| if test_case in _HTTP2_SERVER_TEST_CASES: |
| if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| client_options = interop_only_options + common_options |
| cmdline = bash_cmdline(language.client_cmd(client_options)) |
| cwd = language.client_cwd |
| else: |
| cmdline = bash_cmdline( |
| language.client_cmd_http2interop(common_options)) |
| cwd = language.http2_cwd |
| else: |
| cmdline = bash_cmdline( |
| language.client_cmd(common_options + interop_only_options)) |
| cwd = language.client_cwd |
| |
| environ = language.global_env() |
| if docker_image and language.safename != 'objc': |
| # we can't run client in docker for objc. |
| container_name = dockerjob.random_name( |
| 'interop_client_%s' % language.safename) |
| cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| environ=environ, |
| cwd=cwd, |
| docker_args=['--net=host', |
| '--name=%s' % container_name]) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image) |
| manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
| cwd = None |
| |
| test_job = jobset.JobSpec( |
| cmdline=cmdline, |
| cwd=cwd, |
| environ=environ, |
| shortname='cloud_to_cloud:%s:%s_server:%s:%s' % |
| (language, server_name, test_case, transport_security), |
| timeout_seconds=_TEST_TIMEOUT, |
| flake_retries=4 if args.allow_flakes else 0, |
| timeout_retries=2 if args.allow_flakes else 0, |
| kill_handler=_job_kill_handler) |
| if docker_image: |
| test_job.container_name = container_name |
| return test_job |
| |
| |
| def server_jobspec(language, |
| docker_image, |
| transport_security='tls', |
| manual_cmd_log=None): |
| """Create jobspec for running a server""" |
| container_name = dockerjob.random_name( |
| 'interop_server_%s' % language.safename) |
| server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT] |
| if transport_security == 'tls': |
| server_cmd += ['--use_tls=true'] |
| elif transport_security == 'alts': |
| server_cmd += ['--use_tls=false', '--use_alts=true'] |
| elif transport_security == 'insecure': |
| server_cmd += ['--use_tls=false'] |
| else: |
| print('Invalid transport security option %s in server_jobspec.' % |
| transport_security) |
| sys.exit(1) |
| cmdline = bash_cmdline(language.server_cmd(server_cmd)) |
| environ = language.global_env() |
| docker_args = ['--name=%s' % container_name] |
| if language.safename == 'http2': |
| # we are running the http2 interop server. Open next N ports beginning |
| # with the server port. These ports are used for http2 interop test |
| # (one test case per port). |
| docker_args += list( |
| itertools.chain.from_iterable( |
| ('-p', str(_DEFAULT_SERVER_PORT + i)) |
| for i in range(len(_HTTP2_SERVER_TEST_CASES)))) |
| # Enable docker's healthcheck mechanism. |
| # This runs a Python script inside the container every second. The script |
| # pings the http2 server to verify it is ready. The 'health-retries' flag |
| # specifies the number of consecutive failures before docker will report |
| # the container's status as 'unhealthy'. Prior to the first 'health_retries' |
| # failures or the first success, the status will be 'starting'. 'docker ps' |
| # or 'docker inspect' can be used to see the health of the container on the |
| # command line. |
| docker_args += [ |
| '--health-cmd=python test/http2_test/http2_server_health_check.py ' |
| '--server_host=%s --server_port=%d' % ('localhost', |
| _DEFAULT_SERVER_PORT), |
| '--health-interval=1s', |
| '--health-retries=5', |
| '--health-timeout=10s', |
| ] |
| |
| else: |
| docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] |
| |
| docker_cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| cwd=language.server_cwd, |
| environ=environ, |
| docker_args=docker_args) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image) |
| manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) |
| server_job = jobset.JobSpec( |
| cmdline=docker_cmdline, |
| environ=environ, |
| shortname='interop_server_%s' % language, |
| timeout_seconds=30 * 60) |
| server_job.container_name = container_name |
| return server_job |
| |
| |
| def build_interop_image_jobspec(language, tag=None): |
| """Creates jobspec for building interop docker image for a language""" |
| if not tag: |
| tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) |
| env = { |
| 'INTEROP_IMAGE': tag, |
| 'BASE_NAME': 'grpc_interop_%s' % language.safename |
| } |
| if not args.travis: |
| env['TTY_FLAG'] = '-t' |
| # This env variable is used to get around the github rate limit |
| # error when running the PHP `composer install` command |
| host_file = '%s/.composer/auth.json' % os.environ['HOME'] |
| if language.safename == 'php' and os.path.exists(host_file): |
| env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ |
| '-v %s:/root/.composer/auth.json:ro' % host_file |
| build_job = jobset.JobSpec( |
| cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], |
| environ=env, |
| shortname='build_docker_%s' % (language), |
| timeout_seconds=30 * 60) |
| build_job.tag = tag |
| return build_job |
| |
| |
| def aggregate_http2_results(stdout): |
| match = re.search(r'\{"cases[^\]]*\]\}', stdout) |
| if not match: |
| return None |
| |
| results = json.loads(match.group(0)) |
| skipped = 0 |
| passed = 0 |
| failed = 0 |
| failed_cases = [] |
| for case in results['cases']: |
| if case.get('skipped', False): |
| skipped += 1 |
| else: |
| if case.get('passed', False): |
| passed += 1 |
| else: |
| failed += 1 |
| failed_cases.append(case.get('name', "NONAME")) |
| return { |
| 'passed': passed, |
| 'failed': failed, |
| 'skipped': skipped, |
| 'failed_cases': ', '.join(failed_cases), |
| 'percent': 1.0 * passed / (passed + failed) |
| } |
| |
| |
| # A dictionary of prod servers to test against. |
| # See go/grpc-interop-tests (internal-only) for details. |
| prod_servers = { |
| 'default': 'grpc-test.sandbox.googleapis.com', |
| 'gateway_v4': 'grpc-test4.sandbox.googleapis.com', |
| } |
| |
| argp = argparse.ArgumentParser(description='Run interop tests.') |
| argp.add_argument( |
| '-l', |
| '--language', |
| choices=['all'] + sorted(_LANGUAGES), |
| nargs='+', |
| default=['all'], |
| help='Clients to run. Objc client can be only run on OSX.') |
| argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |
| argp.add_argument( |
| '--cloud_to_prod', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Run cloud_to_prod tests.') |
| argp.add_argument( |
| '--cloud_to_prod_auth', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Run cloud_to_prod_auth tests.') |
| argp.add_argument( |
| '--google_default_creds_use_key_file', |
| default=False, |
| action='store_const', |
| const=True, |
| help=('Whether or not we should use a key file for the ' |
| 'google_default_credentials test case, e.g. by ' |
| 'setting env var GOOGLE_APPLICATION_CREDENTIALS.')) |
| argp.add_argument( |
| '--prod_servers', |
| choices=prod_servers.keys(), |
| default=['default'], |
| nargs='+', |
| help=('The servers to run cloud_to_prod and ' |
| 'cloud_to_prod_auth tests against.')) |
| argp.add_argument( |
| '-s', |
| '--server', |
| choices=['all'] + sorted(_SERVERS), |
| nargs='+', |
| help='Run cloud_to_cloud servers in a separate docker ' + |
| 'image. Servers can only be started automatically if ' + |
| '--use_docker option is enabled.', |
| default=[]) |
| argp.add_argument( |
| '--override_server', |
| action='append', |
| type=lambda kv: kv.split('='), |
| help= |
| 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', |
| default=[]) |
| # TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. |
| argp.add_argument( |
| '--service_account_key_file', |
| type=str, |
| help='The service account key file to use for some auth interop tests.', |
| default='/root/service_account/grpc-testing-ebe7c1ac7381.json') |
| argp.add_argument( |
| '--default_service_account', |
| type=str, |
| help='Default GCE service account email to use for some auth interop tests.', |
| default='830293263384-compute@developer.gserviceaccount.com') |
| argp.add_argument( |
| '-t', '--travis', default=False, action='store_const', const=True) |
| argp.add_argument( |
| '-v', '--verbose', default=False, action='store_const', const=True) |
| argp.add_argument( |
| '--use_docker', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Run all the interop tests under docker. That provides ' + |
| 'additional isolation and prevents the need to install ' + |
| 'language specific prerequisites. Only available on Linux.') |
| argp.add_argument( |
| '--allow_flakes', |
| default=False, |
| action='store_const', |
| const=True, |
| help= |
| 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' |
| ) |
| argp.add_argument( |
| '--manual_run', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Prepare things for running interop tests manually. ' + |
| 'Preserve docker images after building them and skip ' |
| 'actually running the tests. Only print commands to run by ' + 'hand.') |
| argp.add_argument( |
| '--http2_interop', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Enable HTTP/2 client edge case testing. (Bad client, good server)') |
| argp.add_argument( |
| '--http2_server_interop', |
| default=False, |
| action='store_const', |
| const=True, |
| help= |
| 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' |
| ) |
| argp.add_argument( |
| '--transport_security', |
| choices=_TRANSPORT_SECURITY_OPTIONS, |
| default='tls', |
| type=str, |
| nargs='?', |
| const=True, |
| help='Which transport security mechanism to use.') |
| argp.add_argument( |
| '--custom_credentials_type', |
| choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, |
| default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, |
| nargs='+', |
| help= |
| 'Credential types to test in the cloud_to_prod setup. Default is to test with all creds types possible.' |
| ) |
| argp.add_argument( |
| '--skip_compute_engine_creds', |
| default=False, |
| action='store_const', |
| const=True, |
| help='Skip auth tests requiring access to compute engine credentials.') |
| argp.add_argument( |
| '--internal_ci', |
| default=False, |
| action='store_const', |
| const=True, |
| help=( |
| '(Deprecated, has no effect) Put reports into subdirectories to improve ' |
| 'presentation of results by Internal CI.')) |
| argp.add_argument( |
| '--bq_result_table', |
| default='', |
| type=str, |
| nargs='?', |
| help='Upload test results to a specified BQ table.') |
| args = argp.parse_args() |
| |
| servers = set( |
| s |
| for s in itertools.chain.from_iterable( |
| _SERVERS if x == 'all' else [x] for x in args.server)) |
| # ALTS servers are only available for certain languages. |
| if args.transport_security == 'alts': |
| servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) |
| |
| if args.use_docker: |
| if not args.travis: |
| print('Seen --use_docker flag, will run interop tests under docker.') |
| print('') |
| print( |
| 'IMPORTANT: The changes you are testing need to be locally committed' |
| ) |
| print( |
| 'because only the committed changes in the current branch will be') |
| print('copied to the docker environment.') |
| time.sleep(5) |
| |
| if args.manual_run and not args.use_docker: |
| print('--manual_run is only supported with --use_docker option enabled.') |
| sys.exit(1) |
| |
| if not args.use_docker and servers: |
| print( |
| 'Running interop servers is only supported with --use_docker option enabled.' |
| ) |
| sys.exit(1) |
| |
| # we want to include everything but objc in 'all' |
| # because objc won't run on non-mac platforms |
| all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) |
| languages = set(_LANGUAGES[l] |
| for l in itertools.chain.from_iterable( |
| all_but_objc if x == 'all' else [x] for x in args.language)) |
| # ALTS interop clients are only available for certain languages. |
| if args.transport_security == 'alts': |
| alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) |
| languages = languages.intersection(alts_languages) |
| |
| languages_http2_clients_for_http2_server_interop = set() |
| if args.http2_server_interop: |
| languages_http2_clients_for_http2_server_interop = set( |
| _LANGUAGES[l] |
| for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES |
| if 'all' in args.language or l in args.language) |
| |
| http2Interop = Http2Client() if args.http2_interop else None |
| http2InteropServer = Http2Server() if args.http2_server_interop else None |
| |
| docker_images = {} |
| if args.use_docker: |
| # languages for which to build docker images |
| languages_to_build = set( |
| _LANGUAGES[k] |
| for k in set([str(l) for l in languages] + [s for s in servers])) |
| languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop |
| |
| if args.http2_interop: |
| languages_to_build.add(http2Interop) |
| |
| if args.http2_server_interop: |
| languages_to_build.add(http2InteropServer) |
| |
| build_jobs = [] |
| for l in languages_to_build: |
| if str(l) == 'objc': |
| # we don't need to build a docker image for objc |
| continue |
| job = build_interop_image_jobspec(l) |
| docker_images[str(l)] = job.tag |
| build_jobs.append(job) |
| |
| if build_jobs: |
| jobset.message( |
| 'START', 'Building interop docker images.', do_newline=True) |
| if args.verbose: |
| print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) |
| |
| num_failures, build_resultset = jobset.run( |
| build_jobs, newline_on_success=True, maxjobs=args.jobs) |
| |
| report_utils.render_junit_xml_report(build_resultset, |
| _DOCKER_BUILD_XML_REPORT) |
| |
| if num_failures == 0: |
| jobset.message( |
| 'SUCCESS', |
| 'All docker images built successfully.', |
| do_newline=True) |
| else: |
| jobset.message( |
| 'FAILED', |
| 'Failed to build interop docker images.', |
| do_newline=True) |
| for image in six.itervalues(docker_images): |
| dockerjob.remove_image(image, skip_nonexistent=True) |
| sys.exit(1) |
| |
| server_manual_cmd_log = [] if args.manual_run else None |
| client_manual_cmd_log = [] if args.manual_run else None |
| |
| # Start interop servers. |
| server_jobs = {} |
| server_addresses = {} |
| try: |
| for s in servers: |
| lang = str(s) |
| spec = server_jobspec( |
| _LANGUAGES[lang], |
| docker_images.get(lang), |
| args.transport_security, |
| manual_cmd_log=server_manual_cmd_log) |
| if not args.manual_run: |
| job = dockerjob.DockerJob(spec) |
| server_jobs[lang] = job |
| server_addresses[lang] = ('localhost', |
| job.mapped_port(_DEFAULT_SERVER_PORT)) |
| else: |
| # don't run the server, set server port to a placeholder value |
| server_addresses[lang] = ('localhost', '${SERVER_PORT}') |
| |
| http2_server_job = None |
| if args.http2_server_interop: |
| # launch a HTTP2 server emulator that creates edge cases |
| lang = str(http2InteropServer) |
| spec = server_jobspec( |
| http2InteropServer, |
| docker_images.get(lang), |
| manual_cmd_log=server_manual_cmd_log) |
| if not args.manual_run: |
| http2_server_job = dockerjob.DockerJob(spec) |
| server_jobs[lang] = http2_server_job |
| else: |
| # don't run the server, set server port to a placeholder value |
| server_addresses[lang] = ('localhost', '${SERVER_PORT}') |
| |
| jobs = [] |
| if args.cloud_to_prod: |
| if args.transport_security not in ['tls']: |
| print('TLS is always enabled for cloud_to_prod scenarios.') |
| for server_host_nickname in args.prod_servers: |
| for language in languages: |
| for test_case in _TEST_CASES: |
| if not test_case in language.unimplemented_test_cases(): |
| if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE: |
| for transport_security in args.custom_credentials_type: |
| # google_default_credentials not yet supported by all languages |
| if transport_security == 'google_default_credentials' and str( |
| language) not in [ |
| 'c++', 'go', 'java', 'javaokhttp' |
| ]: |
| continue |
| # compute_engine_channel_creds not yet supported by all languages |
| if transport_security == 'compute_engine_channel_creds' and str( |
| language) not in [ |
| 'go', 'java', 'javaokhttp' |
| ]: |
| continue |
| test_job = cloud_to_prod_jobspec( |
| language, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args. |
| google_default_creds_use_key_file, |
| docker_image=docker_images.get( |
| str(language)), |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args. |
| service_account_key_file, |
| default_service_account=args. |
| default_service_account, |
| transport_security=transport_security) |
| jobs.append(test_job) |
| if args.http2_interop: |
| for test_case in _HTTP2_TEST_CASES: |
| test_job = cloud_to_prod_jobspec( |
| http2Interop, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args. |
| google_default_creds_use_key_file, |
| docker_image=docker_images.get(str(http2Interop)), |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args.service_account_key_file, |
| default_service_account=args.default_service_account, |
| transport_security=args.transport_security) |
| jobs.append(test_job) |
| |
| if args.cloud_to_prod_auth: |
| if args.transport_security not in ['tls']: |
| print('TLS is always enabled for cloud_to_prod scenarios.') |
| for server_host_nickname in args.prod_servers: |
| for language in languages: |
| for test_case in _AUTH_TEST_CASES: |
| if (not args.skip_compute_engine_creds or |
| not compute_engine_creds_required( |
| language, test_case)): |
| if not test_case in language.unimplemented_test_cases(): |
| if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: |
| transport_security = 'google_default_credentials' |
| elif test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: |
| transport_security = 'compute_engine_channel_creds' |
| else: |
| transport_security = 'tls' |
| if transport_security not in args.custom_credentials_type: |
| continue |
| test_job = cloud_to_prod_jobspec( |
| language, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args. |
| google_default_creds_use_key_file, |
| docker_image=docker_images.get(str(language)), |
| auth=True, |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args. |
| service_account_key_file, |
| default_service_account=args. |
| default_service_account, |
| transport_security=transport_security) |
| jobs.append(test_job) |
| for server in args.override_server: |
| server_name = server[0] |
| (server_host, server_port) = server[1].split(':') |
| server_addresses[server_name] = (server_host, server_port) |
| |
| for server_name, server_address in server_addresses.items(): |
| (server_host, server_port) = server_address |
| server_language = _LANGUAGES.get(server_name, None) |
| skip_server = [] # test cases unimplemented by server |
| if server_language: |
| skip_server = server_language.unimplemented_test_cases_server() |
| for language in languages: |
| for test_case in _TEST_CASES: |
| if not test_case in language.unimplemented_test_cases(): |
| if not test_case in skip_server: |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| transport_security=args.transport_security, |
| manual_cmd_log=client_manual_cmd_log) |
| jobs.append(test_job) |
| |
| if args.http2_interop: |
| for test_case in _HTTP2_TEST_CASES: |
| if server_name == "go": |
| # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 |
| continue |
| test_job = cloud_to_cloud_jobspec( |
| http2Interop, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=docker_images.get(str(http2Interop)), |
| transport_security=args.transport_security, |
| manual_cmd_log=client_manual_cmd_log) |
| jobs.append(test_job) |
| |
| if args.http2_server_interop: |
| if not args.manual_run: |
| http2_server_job.wait_for_healthy(timeout_seconds=600) |
| for language in languages_http2_clients_for_http2_server_interop: |
| for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( |
| _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): |
| offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
| server_port = _DEFAULT_SERVER_PORT + offset |
| if not args.manual_run: |
| server_port = http2_server_job.mapped_port(server_port) |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| str(http2InteropServer), |
| 'localhost', |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| manual_cmd_log=client_manual_cmd_log) |
| jobs.append(test_job) |
| for language in languages: |
| # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of |
| # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather |
| # than specialized http2 clients, reusing existing test implementations. |
| # For example, in the "data_frame_padding" test, use language's gRPC |
| # interop clients and make them think that theyre running "large_unary" |
| # test case. This avoids implementing a new test case in each language. |
| for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| if test_case not in language.unimplemented_test_cases(): |
| offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
| server_port = _DEFAULT_SERVER_PORT + offset |
| if not args.manual_run: |
| server_port = http2_server_job.mapped_port(server_port) |
| if args.transport_security != 'insecure': |
| print( |
| ('Creating grpc client to http2 server test case ' |
| 'with insecure connection, even though ' |
| 'args.transport_security is not insecure. Http2 ' |
| 'test server only supports insecure connections.')) |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| str(http2InteropServer), |
| 'localhost', |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| transport_security='insecure', |
| manual_cmd_log=client_manual_cmd_log) |
| jobs.append(test_job) |
| |
| if not jobs: |
| print('No jobs to run.') |
| for image in six.itervalues(docker_images): |
| dockerjob.remove_image(image, skip_nonexistent=True) |
| sys.exit(1) |
| |
| if args.manual_run: |
| print('All tests will skipped --manual_run option is active.') |
| |
| if args.verbose: |
| print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) |
| |
| num_failures, resultset = jobset.run( |
| jobs, |
| newline_on_success=True, |
| maxjobs=args.jobs, |
| skip_jobs=args.manual_run) |
| if args.bq_result_table and resultset: |
| upload_interop_results_to_bq(resultset, args.bq_result_table) |
| if num_failures: |
| jobset.message('FAILED', 'Some tests failed', do_newline=True) |
| else: |
| jobset.message('SUCCESS', 'All tests passed', do_newline=True) |
| |
| write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') |
| write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') |
| |
| report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) |
| |
| for name, job in resultset.items(): |
| if "http2" in name: |
| job[0].http2results = aggregate_http2_results(job[0].message) |
| |
| http2_server_test_cases = (_HTTP2_SERVER_TEST_CASES |
| if args.http2_server_interop else []) |
| |
| if num_failures: |
| sys.exit(1) |
| else: |
| sys.exit(0) |
| finally: |
| # Check if servers are still running. |
| for server, job in server_jobs.items(): |
| if not job.is_running(): |
| print('Server "%s" has exited prematurely.' % server) |
| |
| dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) |
| |
| for image in six.itervalues(docker_images): |
| if not args.manual_run: |
| print('Removing docker image %s' % image) |
| dockerjob.remove_image(image) |
| else: |
| print('Preserving docker image: %s' % image) |
