commit a3d2f057746b9de1907320ee13a76d2e4d6c29df
author Jonathan Lee <jonathanjlee@google.com> Fri May 27 20:07:28 2022
committer Chromium LUCI CQ <chromium-scoped@luci-project-accounts.iam.gserviceaccount.com> Fri May 27 20:07:28 2022
tree 868783c02973dc6f7fe2afbb67540f09b3391d58
parent 8038fd5a6e87f5a2dd6555132ad174bf0bd07929

[blinkpy] Gzip HTTP responses by default

The legacy test-results server fails with HTTP 500 when attempting to
serve very large results JSON files (~30 MB). This change modifies the
blink HTTP client to request a gzipped payload so that the request
succeeds.

Bug: 1328703
Change-Id: Id3e452ded6546fd0473e3d26b464c17d0258f264
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3673463
Commit-Queue: Jonathan Lee <jonathanjlee@google.com>
Reviewed-by: Weizhong Xia <weizhong@google.com>
Cr-Commit-Position: refs/heads/main@{#1008397}

third_party/blink/tools/blinkpy/common/net/web.py

diff --git a/third_party/blink/tools/blinkpy/common/net/web.py b/third_party/blink/tools/blinkpy/common/net/web.py
index 02af1e4..8315076 100644
--- a/third_party/blink/tools/blinkpy/common/net/web.py
+++ b/third_party/blink/tools/blinkpy/common/net/web.py
@@ -26,6 +26,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import gzip
+
+import six
 from six.moves import urllib
 
 from blinkpy.common.net.network_transaction import NetworkTransaction
@@ -41,8 +44,22 @@
             return self.http_error_301(req, fp, 301, msg, headers)
 
     def get_binary(self, url, return_none_on_404=False):
-        return NetworkTransaction(return_none_on_404=return_none_on_404).run(
-            lambda: self.request('GET', url).read())
+        def make_request():
+            response = self.request('GET',
+                                    url,
+                                    headers={'Accept-Encoding': 'gzip'})
+            if response.headers.get('Content-Encoding') == 'gzip':
+                # Wrap the HTTP response, which is not fully file-like.
+                # Unfortunately, `six` does not provide `io.BufferedRandom`, so
+                # we need to read the entire payload up-front (may pose a
+                # performance issue).
+                buf = six.BytesIO(response.read())
+                gzip_decoder = gzip.GzipFile(fileobj=buf)
+                return gzip_decoder.read()
+            return response.read()
+
+        return NetworkTransaction(
+            return_none_on_404=return_none_on_404).run(make_request)
 
     def request(self, method, url, data=None, headers=None):
         opener = urllib.request.build_opener(Web._HTTPRedirectHandler2)