| #!/usr/bin/env python |
| # Copyright 2011 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Apply gzip/deflate to separate chunks of data.""" |
| |
| import struct |
| import zlib |
| |
| GZIP_HEADER = ( |
| '\037\213' # magic header |
| '\010' # compression method |
| '\000' # flags (none) |
| '\000\000\000\000' # packed time (use zero) |
| '\002' |
| '\377') |
| |
| |
| def compress_chunks(uncompressed_chunks, use_gzip): |
| """Compress a list of data with gzip or deflate. |
| |
| The returned chunks may be used with HTTP chunked encoding. |
| |
| Args: |
| uncompressed_chunks: a list of strings |
| (e.g. ["this is the first chunk", "and the second"]) |
| use_gzip: if True, compress with gzip. Otherwise, use deflate. |
| |
| Returns: |
| [compressed_chunk_1, compressed_chunk_2, ...] |
| """ |
| if use_gzip: |
| size = 0 |
| crc = zlib.crc32("") & 0xffffffffL |
| compressor = zlib.compressobj( |
| 6, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) |
| else: |
| compressor = zlib.compressobj() |
| compressed_chunks = [] |
| last_index = len(uncompressed_chunks) - 1 |
| for index, data in enumerate(uncompressed_chunks): |
| chunk = '' |
| if use_gzip: |
| size += len(data) |
| crc = zlib.crc32(data, crc) & 0xffffffffL |
| if index == 0: |
| chunk += GZIP_HEADER |
| chunk += compressor.compress(data) |
| if index < last_index: |
| chunk += compressor.flush(zlib.Z_SYNC_FLUSH) |
| else: |
| chunk += (compressor.flush(zlib.Z_FULL_FLUSH) + |
| compressor.flush()) |
| if use_gzip: |
| chunk += (struct.pack("<L", long(crc)) + |
| struct.pack("<L", long(size))) |
| compressed_chunks.append(chunk) |
| return compressed_chunks |
| |
| |
| def uncompress_chunks(compressed_chunks, use_gzip): |
| """Uncompress a list of data compressed with gzip or deflate. |
| |
| Args: |
| compressed_chunks: a list of compressed data |
| use_gzip: if True, uncompress with gzip. Otherwise, use deflate. |
| |
| Returns: |
| [uncompressed_chunk_1, uncompressed_chunk_2, ...] |
| """ |
| if use_gzip: |
| decompress = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress |
| else: |
| decompress = zlib.decompressobj(-zlib.MAX_WBITS).decompress |
| return [decompress(c) for c in compressed_chunks] |