update_engine: De-duplicate zlib blocks of squashfs images

Having duplicate zlib blocks in the squashfs image (e.g. from two
hard linked files) will cause problems for processing puffin. Recently
ARC++ image introduced two files that where hard linked of each other
and this caused the squashfs parsing algorithm fail and the ARC++ image
does not get splitted into smaller files. Hence we got massive OOM and
Timeout problems in the builders. This CL, simpliy removes duplicate
zlib blocks when constructing the list of such items.

BUG=chromium:955145
TEST=cros_generate_update_payload --image gs://chromeos-releases/stable-channel/nautilus/11647.154.0/chromeos_11647.154.0_nautilus_recovery_stable-channel_mp-v2.bin --src_image gs://chromeos-releases/stable-channel/nautilus/10575.55.0/chromeos_10575.55.0_nautilus_recovery_stable-channel_mp.bin --output payload.bin

Change-Id: If4eb4731355e1180953a5b327be3a16ff13e6d96
Reviewed-on: https://chromium-review.googlesource.com/1578219
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Sen Jiang <senj@chromium.org>
1 file changed
tree: 128fbb5270250a37c5b570adf8bc034615c7dc90
  1. binder_bindings/
  2. client_library/
  3. common/
  4. dbus_bindings/
  5. fuzz/
  6. init/
  7. payload_consumer/
  8. payload_generator/
  9. sample_images/
  10. scripts/
  11. update_engine/
  12. update_manager/
  13. update_payload_key/
  14. .clang-format
  15. .gitignore
  16. Android.bp
  17. binder_service_android.cc
  18. binder_service_android.h
  19. binder_service_brillo.cc
  20. binder_service_brillo.h
  21. boot_control_android.cc
  22. boot_control_android.h
  23. boot_control_android_unittest.cc
  24. boot_control_chromeos.cc
  25. boot_control_chromeos.h
  26. boot_control_chromeos_unittest.cc
  27. certificate_checker.cc
  28. certificate_checker.h
  29. certificate_checker_unittest.cc
  30. chrome_browser_proxy_resolver.cc
  31. chrome_browser_proxy_resolver.h
  32. COMMIT-QUEUE.ini
  33. common_service.cc
  34. common_service.h
  35. common_service_unittest.cc
  36. connection_manager.cc
  37. connection_manager.h
  38. connection_manager_android.cc
  39. connection_manager_android.h
  40. connection_manager_interface.h
  41. connection_manager_unittest.cc
  42. connection_utils.cc
  43. connection_utils.h
  44. CPPLINT.cfg
  45. daemon.cc
  46. daemon.h
  47. daemon_state_android.cc
  48. daemon_state_android.h
  49. daemon_state_interface.h
  50. dbus_connection.cc
  51. dbus_connection.h
  52. dbus_service.cc
  53. dbus_service.h
  54. dbus_test_utils.h
  55. dlcservice_chromeos.cc
  56. dlcservice_chromeos.h
  57. dynamic_partition_control_android.cc
  58. dynamic_partition_control_android.h
  59. dynamic_partition_control_interface.h
  60. fake_file_writer.h
  61. fake_p2p_manager.h
  62. fake_p2p_manager_configuration.h
  63. fake_shill_proxy.cc
  64. fake_shill_proxy.h
  65. fake_system_state.cc
  66. fake_system_state.h
  67. generate_pc_file.sh
  68. hardware_android.cc
  69. hardware_android.h
  70. hardware_chromeos.cc
  71. hardware_chromeos.h
  72. hardware_chromeos_unittest.cc
  73. image_properties.h
  74. image_properties_android.cc
  75. image_properties_android_unittest.cc
  76. image_properties_chromeos.cc
  77. image_properties_chromeos_unittest.cc
  78. libcurl_http_fetcher.cc
  79. libcurl_http_fetcher.h
  80. libupdate_engine-client-test.pc.in
  81. libupdate_engine-client.pc.in
  82. local_coverage_rate
  83. main.cc
  84. metrics_constants.h
  85. metrics_reporter_android.cc
  86. metrics_reporter_android.h
  87. metrics_reporter_interface.h
  88. metrics_reporter_omaha.cc
  89. metrics_reporter_omaha.h
  90. metrics_reporter_omaha_unittest.cc
  91. metrics_reporter_stub.cc
  92. metrics_reporter_stub.h
  93. metrics_utils.cc
  94. metrics_utils.h
  95. metrics_utils_unittest.cc
  96. mock_boot_control_hal.h
  97. mock_certificate_checker.h
  98. mock_connection_manager.h
  99. mock_dynamic_partition_control.h
  100. mock_file_writer.h
  101. mock_metrics_reporter.h
  102. mock_omaha_request_params.h
  103. mock_p2p_manager.h
  104. mock_payload_state.h
  105. mock_power_manager.h
  106. mock_service_observer.h
  107. mock_update_attempter.h
  108. MODULE_LICENSE_APACHE2
  109. network_selector.h
  110. network_selector_android.cc
  111. network_selector_android.h
  112. network_selector_interface.h
  113. network_selector_stub.cc
  114. network_selector_stub.h
  115. NOTICE
  116. omaha_request_action.cc
  117. omaha_request_action.h
  118. omaha_request_action_fuzzer.cc
  119. omaha_request_action_unittest.cc
  120. omaha_request_builder_xml.cc
  121. omaha_request_builder_xml.h
  122. omaha_request_builder_xml_unittest.cc
  123. omaha_request_params.cc
  124. omaha_request_params.h
  125. omaha_request_params_unittest.cc
  126. omaha_response.h
  127. omaha_response_handler_action.cc
  128. omaha_response_handler_action.h
  129. omaha_response_handler_action_unittest.cc
  130. omaha_utils.cc
  131. omaha_utils.h
  132. omaha_utils_unittest.cc
  133. OWNERS
  134. p2p_manager.cc
  135. p2p_manager.h
  136. p2p_manager_unittest.cc
  137. parcelable_update_engine_status.cc
  138. parcelable_update_engine_status.h
  139. parcelable_update_engine_status_unittest.cc
  140. payload_state.cc
  141. payload_state.h
  142. payload_state_interface.h
  143. payload_state_unittest.cc
  144. power_manager_android.cc
  145. power_manager_android.h
  146. power_manager_chromeos.cc
  147. power_manager_chromeos.h
  148. power_manager_interface.h
  149. PRESUBMIT.cfg
  150. PREUPLOAD.cfg
  151. pylintrc
  152. real_system_state.cc
  153. real_system_state.h
  154. run_unittests
  155. sample_omaha_v3_response.xml
  156. service_delegate_android_interface.h
  157. service_observer_interface.h
  158. shill_proxy.cc
  159. shill_proxy.h
  160. shill_proxy_interface.h
  161. sideload_main.cc
  162. system_state.h
  163. tar_bunzip2.gypi
  164. test_http_server.cc
  165. test_subprocess.cc
  166. testrunner.cc
  167. unittest_key.pem
  168. unittest_key2.pem
  169. update_attempter.cc
  170. update_attempter.h
  171. update_attempter_android.cc
  172. update_attempter_android.h
  173. update_attempter_android_unittest.cc
  174. update_attempter_unittest.cc
  175. update_boot_flags_action.cc
  176. update_boot_flags_action.h
  177. update_boot_flags_action_unittest.cc
  178. update_engine-client.gyp
  179. update_engine.conf
  180. update_engine.gyp
  181. update_engine.rc
  182. update_engine_client.cc
  183. update_engine_client_android.cc
  184. update_metadata.proto
  185. update_status_utils.cc
  186. update_status_utils.h
  187. UpdateEngine.conf