update_engine: De-duplicate zlib blocks of squashfs images

Having duplicate zlib blocks in the squashfs image (e.g. from two
hard linked files) will cause problems for processing puffin. Recently
ARC++ image introduced two files that where hard linked of each other
and this caused the squashfs parsing algorithm fail and the ARC++ image
does not get splitted into smaller files. Hence we got massive OOM and
Timeout problems in the builders. This CL, simpliy removes duplicate
zlib blocks when constructing the list of such items.

BUG=chromium:955145
TEST=cros_generate_update_payload --image gs://chromeos-releases/stable-channel/nautilus/11647.154.0/chromeos_11647.154.0_nautilus_recovery_stable-channel_mp-v2.bin --src_image gs://chromeos-releases/stable-channel/nautilus/10575.55.0/chromeos_10575.55.0_nautilus_recovery_stable-channel_mp.bin --output payload.bin

Change-Id: If4eb4731355e1180953a5b327be3a16ff13e6d96
Reviewed-on: https://chromium-review.googlesource.com/1578219
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Sen Jiang <senj@chromium.org>
1 file changed
tree: 128fbb5270250a37c5b570adf8bc034615c7dc90
  1. .clang-format
  2. .gitignore
  3. Android.bp
  4. COMMIT-QUEUE.ini
  5. CPPLINT.cfg
  6. MODULE_LICENSE_APACHE2
  7. NOTICE
  8. OWNERS
  9. PRESUBMIT.cfg
  10. PREUPLOAD.cfg
  11. UpdateEngine.conf
  12. binder_bindings/
  13. binder_service_android.cc
  14. binder_service_android.h
  15. binder_service_brillo.cc
  16. binder_service_brillo.h
  17. boot_control_android.cc
  18. boot_control_android.h
  19. boot_control_android_unittest.cc
  20. boot_control_chromeos.cc
  21. boot_control_chromeos.h
  22. boot_control_chromeos_unittest.cc
  23. certificate_checker.cc
  24. certificate_checker.h
  25. certificate_checker_unittest.cc
  26. chrome_browser_proxy_resolver.cc
  27. chrome_browser_proxy_resolver.h
  28. client_library/
  29. common/
  30. common_service.cc
  31. common_service.h
  32. common_service_unittest.cc
  33. connection_manager.cc
  34. connection_manager.h
  35. connection_manager_android.cc
  36. connection_manager_android.h
  37. connection_manager_interface.h
  38. connection_manager_unittest.cc
  39. connection_utils.cc
  40. connection_utils.h
  41. daemon.cc
  42. daemon.h
  43. daemon_state_android.cc
  44. daemon_state_android.h
  45. daemon_state_interface.h
  46. dbus_bindings/
  47. dbus_connection.cc
  48. dbus_connection.h
  49. dbus_service.cc
  50. dbus_service.h
  51. dbus_test_utils.h
  52. dlcservice_chromeos.cc
  53. dlcservice_chromeos.h
  54. dynamic_partition_control_android.cc
  55. dynamic_partition_control_android.h
  56. dynamic_partition_control_interface.h
  57. fake_file_writer.h
  58. fake_p2p_manager.h
  59. fake_p2p_manager_configuration.h
  60. fake_shill_proxy.cc
  61. fake_shill_proxy.h
  62. fake_system_state.cc
  63. fake_system_state.h
  64. fuzz/
  65. generate_pc_file.sh
  66. hardware_android.cc
  67. hardware_android.h
  68. hardware_chromeos.cc
  69. hardware_chromeos.h
  70. hardware_chromeos_unittest.cc
  71. image_properties.h
  72. image_properties_android.cc
  73. image_properties_android_unittest.cc
  74. image_properties_chromeos.cc
  75. image_properties_chromeos_unittest.cc
  76. init/
  77. libcurl_http_fetcher.cc
  78. libcurl_http_fetcher.h
  79. libupdate_engine-client-test.pc.in
  80. libupdate_engine-client.pc.in
  81. local_coverage_rate
  82. main.cc
  83. metrics_constants.h
  84. metrics_reporter_android.cc
  85. metrics_reporter_android.h
  86. metrics_reporter_interface.h
  87. metrics_reporter_omaha.cc
  88. metrics_reporter_omaha.h
  89. metrics_reporter_omaha_unittest.cc
  90. metrics_reporter_stub.cc
  91. metrics_reporter_stub.h
  92. metrics_utils.cc
  93. metrics_utils.h
  94. metrics_utils_unittest.cc
  95. mock_boot_control_hal.h
  96. mock_certificate_checker.h
  97. mock_connection_manager.h
  98. mock_dynamic_partition_control.h
  99. mock_file_writer.h
  100. mock_metrics_reporter.h
  101. mock_omaha_request_params.h
  102. mock_p2p_manager.h
  103. mock_payload_state.h
  104. mock_power_manager.h
  105. mock_service_observer.h
  106. mock_update_attempter.h
  107. network_selector.h
  108. network_selector_android.cc
  109. network_selector_android.h
  110. network_selector_interface.h
  111. network_selector_stub.cc
  112. network_selector_stub.h
  113. omaha_request_action.cc
  114. omaha_request_action.h
  115. omaha_request_action_fuzzer.cc
  116. omaha_request_action_unittest.cc
  117. omaha_request_builder_xml.cc
  118. omaha_request_builder_xml.h
  119. omaha_request_builder_xml_unittest.cc
  120. omaha_request_params.cc
  121. omaha_request_params.h
  122. omaha_request_params_unittest.cc
  123. omaha_response.h
  124. omaha_response_handler_action.cc
  125. omaha_response_handler_action.h
  126. omaha_response_handler_action_unittest.cc
  127. omaha_utils.cc
  128. omaha_utils.h
  129. omaha_utils_unittest.cc
  130. p2p_manager.cc
  131. p2p_manager.h
  132. p2p_manager_unittest.cc
  133. parcelable_update_engine_status.cc
  134. parcelable_update_engine_status.h
  135. parcelable_update_engine_status_unittest.cc
  136. payload_consumer/
  137. payload_generator/
  138. payload_state.cc
  139. payload_state.h
  140. payload_state_interface.h
  141. payload_state_unittest.cc
  142. power_manager_android.cc
  143. power_manager_android.h
  144. power_manager_chromeos.cc
  145. power_manager_chromeos.h
  146. power_manager_interface.h
  147. pylintrc
  148. real_system_state.cc
  149. real_system_state.h
  150. run_unittests
  151. sample_images/
  152. sample_omaha_v3_response.xml
  153. scripts/
  154. service_delegate_android_interface.h
  155. service_observer_interface.h
  156. shill_proxy.cc
  157. shill_proxy.h
  158. shill_proxy_interface.h
  159. sideload_main.cc
  160. system_state.h
  161. tar_bunzip2.gypi
  162. test_http_server.cc
  163. test_subprocess.cc
  164. testrunner.cc
  165. unittest_key.pem
  166. unittest_key2.pem
  167. update_attempter.cc
  168. update_attempter.h
  169. update_attempter_android.cc
  170. update_attempter_android.h
  171. update_attempter_android_unittest.cc
  172. update_attempter_unittest.cc
  173. update_boot_flags_action.cc
  174. update_boot_flags_action.h
  175. update_boot_flags_action_unittest.cc
  176. update_engine-client.gyp
  177. update_engine.conf
  178. update_engine.gyp
  179. update_engine.rc
  180. update_engine/
  181. update_engine_client.cc
  182. update_engine_client_android.cc
  183. update_manager/
  184. update_metadata.proto
  185. update_payload_key/
  186. update_status_utils.cc
  187. update_status_utils.h