update_engine: Add Squashfs (phase 1)
Currently the Android container in CrOS is a Squashfs image (around
500MB) and the update engine treats it as one single file. This patch
uses -map flag in 'unsquashfs' to get the location of files inside the
Squashfs image. and creates a list of files that can be used in payload
generator to create a delta. This patch adds support for both Squashfs
file system and its appropriate unittests. However, we do not activate
it in this patch, further patches will activate and use it in the
payload generator.
BUG=chromium:767120
TEST=cros_workon_make --board=amd64-generic --test update_engine;
CQ-DEPEND=CL:650518,CL:686034
Change-Id: I02179727cd736661704d0ca2d9f881754b332b36
Reviewed-on: https://chromium-review.googlesource.com/636105
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
diff --git a/Android.mk b/Android.mk
index 656aba8..dad0ed7 100644
--- a/Android.mk
+++ b/Android.mk
@@ -642,6 +642,7 @@
payload_generator/payload_generation_config.cc \
payload_generator/payload_signer.cc \
payload_generator/raw_filesystem.cc \
+ payload_generator/squashfs_filesystem.cc \
payload_generator/tarjan.cc \
payload_generator/topological_sort.cc \
payload_generator/xz_android.cc
@@ -951,6 +952,7 @@
payload_generator/payload_file_unittest.cc \
payload_generator/payload_generation_config_unittest.cc \
payload_generator/payload_signer_unittest.cc \
+ payload_generator/squashfs_filesystem_unittest.cc \
payload_generator/tarjan_unittest.cc \
payload_generator/topological_sort_unittest.cc \
payload_generator/zip_unittest.cc \
diff --git a/payload_generator/squashfs_filesystem.cc b/payload_generator/squashfs_filesystem.cc
new file mode 100644
index 0000000..54a2c1a
--- /dev/null
+++ b/payload_generator/squashfs_filesystem.cc
@@ -0,0 +1,280 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+
+#include <fcntl.h>
+
+#include <algorithm>
+#include <string>
+
+#include <base/files/file_util.h>
+#include <base/logging.h>
+#include <base/strings/string_number_conversions.h>
+#include <base/strings/string_split.h>
+#include <brillo/streams/file_stream.h>
+
+#include "update_engine/common/subprocess.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/update_metadata.pb.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace chromeos_update_engine {
+
+namespace {
+
+Extent ExtentForBytes(uint64_t block_size,
+ uint64_t start_bytes,
+ uint64_t size_bytes) {
+ uint64_t start_block = start_bytes / block_size;
+ uint64_t end_block = (start_bytes + size_bytes + block_size - 1) / block_size;
+ return ExtentForRange(start_block, end_block - start_block);
+}
+
+// The size of the squashfs super block.
+constexpr size_t kSquashfsSuperBlockSize = 96;
+constexpr uint64_t kSquashfsCompressedBit = 1 << 24;
+
+bool ReadSquashfsHeader(const brillo::Blob blob,
+ SquashfsFilesystem::SquashfsHeader* header) {
+ if (blob.size() < kSquashfsSuperBlockSize) {
+ return false;
+ }
+
+ memcpy(&header->magic, blob.data(), 4);
+ memcpy(&header->block_size, blob.data() + 12, 4);
+ memcpy(&header->compression_type, blob.data() + 20, 2);
+ memcpy(&header->major_version, blob.data() + 28, 2);
+ return true;
+}
+
+bool CheckHeader(const SquashfsFilesystem::SquashfsHeader& header) {
+ return header.magic == 0x73717368 && header.major_version == 4;
+}
+
+bool GetFileMapContent(const string& sqfs_path, string* map) {
+ // Create a tmp file
+ string map_file;
+ TEST_AND_RETURN_FALSE(
+ utils::MakeTempFile("squashfs_file_map.XXXXXX", &map_file, nullptr));
+ ScopedPathUnlinker map_unlinker(map_file);
+
+ // Run unsquashfs to get the system file map.
+ // unsquashfs -m <map-file> <squashfs-file>
+ vector<string> cmd = {"unsquashfs", "-m", map_file, sqfs_path};
+ string stdout;
+ int exit_code;
+ if (!Subprocess::SynchronousExec(cmd, &exit_code, &stdout) ||
+ exit_code != 0) {
+ LOG(ERROR) << "Failed to run unsquashfs -m. The stdout content was: "
+ << stdout;
+ return false;
+ }
+ TEST_AND_RETURN_FALSE(utils::ReadFile(map_file, map));
+ return true;
+}
+
+} // namespace
+
+bool SquashfsFilesystem::Init(const string& map,
+ size_t size,
+ const SquashfsHeader& header) {
+ size_ = size;
+ // Reading files map. For the format of the file map look at the comments for
+ // |CreateFromFileMap()|.
+ auto lines = base::SplitStringPiece(map,
+ "\n",
+ base::WhitespaceHandling::KEEP_WHITESPACE,
+ base::SplitResult::SPLIT_WANT_NONEMPTY);
+ for (const auto& line : lines) {
+ auto splits =
+ base::SplitStringPiece(line,
+ " \t",
+ base::WhitespaceHandling::TRIM_WHITESPACE,
+ base::SplitResult::SPLIT_WANT_NONEMPTY);
+ // Only filename is invalid.
+ TEST_AND_RETURN_FALSE(splits.size() > 1);
+ uint64_t start;
+ TEST_AND_RETURN_FALSE(base::StringToUint64(splits[1], &start));
+ uint64_t cur_offset = start;
+ for (size_t i = 2; i < splits.size(); ++i) {
+ uint64_t blk_size;
+ TEST_AND_RETURN_FALSE(base::StringToUint64(splits[i], &blk_size));
+ // TODO(ahassani): For puffin push it into a proper list if uncompressed.
+ auto new_blk_size = blk_size & ~kSquashfsCompressedBit;
+ TEST_AND_RETURN_FALSE(new_blk_size <= header.block_size);
+ cur_offset += new_blk_size;
+ }
+
+ // If size is zero do not add the file.
+ if (cur_offset - start > 0) {
+ File file;
+ file.name = splits[0].as_string();
+ file.extents = {ExtentForBytes(kBlockSize, start, cur_offset - start)};
+ files_.emplace_back(file);
+ }
+ }
+
+ // Sort all files by their offset in the squashfs.
+ std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
+ return a.extents[0].start_block() < b.extents[0].start_block();
+ });
+ // If there is any overlap between two consecutive extents, remove them. Here
+ // we are assuming all files have exactly one extent. If this assumption
+ // changes then this implementation needs to change too.
+ for (auto first = files_.begin(), second = first + 1;
+ first != files_.end() && second != files_.end();
+ second = first + 1) {
+ auto first_begin = first->extents[0].start_block();
+ auto first_end = first_begin + first->extents[0].num_blocks();
+ auto second_begin = second->extents[0].start_block();
+ auto second_end = second_begin + second->extents[0].num_blocks();
+ // Remove the first file if the size is zero.
+ if (first_end == first_begin) {
+ first = files_.erase(first);
+ } else if (first_end > second_begin) { // We found a collision.
+ if (second_end <= first_end) {
+ // Second file is inside the first file, remove the second file.
+ second = files_.erase(second);
+ } else if (first_begin == second_begin) {
+ // First file is inside the second file, remove the first file.
+ first = files_.erase(first);
+ } else {
+ // Remove overlapping extents from the first file.
+ first->extents[0].set_num_blocks(second_begin - first_begin);
+ ++first;
+ }
+ } else {
+ ++first;
+ }
+ }
+
+ // Find all the metadata including superblock and add them to the list of
+ // files.
+ ExtentRanges file_extents;
+ for (const auto& file : files_) {
+ file_extents.AddExtents(file.extents);
+ }
+ vector<Extent> full = {
+ ExtentForRange(0, (size_ + kBlockSize - 1) / kBlockSize)};
+ auto metadata_extents = FilterExtentRanges(full, file_extents);
+ // For now there should be at most two extents. One for superblock and one for
+ // metadata at the end. Just create appropriate files with <metadata-i> name.
+ // We can add all these extents as one metadata too, but that violates the
+ // contiguous write optimization.
+ for (size_t i = 0; i < metadata_extents.size(); i++) {
+ File file;
+ file.name = "<metadata-" + std::to_string(i) + ">";
+ file.extents = {metadata_extents[i]};
+ files_.emplace_back(file);
+ }
+
+ // Do one last sort before returning.
+ std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
+ return a.extents[0].start_block() < b.extents[0].start_block();
+ });
+ return true;
+}
+
+unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile(
+ const string& sqfs_path) {
+ if (sqfs_path.empty())
+ return nullptr;
+
+ brillo::StreamPtr sqfs_file =
+ brillo::FileStream::Open(base::FilePath(sqfs_path),
+ brillo::Stream::AccessMode::READ,
+ brillo::FileStream::Disposition::OPEN_EXISTING,
+ nullptr);
+ if (!sqfs_file) {
+ LOG(ERROR) << "Unable to open " << sqfs_path << " for reading.";
+ return nullptr;
+ }
+
+ SquashfsHeader header;
+ brillo::Blob blob(kSquashfsSuperBlockSize);
+ if (!sqfs_file->ReadAllBlocking(blob.data(), blob.size(), nullptr)) {
+ LOG(ERROR) << "Unable to read from file: " << sqfs_path;
+ return nullptr;
+ }
+ if (!ReadSquashfsHeader(blob, &header) || !CheckHeader(header)) {
+ // This is not necessary an error.
+ return nullptr;
+ }
+
+ // Read the map file.
+ string filemap;
+ if (!GetFileMapContent(sqfs_path, &filemap)) {
+ LOG(ERROR) << "Failed to produce squashfs map file: " << sqfs_path;
+ return nullptr;
+ }
+
+ unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
+ if (!sqfs->Init(filemap, sqfs_file->GetSize(), header)) {
+ LOG(ERROR) << "Failed to initialized the Squashfs file system";
+ return nullptr;
+ }
+ // TODO(ahassani): Add a function that initializes the puffin related extents.
+ return sqfs;
+}
+
+unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFileMap(
+ const string& filemap, size_t size, const SquashfsHeader& header) {
+ if (!CheckHeader(header)) {
+ LOG(ERROR) << "Invalid Squashfs super block!";
+ return nullptr;
+ }
+
+ unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
+ if (!sqfs->Init(filemap, size, header)) {
+ LOG(ERROR) << "Failed to initialize the Squashfs file system using filemap";
+ return nullptr;
+ }
+ // TODO(ahassani): Add a function that initializes the puffin related extents.
+ return sqfs;
+}
+
+size_t SquashfsFilesystem::GetBlockSize() const {
+ return kBlockSize;
+}
+
+size_t SquashfsFilesystem::GetBlockCount() const {
+ return size_ / kBlockSize;
+}
+
+bool SquashfsFilesystem::GetFiles(vector<File>* files) const {
+ files->insert(files->end(), files_.begin(), files_.end());
+ return true;
+}
+
+bool SquashfsFilesystem::LoadSettings(brillo::KeyValueStore* store) const {
+ // Settings not supported in squashfs.
+ LOG(ERROR) << "squashfs doesn't support LoadSettings().";
+ return false;
+}
+
+bool SquashfsFilesystem::IsSquashfsImage(const brillo::Blob& blob) {
+ SquashfsHeader header;
+ return ReadSquashfsHeader(blob, &header) && CheckHeader(header);
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_generator/squashfs_filesystem.h b/payload_generator/squashfs_filesystem.h
new file mode 100644
index 0000000..aceebb8
--- /dev/null
+++ b/payload_generator/squashfs_filesystem.h
@@ -0,0 +1,114 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// This class implements a FilesystemInterface, which lets the caller obtain
+// basic information about what files are in the filesystem and where are they
+// located in the disk, but not full access to the uncompressed contents of
+// these files. This class uses the definitions found in
+// fs/squashfs/squashfs_fs.h in the kernel header tree. This class supports
+// squashfs version 4 in little-endian format only.
+
+#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
+#define UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <brillo/secure_blob.h>
+
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/filesystem_interface.h"
+
+namespace chromeos_update_engine {
+
+class SquashfsFilesystem : public FilesystemInterface {
+ public:
+ // From an squashfs image we need: (offset, bytes)
+ // - magic: (0, 4)
+ // * Acceptable value is: 0x73717368
+ // - block size: (12, 4)
+ // - compression type: (20, 2)
+ // * 1 is for zlib, gzip
+ // - major number: (28, 2)
+ // * We only support version 4 for now.
+ struct SquashfsHeader {
+ uint32_t magic;
+ uint32_t block_size;
+ uint16_t compression_type;
+ uint16_t major_version;
+ };
+
+ ~SquashfsFilesystem() override = default;
+
+ // Creates the file system from the Squashfs file itself.
+ static std::unique_ptr<SquashfsFilesystem> CreateFromFile(
+ const std::string& sqfs_path);
+
+ // Creates the file system from a file map |filemap| which is a multi-line
+ // string with each line with the following format:
+ //
+ // file_path start_byte compressed_size_1 ... compressed_size_2
+ //
+ // file_path: The name of the file inside the Squashfs File.
+ // start_byte: The byte address of the start of the file.
+ // compressed_size_i: The compressed size of the ith block of the file.
+ //
+ // The 25th bit of compressed_size_i is set if the block is uncompressed.
+ // |size| is the size of the Squashfs image.
+ static std::unique_ptr<SquashfsFilesystem> CreateFromFileMap(
+ const std::string& filemap, size_t size, const SquashfsHeader& header);
+
+ // FilesystemInterface overrides.
+ size_t GetBlockSize() const override;
+ size_t GetBlockCount() const override;
+
+ // Returns one FilesystemInterface::File for every file (that is not added to
+ // fragments) in the squashfs image.
+ //
+ // It also returns the following metadata files:
+ // <fragment-i>: The ith fragment in the Sqauashfs file.
+ // <metadata-i>: The part of the file system that does not belong to any
+ // file. Normally, there is only two: one for superblock and
+ // one for the metadata at the end.
+ bool GetFiles(std::vector<File>* files) const override;
+
+ // Squashfs image does not support this yet.
+ bool LoadSettings(brillo::KeyValueStore* store) const override;
+
+ // Returns true if the first few bytes of a file indicates a valid Squashfs
+ // image. The size of the |blob| should be at least
+ // sizeof(SquashfsHeader) or for now 96 bytes.
+ static bool IsSquashfsImage(const brillo::Blob& blob);
+
+ private:
+ SquashfsFilesystem() = default;
+
+ // Initialize and populates the files in the file system.
+ bool Init(const std::string& map, size_t size, const SquashfsHeader& header);
+
+ // The size of the image in bytes.
+ size_t size_;
+
+ // All the files in the filesystem.
+ std::vector<File> files_;
+
+ DISALLOW_COPY_AND_ASSIGN(SquashfsFilesystem);
+};
+
+} // namespace chromeos_update_engine
+#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
diff --git a/payload_generator/squashfs_filesystem_unittest.cc b/payload_generator/squashfs_filesystem_unittest.cc
new file mode 100644
index 0000000..77f4ec9
--- /dev/null
+++ b/payload_generator/squashfs_filesystem_unittest.cc
@@ -0,0 +1,315 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <base/format_macros.h>
+#include <base/logging.h>
+#include <base/strings/string_number_conversions.h>
+#include <base/strings/string_util.h>
+#include <base/strings/stringprintf.h>
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+namespace chromeos_update_engine {
+
+using std::map;
+using std::set;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using test_utils::GetBuildArtifactsPath;
+
+namespace {
+
+constexpr uint64_t kTestBlockSize = 4096;
+constexpr uint64_t kTestSqfsBlockSize = 1 << 15;
+
+// Checks that all the blocks in |extents| are in the range [0, total_blocks).
+void ExpectBlocksInRange(const vector<Extent>& extents, uint64_t total_blocks) {
+ for (const Extent& extent : extents) {
+ EXPECT_LE(0U, extent.start_block());
+ EXPECT_LE(extent.start_block() + extent.num_blocks(), total_blocks);
+ }
+}
+
+SquashfsFilesystem::SquashfsHeader GetSimpleHeader() {
+ // These properties are enough for now. Add more as needed.
+ return {
+ .magic = 0x73717368,
+ .block_size = kTestSqfsBlockSize,
+ .compression_type = 1, // For gzip.
+ .major_version = 4,
+ };
+}
+
+} // namespace
+
+class SquashfsFilesystemTest : public ::testing::Test {
+ public:
+ void CheckSquashfs(const unique_ptr<SquashfsFilesystem>& fs) {
+ ASSERT_TRUE(fs);
+ EXPECT_EQ(kTestBlockSize, fs->GetBlockSize());
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+
+ map<string, FilesystemInterface::File> map_files;
+ for (const auto& file : files) {
+ EXPECT_EQ(map_files.end(), map_files.find(file.name))
+ << "File " << file.name << " repeated in the list.";
+ map_files[file.name] = file;
+ ExpectBlocksInRange(file.extents, fs->GetBlockCount());
+ }
+
+ // Checking the sortness.
+ EXPECT_TRUE(std::is_sorted(files.begin(),
+ files.end(),
+ [](const FilesystemInterface::File& a,
+ const FilesystemInterface::File& b) {
+ return a.extents[0].start_block() <
+ b.extents[0].start_block();
+ }));
+
+ auto overlap_check = [](const FilesystemInterface::File& a,
+ const FilesystemInterface::File& b) {
+ // Return true if overlapping.
+ return a.extents[0].start_block() + a.extents[0].num_blocks() >
+ b.extents[0].start_block();
+ };
+ // Check files are not overlapping.
+ EXPECT_EQ(std::adjacent_find(files.begin(), files.end(), overlap_check),
+ files.end());
+ }
+};
+
+TEST_F(SquashfsFilesystemTest, EmptyFilesystemTest) {
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
+ GetBuildArtifactsPath("gen/disk_sqfs_empty.img"));
+ CheckSquashfs(fs);
+
+ // Even an empty squashfs filesystem is rounded up to 4K.
+ EXPECT_EQ(4096 / kTestBlockSize, fs->GetBlockCount());
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+
+ FilesystemInterface::File file;
+ file.name = "<metadata-0>";
+ file.extents.emplace_back();
+ file.extents[0].set_start_block(0);
+ file.extents[0].set_num_blocks(1);
+ EXPECT_EQ(files[0].name, file.name);
+ EXPECT_EQ(files[0].extents, file.extents);
+}
+
+TEST_F(SquashfsFilesystemTest, DefaultFilesystemTest) {
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
+ GetBuildArtifactsPath("gen/disk_sqfs_default.img"));
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+
+ FilesystemInterface::File file;
+ file.name = "<fragment-0>";
+ file.extents.emplace_back();
+ file.extents[0].set_start_block(0);
+ file.extents[0].set_num_blocks(1);
+ EXPECT_EQ(files[0].name, file.name);
+ EXPECT_EQ(files[0].extents, file.extents);
+}
+
+TEST_F(SquashfsFilesystemTest, SimpleFileMapTest) {
+ string filemap = R"(dir1/file1 96 4000
+ dir1/file2 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ EXPECT_EQ(files.size(), 2u);
+}
+
+TEST_F(SquashfsFilesystemTest, FileMapZeroSizeFileTest) {
+ // The second file's size is zero.
+ string filemap = R"(dir1/file1 96 4000
+ dir1/file2 4096
+ dir1/file3 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ // The second and third files are removed. The file with size zero is removed.
+ EXPECT_EQ(files.size(), 2u);
+}
+
+// Testing the compressed bit.
+TEST_F(SquashfsFilesystemTest, CompressedBitTest) {
+ string filemap = "dir1/file1 0 " + std::to_string(4000 | (1 << 24)) + "\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 1u);
+}
+
+// Test overlap.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles1Test) {
+ string filemap = R"(file1 0 6000
+ file2 5000 5000)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 3, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 2u);
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 1u);
+ EXPECT_EQ(files[1].extents[0].num_blocks(), 2u);
+}
+
+// Test overlap, first inside second.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles2Test) {
+ string filemap = R"(file1 0 4000
+ file2 0 6000)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].name, "file2");
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 2u);
+}
+
+// Test overlap, second inside first.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles3Test) {
+ string filemap = R"(file1 0 8000
+ file2 100 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].name, "file1");
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 2u);
+}
+
+// Fail a line with only one argument.
+TEST_F(SquashfsFilesystemTest, FailOnlyFileNameTest) {
+ string filemap = "dir1/file1\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail a line with space separated filen name
+TEST_F(SquashfsFilesystemTest, FailSpaceInFileNameTest) {
+ string filemap = "dir1 file1 0 10\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail empty line
+TEST_F(SquashfsFilesystemTest, FailEmptyLineTest) {
+ // The second file's size is zero.
+ string filemap = R"(
+ /t
+ dir1/file3 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail on bad magic or major
+TEST_F(SquashfsFilesystemTest, FailBadMagicOrMajorTest) {
+ string filemap = "dir1/file1 0 10\n";
+ auto header = GetSimpleHeader();
+ header.magic = 1;
+ EXPECT_FALSE(
+ SquashfsFilesystem::CreateFromFileMap(filemap, kTestBlockSize, header));
+
+ header = GetSimpleHeader();
+ header.major_version = 3;
+ EXPECT_FALSE(
+ SquashfsFilesystem::CreateFromFileMap(filemap, kTestBlockSize, header));
+}
+
+// Fail size with larger than block_size
+TEST_F(SquashfsFilesystemTest, FailLargerThanBlockSizeTest) {
+ string filemap = "file1 0 " + std::to_string(kTestSqfsBlockSize + 1) + "\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Test is squashfs image.
+TEST_F(SquashfsFilesystemTest, IsSquashfsImageTest) {
+ // Some sample from a recent squashfs file.
+ brillo::Blob super_block = {
+ 0x68, 0x73, 0x71, 0x73, 0x59, 0x05, 0x00, 0x00, 0x09, 0x3a, 0x89, 0x58,
+ 0x00, 0x00, 0x02, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x01, 0x00, 0x11, 0x00,
+ 0xc0, 0x00, 0x06, 0x00, 0x04, 0x00, 0x00, 0x00, 0x89, 0x18, 0xf7, 0x7c,
+ 0x00, 0x00, 0x00, 0x00, 0x2e, 0x33, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00,
+ 0x3a, 0x30, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00, 0x16, 0x33, 0xcd, 0x16,
+ 0x00, 0x00, 0x00, 0x00, 0x07, 0x62, 0xcc, 0x16, 0x00, 0x00, 0x00, 0x00,
+ 0x77, 0xe6, 0xcc, 0x16, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x25, 0xcd, 0x16,
+ 0x00, 0x00, 0x00, 0x00, 0x10, 0x30, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00};
+
+ EXPECT_TRUE(SquashfsFilesystem::IsSquashfsImage(super_block));
+
+ // Bad magic
+ auto bad_super_block = super_block;
+ bad_super_block[1] = 0x02;
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+
+ // Bad major
+ bad_super_block = super_block;
+ bad_super_block[28] = 0x03;
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+
+ // Small size;
+ bad_super_block = super_block;
+ bad_super_block.resize(10);
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+}
+
+} // namespace chromeos_update_engine
diff --git a/sample_images/generate_images.sh b/sample_images/generate_images.sh
index 6a0d1ea..8478682 100755
--- a/sample_images/generate_images.sh
+++ b/sample_images/generate_images.sh
@@ -26,12 +26,15 @@
# cleanup <path>
# Unmount and remove the mountpoint <path>
cleanup() {
- if ! sudo umount "$1" 2>/dev/null; then
- if mountpoint -q "$1"; then
- sync && sudo umount "$1"
+ local path="$1"
+ if ! sudo umount "${path}" 2>/dev/null; then
+ if mountpoint -q "${path}"; then
+ sync && sudo umount "${path}"
fi
fi
- rmdir "$1"
+ if [ -n "${path}" ]; then
+ sudo rm -rf "${path}"
+ fi
}
# add_files_default <mntdir> <block_size>
@@ -203,10 +206,11 @@
# generate_fs <filename> <kind> <size> [block_size] [block_groups]
generate_fs() {
local filename="$1"
- local kind="$2"
- local size="$3"
- local block_size="${4:-4096}"
- local block_groups="${5:-}"
+ local type="$2"
+ local kind="$3"
+ local size="$4"
+ local block_size="${5:-4096}"
+ local block_groups="${6:-}"
local mkfs_opts=( -q -F -b "${block_size}" -L "ROOT-TEST" -t ext2 )
if [[ -n "${block_groups}" ]]; then
@@ -215,16 +219,17 @@
local mntdir=$(mktemp --tmpdir -d generate_ext2.XXXXXX)
trap 'cleanup "${mntdir}"; rm -f "${filename}"' INT TERM EXIT
-
# Cleanup old image.
if [[ -e "${filename}" ]]; then
rm -f "${filename}"
fi
- truncate --size="${size}" "${filename}"
- mkfs.ext2 "${mkfs_opts[@]}" "${filename}"
- sudo mount "${filename}" "${mntdir}" -o loop
+ if [[ "${type}" == "ext2" ]]; then
+ truncate --size="${size}" "${filename}"
+ mkfs.ext2 "${mkfs_opts[@]}" "${filename}"
+ sudo mount "${filename}" "${mntdir}" -o loop
+ fi
case "${kind}" in
unittest)
add_files_ue_settings "${mntdir}" "${block_size}"
@@ -237,6 +242,10 @@
;;
esac
+ if [[ "${type}" == "sqfs" ]]; then
+ mksquashfs "${mntdir}" "${filename}"
+ fi
+
cleanup "${mntdir}"
trap - INT TERM EXIT
}
@@ -253,10 +262,14 @@
main() {
# Add more sample images here.
- generate_image disk_ext2_1k default $((1024 * 1024)) 1024
- generate_image disk_ext2_4k default $((1024 * 4096)) 4096
- generate_image disk_ext2_4k_empty empty $((1024 * 4096)) 4096
- generate_image disk_ext2_unittest unittest $((1024 * 4096)) 4096
+ generate_image disk_ext2_1k ext2 default $((1024 * 1024)) 1024
+ generate_image disk_ext2_4k ext2 default $((1024 * 4096)) 4096
+ generate_image disk_ext2_4k_empty ext2 empty $((1024 * 4096)) 4096
+ generate_image disk_ext2_unittest ext2 unittest $((1024 * 4096)) 4096
+
+ # Add squashfs sample images.
+ generate_image disk_sqfs_empty sqfs empty $((1024 * 4096)) 4096
+ generate_image disk_sqfs_default sqfs default $((1024 * 4096)) 4096
# Generate the tarball and delete temporary images.
echo "Packing tar file sample_images.tar.bz2"
diff --git a/sample_images/sample_images.tar.bz2 b/sample_images/sample_images.tar.bz2
index 72f4eb5..6215482 100644
--- a/sample_images/sample_images.tar.bz2
+++ b/sample_images/sample_images.tar.bz2
Binary files differ
diff --git a/update_engine.gyp b/update_engine.gyp
index 0e0ba9f..e8e1fe9 100644
--- a/update_engine.gyp
+++ b/update_engine.gyp
@@ -412,6 +412,7 @@
'payload_generator/payload_generation_config.cc',
'payload_generator/payload_signer.cc',
'payload_generator/raw_filesystem.cc',
+ 'payload_generator/squashfs_filesystem.cc',
'payload_generator/tarjan.cc',
'payload_generator/topological_sort.cc',
'payload_generator/xz_chromeos.cc',
@@ -559,6 +560,7 @@
'payload_generator/payload_file_unittest.cc',
'payload_generator/payload_generation_config_unittest.cc',
'payload_generator/payload_signer_unittest.cc',
+ 'payload_generator/squashfs_filesystem_unittest.cc',
'payload_generator/tarjan_unittest.cc',
'payload_generator/topological_sort_unittest.cc',
'payload_generator/zip_unittest.cc',
