blob: a9ee4afcff1501c61274cf9a40a77a2591148288 [file] [log] [blame]
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import("//build/config/dcheck_always_on.gni")
import("//build/config/sanitizers/sanitizers.gni")
import("//testing/libfuzzer/fuzzer_test.gni")
import("//third_party/protobuf/proto_library.gni")
# Compile-time options passed to SQLite.
#
# These options are used when building our own SQLite library, which happens
# everywhere except on iOS. These compile-time options are exported via a
# public_config to all targets using SQLite, because they're needed by the
# sqlite.h header. To avoid name clashes (macro names are resolved using a
# global namespace), this block should only contain preprocessor macros that
# are unambiguously connected to SQLite.
#
# The vast majority of the macros here are documented at
# https://www.sqlite.org/compile.html
config("chromium_sqlite3_compile_options") {
defines = [
# Skip writing transaction rollback journals on f2fs.
# f2fs tends to be used on Android, and may be used on ChromeOS.
"SQLITE_ENABLE_BATCH_ATOMIC_WRITE",
"SQLITE_ENABLE_FTS3",
# New unicode61 tokenizer with built-in tables.
"SQLITE_DISABLE_FTS3_UNICODE",
# Chrome does not enable fts4, disable extra code.
"SQLITE_DISABLE_FTS4_DEFERRED",
"SQLITE_ENABLE_ICU",
# Defaults the secure_delete pragma to 1.
#
# This causes SQLite to overwrite all deleted information with zeroes,
# trading additional I/O for better privacy guarantees.
"SQLITE_SECURE_DELETE",
# TODO(pwnall): SQLite adds mutexes to protect structures which cross
# threads. In theory Chrome should be able to turn this to "2" which
# should give a slight speed boost. "2" is safe as long as a single
# connection is not used by more than one thread at a time.
"SQLITE_THREADSAFE=1",
# SQLite can spawn threads to sort in parallel if configured
# appropriately. Chrome doesn't configure SQLite for that, and would
# prefer to control distribution to worker threads.
"SQLITE_MAX_WORKER_THREADS=0",
# Allow 256MB mmap footprint per connection. Should not be too open-ended
# as that could cause memory fragmentation. 50MB encompasses the 99th
# percentile of Chrome databases in the wild.
# TODO(pwnall): A 64-bit-specific value could be 1G or more.
# TODO(pwnall): Figure out if exceeding this is costly.
"SQLITE_MAX_MMAP_SIZE=268435456",
# The default POSIX permissions for a newly created SQLite database.
#
# If unspecified, this defaults to 0644. All the data stored by Chrome is
# private, so our databases use stricter settings.
"SQLITE_DEFAULT_FILE_PERMISSIONS=0600",
# Needed by the SQL MemoryDumpProvider.
#
# Setting this to 1 is needed to collect the information reported by
# sqlite3_status64(SQLITE_STATUS_MEMORY_USED). Without this setting, the API
# still exists, but does not work as promised.
"SQLITE_DEFAULT_MEMSTATUS=1",
# Must match sql::Database::kDefaultPageSize.
"SQLITE_DEFAULT_PAGE_SIZE=4096",
# By default SQLite pre-allocates 100 pages of pcache data, which will not
# be released until the handle is closed. This is contrary to Chrome's
# memory-usage goals.
"SQLITE_DEFAULT_PCACHE_INITSZ=0",
# Some defines can affect the amalgamation. Those should be added to
# google_generate_amalgamation.sh, and the amalgamation re-generated.
# Usually this involves disabling features which include keywords or
# syntax, for instance SQLITE_OMIT_VIRTUALTABLE omits the virtual table
# syntax entirely. Missing an item usually results in syntax working but
# execution failing. Review:
# patched/src/parse.py
# patched/tool/mkkeywordhash.c
# The flags below are recommended in the SQLite documentation, and disable
# features Chrome doesn't use.
"SQLITE_LIKE_DOESNT_MATCH_BLOBS",
"SQLITE_OMIT_DEPRECATED",
"SQLITE_OMIT_PROGRESS_CALLBACK",
"SQLITE_OMIT_SHARED_CACHE",
"SQLITE_USE_ALLOCA",
# Chrome doesn't use the ANALYZE SQLite extension.
#
# ANALYZE [1] is non-standard, and currently performs a table scan to
# update statistics used by the query planner. Chrome uses straightforward
# database schemas which do not require the level of fine tuning provided
# by ANALYZE, and we generally cannot afford the I/O cost of the required
# table scans.
#
# [1] https://www.sqlite.org/lang_analyze.html
"SQLITE_OMIT_ANALYZE",
# Chrome initializes SQLite manually in //sql/connection.cc.
"SQLITE_OMIT_AUTOINIT",
# Chrome calls sqlite3_reset() correctly to reset prepared statements.
"SQLITE_OMIT_AUTORESET",
# Chromium does not use sqlite3_{get,free}_table().
# Chrome doesn't use sqlite3_compileoption_{used,get}().
"SQLITE_OMIT_COMPILEOPTION_DIAGS",
# Chrome doesn't ship the SQLite shell, so command auto-completion is not
# needed. Chrome developers who build the SQLite shell living at
# //third_party/sqlite:sqlite_shell for diagnostic purposes will have to
# live without auto-completion.
"SQLITE_OMIT_COMPLETE",
# Chrome does not use sqlite3_column_decltype().
"SQLITE_OMIT_DECLTYPE",
# EXPLAIN's output is not stable across releases [1], so it should not be
# used in Chrome. Skipping the EXPLAIN machinery also results in
# non-trivial binary savings.
#
# [1] https://www.sqlite.org/lang_explain.html
"SQLITE_OMIT_EXPLAIN",
# Chrome does not use sqlite3_{get,free}_table().
"SQLITE_OMIT_GET_TABLE",
# Chrome does not use sqlite3_{enable_}load_extension().
# Asides from giving us fairly minor code savings, this option disables code
# that breaks our method for renaming SQLite's exported symbols. Last,
# there's a tiny security benefit to knowing that WebSQL can't possibly
# reach extension loading code.
"SQLITE_OMIT_LOAD_EXTENSION",
# Chrome already depends on malloc being very efficient, so we disable
# SQLite's arena allocator.
"SQLITE_DEFAULT_LOOKASIDE=0,0",
"SQLITE_OMIT_LOOKASIDE",
# Chrome doesn't use TCL variables.
"SQLITE_OMIT_TCL_VARIABLE",
# The REINDEX statemnt is only useful if a collation sequence's definition
# changes [1]. Chrome never defines its own collation sequences [2, 3], so
# it never needs to call REINDEX.
#
# [1] https://www.sqlite.org/lang_reindex.html
# [2] https://www.sqlite.org/datatype3.html#collating_sequences
# [3] https://www.sqlite.org/c3ref/create_collation.html
"SQLITE_OMIT_REINDEX",
# Chrome doesn't use sqlite3_{profile,trace}().
"SQLITE_OMIT_TRACE",
# Chrome doesn't use UPSERT.
"SQLITE_OMIT_UPSERT",
# Chrome doesn't use window functions in SQL.
"SQLITE_OMIT_WINDOWFUNC",
# Uses isnan() in the C99 standard library.
"SQLITE_HAVE_ISNAN",
# Forces SQLite to only store temporary data in memory.
#
# This is the only supported setting on Android. Using it everywhere removes
# a source of cross-platform variation, and saves us from having to reason
# about SQLite storing data in the operating system's temporary directory.
"SQLITE_TEMP_STORE=3",
]
# On OSX, SQLite has extra logic for detecting the use of network
# filesystems (e.g., AFS, NFS) and for working around locking problems in
# these filesystems. This logic is gated by SQLITE_ENABLE_LOCKING_STYLE, which
# is 1 by default on OSX and iOS, and 0 everywhere else.
#
# When enabled, SQLITE_ENABLE_LOCKING_STYLE results in a compile-time warning
# on iOS. The recommended solution is to disable the flag on iOS, because
# iOS doesn't (yet?) have networked filesystems. Since we have to do this,
# might as well be explicit about the flag everywhere.
if (is_mac) {
defines += [ "SQLITE_ENABLE_LOCKING_STYLE=1" ]
} else {
defines += [ "SQLITE_ENABLE_LOCKING_STYLE=0" ]
}
if (using_sanitizer) {
defines += [
# Limit max length of data blobs and queries to 128 MB for fuzzing builds.
"SQLITE_MAX_LENGTH=128000000",
"SQLITE_MAX_SQL_LENGTH=128000000",
"SQLITE_PRINTF_PRECISION_LIMIT=1280000",
]
# During fuzz testing, valid SQL queries generated by fuzzing engine may
# lead to large memory allocations. If that happens, fuzzer reports an
# out-of-memory error. However, such errors are not valid bugs.
# To avoid hitting those irrelevant OOMs, we limit max number of memory
# pages, so fuzzer will not crash when reaching the limit.
# Apply this for fuzzing builds only, not for all builds with sanitizers.
if (use_fuzzing_engine) {
defines += [
"SQLITE_MAX_PAGE_COUNT=16384",
# Used to deserialize a database from a libfuzzer-provided data blob.
# This is to fuzz SQLite's resilience to database corruption.
"SQLITE_ENABLE_DESERIALIZE",
]
# The progress callback is used in fuzzing to cancel long-running queries
# so we don't spend too much time on them.
defines -= [ "SQLITE_OMIT_PROGRESS_CALLBACK" ]
}
}
if (is_debug || dcheck_always_on) {
if (use_fuzzing_engine && use_sanitizer_coverage) {
# Enable SQLite's assert() macros.
#
# TODO(pwnall): Fix all the bugs preventing us from enabling this flag for
# all DCHECK builds. See https://crbug.com/907371 and
# https://crrev.com/c/1343529.
defines += [ "SQLITE_DEBUG" ]
}
# Check preconditions when SQLite APIs are called. See
# https://sqlite.org/compile.html#enable_api_armor
#
# fuzzing builds have this disabled because the fuzzers are guaranteed to
# use the API correctly, and removing the checks opens up the possibility
# that the fuzzers will get more code coverage.
defines += [ "SQLITE_ENABLE_API_ARMOR" ]
}
}
config("sqlite_warnings") {
cflags = []
if (is_clang) {
# sqlite contains a few functions that are unused, at least on
# Windows with Chrome's sqlite patches applied
# (interiorCursorEOF fts3EvalDeferredPhrase
# fts3EvalSelectDeferred sqlite3Fts3InitHashTable
# sqlite3Fts3InitTok).
cflags += [ "-Wno-unused-function" ]
if (is_debug || dcheck_always_on) {
cflags += [
# SQLite uses assert(!"description") to express
# NOT_REACHED() << "description". This is considered an implicit
# conversion from char[] to bool, and triggers a warning.
"-Wno-string-conversion",
]
}
}
if (is_linux) {
cflags += [
# SQLite doesn't believe in compiler warnings, preferring testing.
# http://www.sqlite.org/faq.html#q17
"-Wno-int-to-pointer-cast",
"-Wno-pointer-to-int-cast",
]
}
if (is_ios) {
cflags += [
# SQLite issues a #pragma warning on iOS.
# http://sqlite.1065341.n5.nabble.com/Compiler-warning-quot-gethostuuid-is-disabled-quot-building-SQLite-for-iOS-td96881.html
#
# Contrary to what is said on the mailing list, setting
# SQLITE_ENABLE_LOCKING_STYLE to 0 does not make the warning go away.
"-Wno-#warnings",
]
}
if (is_win && !is_clang) {
cflags += [ "/wd4101" ] # 'zTrace' unreferenced variable in patched/vdbe.c
}
}
# Naming the library "sqlite3" can cause conflicts with the system library.
component("chromium_sqlite3") {
visibility = [ ":*" ]
public = [ "sqlite3.h" ]
sources = [
"amalgamation/sqlite3.h",
"sqlite3_shim.c",
]
inputs = [
# This file is #included into sqlite3_shim.c, which injects Chrome-specific
# definitions into the SQLite amalgamation code.
"amalgamation/sqlite3.c",
]
cflags = []
defines = []
if (is_component_build) {
if (is_win) {
defines += [ "SQLITE_API=__declspec(dllexport)" ]
} else {
defines += [ "SQLITE_API=__attribute__((visibility(\"default\")))" ]
}
}
if (is_linux || is_android) {
defines += [
# Linux provides fdatasync(), a faster equivalent of fsync().
"fdatasync=fdatasync",
]
}
if (is_posix || is_fuchsia) {
defines += [
# Allow xSleep() call on Unix to use usleep() rather than sleep(), so it
# will have microsecond precision. Should only affect contended
# databases via the busy callback. Browser profile databases are mostly
# exclusive, but renderer databases may allow for contention.
"HAVE_USLEEP=1",
# Use pread/pwrite directly rather than emulating them.
"USE_PREAD=1",
]
}
include_dirs = [
".", # sqlite3.h here must override the one in amalgamation/.
"amalgamation",
]
configs -= [ "//build/config/compiler:chromium_code" ]
configs += [
":chromium_sqlite3_compile_options",
"//build/config/compiler:no_chromium_code",
"//build/config/sanitizers:cfi_icall_generalize_pointers",
# Must be after no_chromium_code for warning flags to be ordered correctly.
":sqlite_warnings",
]
if (is_mac || is_ios) {
libs = [ "CoreFoundation.framework" ]
if (!is_ios) {
libs += [ "CoreServices.framework" ]
}
} else if (is_android) {
defines += [
"SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576",
"SQLITE_DEFAULT_AUTOVACUUM=1",
]
}
deps = [ "//third_party/icu" ]
}
config("sqlite_export") {
if (is_component_build && is_win) {
defines = [ "SQLITE_API=__declspec(dllimport)" ]
}
}
# This is used to allow the SQLITE_API definition to be different when
# building sqlite3.c than it is when clients include sqlite3.h.
group("sqlite") {
public_deps = [ ":chromium_sqlite3" ]
public_configs = [
":chromium_sqlite3_compile_options",
":sqlite_export",
]
}
if (is_win || is_mac || is_linux) {
executable("sqlite_shell") {
include_dirs = [
# SQLite's shell.c contains an '#include "sqlite3.h", which we want to be
# resolved to //third_party/sqlite/sqlite3.h.
".",
]
sources = [
"sqlite_shell_icu_helper.cc",
"sqlite_shell_icu_helper.h",
"sqlite_shell_shim.c",
]
deps = [
":sqlite",
"//base",
"//base:i18n",
"//third_party/icu",
]
configs -= [ "//build/config/compiler:chromium_code" ]
configs += [
":chromium_sqlite3_compile_options",
"//build/config/compiler:no_chromium_code",
# Must be after no_chromium_code for warning flags to be ordered
# correctly.
":sqlite_warnings",
]
}
}
# Libfuzzer-based fuzzer test from SQLite source tree.
fuzzer_test("sqlite3_ossfuzz_fuzzer") {
include_dirs = [ "." ]
sources = [ "patched/test/ossfuzz.c" ]
deps = [ ":sqlite" ]
dict = "fuzz/sql.dict"
}
source_set("sqlite3_lpm_fuzzer_core") {
sources = [
"fuzz/disabled_queries_parser.cc",
"fuzz/disabled_queries_parser.h",
"fuzz/sql_run_queries.cc",
"fuzz/sql_run_queries.h",
]
deps = [ ":sqlite" ]
public_deps = [ ":sqlite3_lpm_fuzzer_input" ]
configs += [
":sqlite_warnings",
":chromium_sqlite3_compile_options",
]
all_dependent_configs = [
":lpm_fuzzer_omit_non_websql",
":chromium_sqlite3_compile_options",
]
}
# LPM-based fuzzer test.
fuzzer_test("sqlite3_lpm_fuzzer") {
sources = [
"fuzz/sql_fuzzer.cc",
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//third_party/libprotobuf-mutator",
]
additional_configs = [ ":sqlite_warnings" ]
libfuzzer_options = [
"max_len=2111000",
"len_control=0",
]
seed_corpus = "fuzz/lpm_fuzzer_seed_corpus/"
}
# FTS3-focused LPM-based fuzzer test.
fuzzer_test("sqlite3_fts3_lpm_fuzzer") {
sources = [
"fuzz/sql_fuzzer.cc",
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//third_party/libprotobuf-mutator",
]
additional_configs = [
":sqlite_warnings",
":sqlite3_fts3_lpm_fuzzer_config",
]
libfuzzer_options = [
"max_len=2111000",
"len_control=0",
]
}
fuzzer_test("sqlite3_shadow_table_fuzzer") {
sources = [ "fuzz/shadow_table_fuzzer.cc" ]
deps = [ ":sqlite" ]
additional_configs = [ ":sqlite_warnings" ]
}
fuzzer_test("sqlite3_select_printf_lpm_fuzzer") {
sources = [
"fuzz/sql_printf_fuzzer.cc",
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//third_party/libprotobuf-mutator",
]
libfuzzer_options = [ "max_len=111000" ]
}
fuzzer_test("sqlite3_select_strftime_lpm_fuzzer") {
sources = [
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
"fuzz/sql_strftime_fuzzer.cc",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//third_party/libprotobuf-mutator",
]
libfuzzer_options = [ "max_len=111000" ]
}
fuzzer_test("sqlite3_select_expr_lpm_fuzzer") {
sources = [
"fuzz/sql_expr_fuzzer.cc",
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//third_party/libprotobuf-mutator",
]
libfuzzer_options = [
"max_len=111000",
"len_control=0",
]
}
config("sqlite3_fts3_lpm_fuzzer_config") {
defines = [ "FUZZ_FTS3" ]
}
config("lpm_fuzzer_omit_non_websql") {
defines = [
"FUZZ_OMIT_SAVEPOINT",
"FUZZ_OMIT_PRAGMA",
]
}
proto_library("sqlite3_lpm_fuzzer_input") {
sources = [
"fuzz/icu_codes.proto",
"fuzz/sql_queries.proto",
"fuzz/sql_query_grammar.proto",
]
}
# Generates a good corpus for the sqlite_lpm_fuzzer
# Don't build this tool on Windows since it uses a POSIX-only API and because it
# only needs to be used on devs' machines.
if (use_fuzzing_engine && !is_win) {
executable("sqlite3_lpm_corpus_gen") {
sources = [
"fuzz/sql_generate_corpus.cc",
"fuzz/sql_query_proto_to_string.cc",
"fuzz/sql_query_proto_to_string.h",
]
deps = [
":sqlite3_lpm_fuzzer_core",
"//base",
"//third_party/protobuf:protobuf_full",
]
}
}
config("sqlite3_dbfuzz2_config") {
cflags = [ "-Wno-sign-compare" ]
configs = [ ":sqlite_warnings" ]
}
# Upstream fuzzer that tests corrupted database files.
if (use_fuzzing_engine) {
fuzzer_test("sqlite3_dbfuzz2_fuzzer") {
include_dirs = [ "." ]
sources = [ "patched/test/dbfuzz2.c" ]
deps = [ ":sqlite" ]
additional_configs = [ ":sqlite3_dbfuzz2_config" ]
seed_corpus = "fuzz/db_corpus"
}
}