blob: 24c7a2d5692661034d80ea5d6f2bc1ead29d2fb3 [file] [log] [blame]
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of the page cache subsystem or "pager".
**
** The pager is used to access a database disk file. It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file. The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "wal.h"
/******************* NOTES ON THE DESIGN OF THE PAGER ************************
**
** This comment block describes invariants that hold when using a rollback
** journal. These invariants do not apply for journal_mode=WAL,
** journal_mode=MEMORY, or journal_mode=OFF.
**
** Within this comment block, a page is deemed to have been synced
** automatically as soon as it is written when PRAGMA synchronous=OFF.
** Otherwise, the page is not synced until the xSync method of the VFS
** is called successfully on the file containing the page.
**
** Definition: A page of the database file is said to be "overwriteable" if
** one or more of the following are true about the page:
**
** (a) The original content of the page as it was at the beginning of
** the transaction has been written into the rollback journal and
** synced.
**
** (b) The page was a freelist leaf page at the start of the transaction.
**
** (c) The page number is greater than the largest page that existed in
** the database file at the start of the transaction.
**
** (1) A page of the database file is never overwritten unless one of the
** following are true:
**
** (a) The page and all other pages on the same sector are overwriteable.
**
** (b) The atomic page write optimization is enabled, and the entire
** transaction other than the update of the transaction sequence
** number consists of a single page change.
**
** (2) The content of a page written into the rollback journal exactly matches
** both the content in the database when the rollback journal was written
** and the content in the database at the beginning of the current
** transaction.
**
** (3) Writes to the database file are an integer multiple of the page size
** in length and are aligned on a page boundary.
**
** (4) Reads from the database file are either aligned on a page boundary and
** an integer multiple of the page size in length or are taken from the
** first 100 bytes of the database file.
**
** (5) All writes to the database file are synced prior to the rollback journal
** being deleted, truncated, or zeroed.
**
** (6) If a master journal file is used, then all writes to the database file
** are synced prior to the master journal being deleted.
**
** Definition: Two databases (or the same database at two points it time)
** are said to be "logically equivalent" if they give the same answer to
** all queries. Note in particular the content of freelist leaf
** pages can be changed arbitrarily without affecting the logical equivalence
** of the database.
**
** (7) At any time, if any subset, including the empty set and the total set,
** of the unsynced changes to a rollback journal are removed and the
** journal is rolled back, the resulting database file will be logically
** equivalent to the database file at the beginning of the transaction.
**
** (8) When a transaction is rolled back, the xTruncate method of the VFS
** is called to restore the database file to the same size it was at
** the beginning of the transaction. (In some VFSes, the xTruncate
** method is a no-op, but that does not change the fact the SQLite will
** invoke it.)
**
** (9) Whenever the database file is modified, at least one bit in the range
** of bytes from 24 through 39 inclusive will be changed prior to releasing
** the EXCLUSIVE lock, thus signaling other connections on the same
** database to flush their caches.
**
** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less
** than one billion transactions.
**
** (11) A database file is well-formed at the beginning and at the conclusion
** of every transaction.
**
** (12) An EXCLUSIVE lock is held on the database file when writing to
** the database file.
**
** (13) A SHARED lock is held on the database file while reading any
** content out of the database file.
**
******************************************************************************/
/*
** Macros for troubleshooting. Normally turned off
*/
#if 0
int sqlite3PagerTrace=1; /* True to enable tracing */
#define sqlite3DebugPrintf printf
#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; }
#else
#define PAGERTRACE(X)
#endif
/*
** The following two macros are used within the PAGERTRACE() macros above
** to print out file-descriptors.
**
** PAGERID() takes a pointer to a Pager struct as its argument. The
** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file
** struct as its argument.
*/
#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd))
#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd))
/*
** The Pager.eState variable stores the current 'state' of a pager. A
** pager may be in any one of the seven states shown in the following
** state diagram.
**
** OPEN <------+------+
** | | |
** V | |
** +---------> READER-------+ |
** | | |
** | V |
** |<-------WRITER_LOCKED------> ERROR
** | | ^
** | V |
** |<------WRITER_CACHEMOD-------->|
** | | |
** | V |
** |<-------WRITER_DBMOD---------->|
** | | |
** | V |
** +<------WRITER_FINISHED-------->+
**
**
** List of state transitions and the C [function] that performs each:
**
** OPEN -> READER [sqlite3PagerSharedLock]
** READER -> OPEN [pager_unlock]
**
** READER -> WRITER_LOCKED [sqlite3PagerBegin]
** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal]
** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal]
** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne]
** WRITER_*** -> READER [pager_end_transaction]
**
** WRITER_*** -> ERROR [pager_error]
** ERROR -> OPEN [pager_unlock]
**
**
** OPEN:
**
** The pager starts up in this state. Nothing is guaranteed in this
** state - the file may or may not be locked and the database size is
** unknown. The database may not be read or written.
**
** * No read or write transaction is active.
** * Any lock, or no lock at all, may be held on the database file.
** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted.
**
** READER:
**
** In this state all the requirements for reading the database in
** rollback (non-WAL) mode are met. Unless the pager is (or recently
** was) in exclusive-locking mode, a user-level read transaction is
** open. The database size is known in this state.
**
** A connection running with locking_mode=normal enters this state when
** it opens a read-transaction on the database and returns to state
** OPEN after the read-transaction is completed. However a connection
** running in locking_mode=exclusive (including temp databases) remains in
** this state even after the read-transaction is closed. The only way
** a locking_mode=exclusive connection can transition from READER to OPEN
** is via the ERROR state (see below).
**
** * A read transaction may be active (but a write-transaction cannot).
** * A SHARED or greater lock is held on the database file.
** * The dbSize variable may be trusted (even if a user-level read
** transaction is not active). The dbOrigSize and dbFileSize variables
** may not be trusted at this point.
** * If the database is a WAL database, then the WAL connection is open.
** * Even if a read-transaction is not open, it is guaranteed that
** there is no hot-journal in the file-system.
**
** WRITER_LOCKED:
**
** The pager moves to this state from READER when a write-transaction
** is first opened on the database. In WRITER_LOCKED state, all locks
** required to start a write-transaction are held, but no actual
** modifications to the cache or database have taken place.
**
** In rollback mode, a RESERVED or (if the transaction was opened with
** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when
** moving to this state, but the journal file is not written to or opened
** to in this state. If the transaction is committed or rolled back while
** in WRITER_LOCKED state, all that is required is to unlock the database
** file.
**
** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file.
** If the connection is running with locking_mode=exclusive, an attempt
** is made to obtain an EXCLUSIVE lock on the database file.
**
** * A write transaction is active.
** * If the connection is open in rollback-mode, a RESERVED or greater
** lock is held on the database file.
** * If the connection is open in WAL-mode, a WAL write transaction
** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully
** called).
** * The dbSize, dbOrigSize and dbFileSize variables are all valid.
** * The contents of the pager cache have not been modified.
** * The journal file may or may not be open.
** * Nothing (not even the first header) has been written to the journal.
**
** WRITER_CACHEMOD:
**
** A pager moves from WRITER_LOCKED state to this state when a page is
** first modified by the upper layer. In rollback mode the journal file
** is opened (if it is not already open) and a header written to the
** start of it. The database file on disk has not been modified.
**
** * A write transaction is active.
** * A RESERVED or greater lock is held on the database file.
** * The journal file is open and the first header has been written
** to it, but the header has not been synced to disk.
** * The contents of the page cache have been modified.
**
** WRITER_DBMOD:
**
** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state
** when it modifies the contents of the database file. WAL connections
** never enter this state (since they do not modify the database file,
** just the log file).
**
** * A write transaction is active.
** * An EXCLUSIVE or greater lock is held on the database file.
** * The journal file is open and the first header has been written
** and synced to disk.
** * The contents of the page cache have been modified (and possibly
** written to disk).
**
** WRITER_FINISHED:
**
** It is not possible for a WAL connection to enter this state.
**
** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD
** state after the entire transaction has been successfully written into the
** database file. In this state the transaction may be committed simply
** by finalizing the journal file. Once in WRITER_FINISHED state, it is
** not possible to modify the database further. At this point, the upper
** layer must either commit or rollback the transaction.
**
** * A write transaction is active.
** * An EXCLUSIVE or greater lock is held on the database file.
** * All writing and syncing of journal and database data has finished.
** If no error occurred, all that remains is to finalize the journal to
** commit the transaction. If an error did occur, the caller will need
** to rollback the transaction.
**
** ERROR:
**
** The ERROR state is entered when an IO or disk-full error (including
** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it
** difficult to be sure that the in-memory pager state (cache contents,
** db size etc.) are consistent with the contents of the file-system.
**
** Temporary pager files may enter the ERROR state, but in-memory pagers
** cannot.
**
** For example, if an IO error occurs while performing a rollback,
** the contents of the page-cache may be left in an inconsistent state.
** At this point it would be dangerous to change back to READER state
** (as usually happens after a rollback). Any subsequent readers might
** report database corruption (due to the inconsistent cache), and if
** they upgrade to writers, they may inadvertently corrupt the database
** file. To avoid this hazard, the pager switches into the ERROR state
** instead of READER following such an error.
**
** Once it has entered the ERROR state, any attempt to use the pager
** to read or write data returns an error. Eventually, once all
** outstanding transactions have been abandoned, the pager is able to
** transition back to OPEN state, discarding the contents of the
** page-cache and any other in-memory state at the same time. Everything
** is reloaded from disk (and, if necessary, hot-journal rollback peformed)
** when a read-transaction is next opened on the pager (transitioning
** the pager into READER state). At that point the system has recovered
** from the error.
**
** Specifically, the pager jumps into the ERROR state if:
**
** 1. An error occurs while attempting a rollback. This happens in
** function sqlite3PagerRollback().
**
** 2. An error occurs while attempting to finalize a journal file
** following a commit in function sqlite3PagerCommitPhaseTwo().
**
** 3. An error occurs while attempting to write to the journal or
** database file in function pagerStress() in order to free up
** memory.
**
** In other cases, the error is returned to the b-tree layer. The b-tree
** layer then attempts a rollback operation. If the error condition
** persists, the pager enters the ERROR state via condition (1) above.
**
** Condition (3) is necessary because it can be triggered by a read-only
** statement executed within a transaction. In this case, if the error
** code were simply returned to the user, the b-tree layer would not
** automatically attempt a rollback, as it assumes that an error in a
** read-only statement cannot leave the pager in an internally inconsistent
** state.
**
** * The Pager.errCode variable is set to something other than SQLITE_OK.
** * There are one or more outstanding references to pages (after the
** last reference is dropped the pager should move back to OPEN state).
** * The pager is not an in-memory pager.
**
**
** Notes:
**
** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the
** connection is open in WAL mode. A WAL connection is always in one
** of the first four states.
**
** * Normally, a connection open in exclusive mode is never in PAGER_OPEN
** state. There are two exceptions: immediately after exclusive-mode has
** been turned on (and before any read or write transactions are
** executed), and when the pager is leaving the "error state".
**
** * See also: assert_pager_state().
*/
#define PAGER_OPEN 0
#define PAGER_READER 1
#define PAGER_WRITER_LOCKED 2
#define PAGER_WRITER_CACHEMOD 3
#define PAGER_WRITER_DBMOD 4
#define PAGER_WRITER_FINISHED 5
#define PAGER_ERROR 6
/*
** The Pager.eLock variable is almost always set to one of the
** following locking-states, according to the lock currently held on
** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK.
** This variable is kept up to date as locks are taken and released by
** the pagerLockDb() and pagerUnlockDb() wrappers.
**
** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY
** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not
** the operation was successful. In these circumstances pagerLockDb() and
** pagerUnlockDb() take a conservative approach - eLock is always updated
** when unlocking the file, and only updated when locking the file if the
** VFS call is successful. This way, the Pager.eLock variable may be set
** to a less exclusive (lower) value than the lock that is actually held
** at the system level, but it is never set to a more exclusive value.
**
** This is usually safe. If an xUnlock fails or appears to fail, there may
** be a few redundant xLock() calls or a lock may be held for longer than
** required, but nothing really goes wrong.
**
** The exception is when the database file is unlocked as the pager moves
** from ERROR to OPEN state. At this point there may be a hot-journal file
** in the file-system that needs to be rolled back (as part of an OPEN->SHARED
** transition, by the same pager or any other). If the call to xUnlock()
** fails at this point and the pager is left holding an EXCLUSIVE lock, this
** can confuse the call to xCheckReservedLock() call made later as part
** of hot-journal detection.
**
** xCheckReservedLock() is defined as returning true "if there is a RESERVED
** lock held by this process or any others". So xCheckReservedLock may
** return true because the caller itself is holding an EXCLUSIVE lock (but
** doesn't know it because of a previous error in xUnlock). If this happens
** a hot-journal may be mistaken for a journal being created by an active
** transaction in another process, causing SQLite to read from the database
** without rolling it back.
**
** To work around this, if a call to xUnlock() fails when unlocking the
** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It
** is only changed back to a real locking state after a successful call
** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition
** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK
** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE
** lock on the database file before attempting to roll it back. See function
** PagerSharedLock() for more detail.
**
** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in
** PAGER_OPEN state.
*/
#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1)
/*
** A macro used for invoking the codec if there is one
*/
#ifdef SQLITE_HAS_CODEC
# define CODEC1(P,D,N,X,E) \
if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; }
# define CODEC2(P,D,N,X,E,O) \
if( P->xCodec==0 ){ O=(char*)D; }else \
if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; }
#else
# define CODEC1(P,D,N,X,E) /* NO-OP */
# define CODEC2(P,D,N,X,E,O) O=(char*)D
#endif
/*
** The maximum allowed sector size. 64KiB. If the xSectorsize() method
** returns a value larger than this, then MAX_SECTOR_SIZE is used instead.
** This could conceivably cause corruption following a power failure on
** such a system. This is currently an undocumented limit.
*/
#define MAX_SECTOR_SIZE 0x10000
/*
** An instance of the following structure is allocated for each active
** savepoint and statement transaction in the system. All such structures
** are stored in the Pager.aSavepoint[] array, which is allocated and
** resized using sqlite3Realloc().
**
** When a savepoint is created, the PagerSavepoint.iHdrOffset field is
** set to 0. If a journal-header is written into the main journal while
** the savepoint is active, then iHdrOffset is set to the byte offset
** immediately following the last journal record written into the main
** journal before the journal-header. This is required during savepoint
** rollback (see pagerPlaybackSavepoint()).
*/
typedef struct PagerSavepoint PagerSavepoint;
struct PagerSavepoint {
i64 iOffset; /* Starting offset in main journal */
i64 iHdrOffset; /* See above */
Bitvec *pInSavepoint; /* Set of pages in this savepoint */
Pgno nOrig; /* Original number of pages in file */
Pgno iSubRec; /* Index of first record in sub-journal */
#ifndef SQLITE_OMIT_WAL
u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */
#endif
};
/*
** Bits of the Pager.doNotSpill flag. See further description below.
*/
#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */
#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */
#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */
/*
** An open page cache is an instance of struct Pager. A description of
** some of the more important member variables follows:
**
** eState
**
** The current 'state' of the pager object. See the comment and state
** diagram above for a description of the pager state.
**
** eLock
**
** For a real on-disk database, the current lock held on the database file -
** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK.
**
** For a temporary or in-memory database (neither of which require any
** locks), this variable is always set to EXCLUSIVE_LOCK. Since such
** databases always have Pager.exclusiveMode==1, this tricks the pager
** logic into thinking that it already has all the locks it will ever
** need (and no reason to release them).
**
** In some (obscure) circumstances, this variable may also be set to
** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for
** details.
**
** changeCountDone
**
** This boolean variable is used to make sure that the change-counter
** (the 4-byte header field at byte offset 24 of the database file) is
** not updated more often than necessary.
**
** It is set to true when the change-counter field is updated, which
** can only happen if an exclusive lock is held on the database file.
** It is cleared (set to false) whenever an exclusive lock is
** relinquished on the database file. Each time a transaction is committed,
** The changeCountDone flag is inspected. If it is true, the work of
** updating the change-counter is omitted for the current transaction.
**
** This mechanism means that when running in exclusive mode, a connection
** need only update the change-counter once, for the first transaction
** committed.
**
** setMaster
**
** When PagerCommitPhaseOne() is called to commit a transaction, it may
** (or may not) specify a master-journal name to be written into the
** journal file before it is synced to disk.
**
** Whether or not a journal file contains a master-journal pointer affects
** the way in which the journal file is finalized after the transaction is
** committed or rolled back when running in "journal_mode=PERSIST" mode.
** If a journal file does not contain a master-journal pointer, it is
** finalized by overwriting the first journal header with zeroes. If
** it does contain a master-journal pointer the journal file is finalized
** by truncating it to zero bytes, just as if the connection were
** running in "journal_mode=truncate" mode.
**
** Journal files that contain master journal pointers cannot be finalized
** simply by overwriting the first journal-header with zeroes, as the
** master journal pointer could interfere with hot-journal rollback of any
** subsequently interrupted transaction that reuses the journal file.
**
** The flag is cleared as soon as the journal file is finalized (either
** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the
** journal file from being successfully finalized, the setMaster flag
** is cleared anyway (and the pager will move to ERROR state).
**
** doNotSpill
**
** This variables control the behavior of cache-spills (calls made by
** the pcache module to the pagerStress() routine to write cached data
** to the file-system in order to free up memory).
**
** When bits SPILLFLAG_OFF or SPILLFLAG_ROLLBACK of doNotSpill are set,
** writing to the database from pagerStress() is disabled altogether.
** The SPILLFLAG_ROLLBACK case is done in a very obscure case that
** comes up during savepoint rollback that requires the pcache module
** to allocate a new page to prevent the journal file from being written
** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF
** case is a user preference.
**
** If the SPILLFLAG_NOSYNC bit is set, writing to the database from
** pagerStress() is permitted, but syncing the journal file is not.
** This flag is set by sqlite3PagerWrite() when the file-system sector-size
** is larger than the database page-size in order to prevent a journal sync
** from happening in between the journalling of two pages on the same sector.
**
** subjInMemory
**
** This is a boolean variable. If true, then any required sub-journal
** is opened as an in-memory journal file. If false, then in-memory
** sub-journals are only used for in-memory pager files.
**
** This variable is updated by the upper layer each time a new
** write-transaction is opened.
**
** dbSize, dbOrigSize, dbFileSize
**
** Variable dbSize is set to the number of pages in the database file.
** It is valid in PAGER_READER and higher states (all states except for
** OPEN and ERROR).
**
** dbSize is set based on the size of the database file, which may be
** larger than the size of the database (the value stored at offset
** 28 of the database header by the btree). If the size of the file
** is not an integer multiple of the page-size, the value stored in
** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2).
** Except, any file that is greater than 0 bytes in size is considered
** to have at least one page. (i.e. a 1KB file with 2K page-size leads
** to dbSize==1).
**
** During a write-transaction, if pages with page-numbers greater than
** dbSize are modified in the cache, dbSize is updated accordingly.
** Similarly, if the database is truncated using PagerTruncateImage(),
** dbSize is updated.
**
** Variables dbOrigSize and dbFileSize are valid in states
** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize
** variable at the start of the transaction. It is used during rollback,
** and to determine whether or not pages need to be journalled before
** being modified.
**
** Throughout a write-transaction, dbFileSize contains the size of
** the file on disk in pages. It is set to a copy of dbSize when the
** write-transaction is first opened, and updated when VFS calls are made
** to write or truncate the database file on disk.
**
** The only reason the dbFileSize variable is required is to suppress
** unnecessary calls to xTruncate() after committing a transaction. If,
** when a transaction is committed, the dbFileSize variable indicates
** that the database file is larger than the database image (Pager.dbSize),
** pager_truncate() is called. The pager_truncate() call uses xFilesize()
** to measure the database file on disk, and then truncates it if required.
** dbFileSize is not used when rolling back a transaction. In this case
** pager_truncate() is called unconditionally (which means there may be
** a call to xFilesize() that is not strictly required). In either case,
** pager_truncate() may cause the file to become smaller or larger.
**
** dbHintSize
**
** The dbHintSize variable is used to limit the number of calls made to
** the VFS xFileControl(FCNTL_SIZE_HINT) method.
**
** dbHintSize is set to a copy of the dbSize variable when a
** write-transaction is opened (at the same time as dbFileSize and
** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called,
** dbHintSize is increased to the number of pages that correspond to the
** size-hint passed to the method call. See pager_write_pagelist() for
** details.
**
** errCode
**
** The Pager.errCode variable is only ever used in PAGER_ERROR state. It
** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode
** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX
** sub-codes.
**
** syncFlags, walSyncFlags
**
** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03).
** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode
** and contains the flags used to sync the checkpoint operations in the
** lower two bits, and sync flags used for transaction commits in the WAL
** file in bits 0x04 and 0x08. In other words, to get the correct sync flags
** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct
** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note
** that with synchronous=NORMAL in WAL mode, transaction commit is not synced
** meaning that the 0x04 and 0x08 bits are both zero.
*/
struct Pager {
sqlite3_vfs *pVfs; /* OS functions to use for IO */
u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */
u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */
u8 useJournal; /* Use a rollback journal on this file */
u8 noSync; /* Do not sync the journal if true */
u8 fullSync; /* Do extra syncs of the journal for robustness */
u8 extraSync; /* sync directory after journal delete */
u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */
u8 walSyncFlags; /* See description above */
u8 tempFile; /* zFilename is a temporary or immutable file */
u8 noLock; /* Do not lock (except in WAL mode) */
u8 readOnly; /* True for a read-only database */
u8 memDb; /* True to inhibit all file I/O */
/**************************************************************************
** The following block contains those class members that change during
** routine operation. Class members not in this block are either fixed
** when the pager is first created or else only change when there is a
** significant mode change (such as changing the page_size, locking_mode,
** or the journal_mode). From another view, these class members describe
** the "state" of the pager, while other class members describe the
** "configuration" of the pager.
*/
u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */
u8 eLock; /* Current lock held on database file */
u8 changeCountDone; /* Set after incrementing the change-counter */
u8 setMaster; /* True if a m-j name has been written to jrnl */
u8 doNotSpill; /* Do not spill the cache when non-zero */
u8 subjInMemory; /* True to use in-memory sub-journals */
u8 bUseFetch; /* True to use xFetch() */
u8 hasHeldSharedLock; /* True if a shared lock has ever been held */
Pgno dbSize; /* Number of pages in the database */
Pgno dbOrigSize; /* dbSize before the current transaction */
Pgno dbFileSize; /* Number of pages in the database file */
Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */
int errCode; /* One of several kinds of errors */
int nRec; /* Pages journalled since last j-header written */
u32 cksumInit; /* Quasi-random value added to every checksum */
u32 nSubRec; /* Number of records written to sub-journal */
Bitvec *pInJournal; /* One bit for each page in the database file */
sqlite3_file *fd; /* File descriptor for database */
sqlite3_file *jfd; /* File descriptor for main journal */
sqlite3_file *sjfd; /* File descriptor for sub-journal */
i64 journalOff; /* Current write offset in the journal file */
i64 journalHdr; /* Byte offset to previous journal header */
sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */
PagerSavepoint *aSavepoint; /* Array of active savepoints */
int nSavepoint; /* Number of elements in aSavepoint[] */
u32 iDataVersion; /* Changes whenever database content changes */
char dbFileVers[16]; /* Changes whenever database file changes */
int nMmapOut; /* Number of mmap pages currently outstanding */
sqlite3_int64 szMmap; /* Desired maximum mmap size */
PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */
/*
** End of the routinely-changing class members
***************************************************************************/
u16 nExtra; /* Add this many bytes to each in-memory page */
i16 nReserve; /* Number of unused bytes at end of each page */
u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */
u32 sectorSize; /* Assumed sector size during rollback */
int pageSize; /* Number of bytes in a page */
Pgno mxPgno; /* Maximum allowed size of the database */
i64 journalSizeLimit; /* Size limit for persistent journal files */
char *zFilename; /* Name of the database file */
char *zJournal; /* Name of the journal file */
int (*xBusyHandler)(void*); /* Function to call when busy */
void *pBusyHandlerArg; /* Context argument for xBusyHandler */
int aStat[4]; /* Total cache hits, misses, writes, spills */
#ifdef SQLITE_TEST
int nRead; /* Database pages read */
#endif
void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */
int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */
#ifdef SQLITE_HAS_CODEC
void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */
void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */
void (*xCodecFree)(void*); /* Destructor for the codec */
void *pCodec; /* First argument to xCodec... methods */
#endif
char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */
PCache *pPCache; /* Pointer to page cache object */
#ifndef SQLITE_OMIT_WAL
Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */
char *zWal; /* File name for write-ahead log */
#endif
};
/*
** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains
** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS
** or CACHE_WRITE to sqlite3_db_status().
*/
#define PAGER_STAT_HIT 0
#define PAGER_STAT_MISS 1
#define PAGER_STAT_WRITE 2
#define PAGER_STAT_SPILL 3
/*
** The following global variables hold counters used for
** testing purposes only. These variables do not exist in
** a non-testing build. These variables are not thread-safe.
*/
#ifdef SQLITE_TEST
int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */
int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */
int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */
# define PAGER_INCR(v) v++
#else
# define PAGER_INCR(v)
#endif
/*
** Journal files begin with the following magic string. The data
** was obtained from /dev/random. It is used only as a sanity check.
**
** Since version 2.8.0, the journal format contains additional sanity
** checking information. If the power fails while the journal is being
** written, semi-random garbage data might appear in the journal
** file after power is restored. If an attempt is then made
** to roll the journal back, the database could be corrupted. The additional
** sanity checking data is an attempt to discover the garbage in the
** journal and ignore it.
**
** The sanity checking information for the new journal format consists
** of a 32-bit checksum on each page of data. The checksum covers both
** the page number and the pPager->pageSize bytes of data for the page.
** This cksum is initialized to a 32-bit random value that appears in the
** journal file right after the header. The random initializer is important,
** because garbage data that appears at the end of a journal is likely
** data that was once in other files that have now been deleted. If the
** garbage data came from an obsolete journal file, the checksums might
** be correct. But by initializing the checksum to random value which
** is different for every journal, we minimize that risk.
*/
static const unsigned char aJournalMagic[] = {
0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7,
};
/*
** The size of the of each page record in the journal is given by
** the following macro.
*/
#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8)
/*
** The journal header size for this pager. This is usually the same
** size as a single disk sector. See also setSectorSize().
*/
#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize)
/*
** The macro MEMDB is true if we are dealing with an in-memory database.
** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set,
** the value of MEMDB will be a constant and the compiler will optimize
** out code that would never execute.
*/
#ifdef SQLITE_OMIT_MEMORYDB
# define MEMDB 0
#else
# define MEMDB pPager->memDb
#endif
/*
** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch
** interfaces to access the database using memory-mapped I/O.
*/
#if SQLITE_MAX_MMAP_SIZE>0
# define USEFETCH(x) ((x)->bUseFetch)
#else
# define USEFETCH(x) 0
#endif
/*
** The maximum legal page number is (2^31 - 1).
*/
#define PAGER_MAX_PGNO 2147483647
/*
** The argument to this macro is a file descriptor (type sqlite3_file*).
** Return 0 if it is not open, or non-zero (but not 1) if it is.
**
** This is so that expressions can be written as:
**
** if( isOpen(pPager->jfd) ){ ...
**
** instead of
**
** if( pPager->jfd->pMethods ){ ...
*/
#define isOpen(pFd) ((pFd)->pMethods!=0)
#ifdef SQLITE_DIRECT_OVERFLOW_READ
/*
** Return true if page pgno can be read directly from the database file
** by the b-tree layer. This is the case if:
**
** * the database file is open,
** * there are no dirty pages in the cache, and
** * the desired page is not currently in the wal file.
*/
int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){
if( pPager->fd->pMethods==0 ) return 0;
if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0;
#ifdef SQLITE_HAS_CODEC
if( pPager->xCodec!=0 ) return 0;
#endif
#ifndef SQLITE_OMIT_WAL
if( pPager->pWal ){
u32 iRead = 0;
int rc;
rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead);
return (rc==SQLITE_OK && iRead==0);
}
#endif
return 1;
}
#endif
#ifndef SQLITE_OMIT_WAL
# define pagerUseWal(x) ((x)->pWal!=0)
#else
# define pagerUseWal(x) 0
# define pagerRollbackWal(x) 0
# define pagerWalFrames(v,w,x,y) 0
# define pagerOpenWalIfPresent(z) SQLITE_OK
# define pagerBeginReadTransaction(z) SQLITE_OK
#endif
#ifndef NDEBUG
/*
** Usage:
**
** assert( assert_pager_state(pPager) );
**
** This function runs many asserts to try to find inconsistencies in
** the internal state of the Pager object.
*/
static int assert_pager_state(Pager *p){
Pager *pPager = p;
/* State must be valid. */
assert( p->eState==PAGER_OPEN
|| p->eState==PAGER_READER
|| p->eState==PAGER_WRITER_LOCKED
|| p->eState==PAGER_WRITER_CACHEMOD
|| p->eState==PAGER_WRITER_DBMOD
|| p->eState==PAGER_WRITER_FINISHED
|| p->eState==PAGER_ERROR
);
/* Regardless of the current state, a temp-file connection always behaves
** as if it has an exclusive lock on the database file. It never updates
** the change-counter field, so the changeCountDone flag is always set.
*/
assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK );
assert( p->tempFile==0 || pPager->changeCountDone );
/* If the useJournal flag is clear, the journal-mode must be "OFF".
** And if the journal-mode is "OFF", the journal file must not be open.
*/
assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal );
assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) );
/* Check that MEMDB implies noSync. And an in-memory journal. Since
** this means an in-memory pager performs no IO at all, it cannot encounter
** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing
** a journal file. (although the in-memory journal implementation may
** return SQLITE_IOERR_NOMEM while the journal file is being written). It
** is therefore not possible for an in-memory pager to enter the ERROR
** state.
*/
if( MEMDB ){
assert( !isOpen(p->fd) );
assert( p->noSync );
assert( p->journalMode==PAGER_JOURNALMODE_OFF
|| p->journalMode==PAGER_JOURNALMODE_MEMORY
);
assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN );
assert( pagerUseWal(p)==0 );
}
/* If changeCountDone is set, a RESERVED lock or greater must be held
** on the file.
*/
assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK );
assert( p->eLock!=PENDING_LOCK );
switch( p->eState ){
case PAGER_OPEN:
assert( !MEMDB );
assert( pPager->errCode==SQLITE_OK );
assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile );
break;
case PAGER_READER:
assert( pPager->errCode==SQLITE_OK );
assert( p->eLock!=UNKNOWN_LOCK );
assert( p->eLock>=SHARED_LOCK );
break;
case PAGER_WRITER_LOCKED:
assert( p->eLock!=UNKNOWN_LOCK );
assert( pPager->errCode==SQLITE_OK );
if( !pagerUseWal(pPager) ){
assert( p->eLock>=RESERVED_LOCK );
}
assert( pPager->dbSize==pPager->dbOrigSize );
assert( pPager->dbOrigSize==pPager->dbFileSize );
assert( pPager->dbOrigSize==pPager->dbHintSize );
assert( pPager->setMaster==0 );
break;
case PAGER_WRITER_CACHEMOD:
assert( p->eLock!=UNKNOWN_LOCK );
assert( pPager->errCode==SQLITE_OK );
if( !pagerUseWal(pPager) ){
/* It is possible that if journal_mode=wal here that neither the
** journal file nor the WAL file are open. This happens during
** a rollback transaction that switches from journal_mode=off
** to journal_mode=wal.
*/
assert( p->eLock>=RESERVED_LOCK );
assert( isOpen(p->jfd)
|| p->journalMode==PAGER_JOURNALMODE_OFF
|| p->journalMode==PAGER_JOURNALMODE_WAL
);
}
assert( pPager->dbOrigSize==pPager->dbFileSize );
assert( pPager->dbOrigSize==pPager->dbHintSize );
break;
case PAGER_WRITER_DBMOD:
assert( p->eLock==EXCLUSIVE_LOCK );
assert( pPager->errCode==SQLITE_OK );
assert( !pagerUseWal(pPager) );
assert( p->eLock>=EXCLUSIVE_LOCK );
assert( isOpen(p->jfd)
|| p->journalMode==PAGER_JOURNALMODE_OFF
|| p->journalMode==PAGER_JOURNALMODE_WAL
|| (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
);
assert( pPager->dbOrigSize<=pPager->dbHintSize );
break;
case PAGER_WRITER_FINISHED:
assert( p->eLock==EXCLUSIVE_LOCK );
assert( pPager->errCode==SQLITE_OK );
assert( !pagerUseWal(pPager) );
assert( isOpen(p->jfd)
|| p->journalMode==PAGER_JOURNALMODE_OFF
|| p->journalMode==PAGER_JOURNALMODE_WAL
|| (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
);
break;
case PAGER_ERROR:
/* There must be at least one outstanding reference to the pager if
** in ERROR state. Otherwise the pager should have already dropped
** back to OPEN state.
*/
assert( pPager->errCode!=SQLITE_OK );
assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile );
break;
}
return 1;
}
#endif /* ifndef NDEBUG */
#ifdef SQLITE_DEBUG
/*
** Return a pointer to a human readable string in a static buffer
** containing the state of the Pager object passed as an argument. This
** is intended to be used within debuggers. For example, as an alternative
** to "print *pPager" in gdb:
**
** (gdb) printf "%s", print_pager_state(pPager)
**
** This routine has external linkage in order to suppress compiler warnings
** about an unused function. It is enclosed within SQLITE_DEBUG and so does
** not appear in normal builds.
*/
char *print_pager_state(Pager *p){
static char zRet[1024];
sqlite3_snprintf(1024, zRet,
"Filename: %s\n"
"State: %s errCode=%d\n"
"Lock: %s\n"
"Locking mode: locking_mode=%s\n"
"Journal mode: journal_mode=%s\n"
"Backing store: tempFile=%d memDb=%d useJournal=%d\n"
"Journal: journalOff=%lld journalHdr=%lld\n"
"Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n"
, p->zFilename
, p->eState==PAGER_OPEN ? "OPEN" :
p->eState==PAGER_READER ? "READER" :
p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" :
p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" :
p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" :
p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" :
p->eState==PAGER_ERROR ? "ERROR" : "?error?"
, (int)p->errCode
, p->eLock==NO_LOCK ? "NO_LOCK" :
p->eLock==RESERVED_LOCK ? "RESERVED" :
p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" :
p->eLock==SHARED_LOCK ? "SHARED" :
p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?"
, p->exclusiveMode ? "exclusive" : "normal"
, p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" :
p->journalMode==PAGER_JOURNALMODE_OFF ? "off" :
p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" :
p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" :
p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" :
p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?"
, (int)p->tempFile, (int)p->memDb, (int)p->useJournal
, p->journalOff, p->journalHdr
, (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize
);
return zRet;
}
#endif
/* Forward references to the various page getters */
static int getPageNormal(Pager*,Pgno,DbPage**,int);
static int getPageError(Pager*,Pgno,DbPage**,int);
#if SQLITE_MAX_MMAP_SIZE>0
static int getPageMMap(Pager*,Pgno,DbPage**,int);
#endif
/*
** Set the Pager.xGet method for the appropriate routine used to fetch
** content from the pager.
*/
static void setGetterMethod(Pager *pPager){
if( pPager->errCode ){
pPager->xGet = getPageError;
#if SQLITE_MAX_MMAP_SIZE>0
}else if( USEFETCH(pPager)
#ifdef SQLITE_HAS_CODEC
&& pPager->xCodec==0
#endif
){
pPager->xGet = getPageMMap;
#endif /* SQLITE_MAX_MMAP_SIZE>0 */
}else{
pPager->xGet = getPageNormal;
}
}
/*
** Return true if it is necessary to write page *pPg into the sub-journal.
** A page needs to be written into the sub-journal if there exists one
** or more open savepoints for which:
**
** * The page-number is less than or equal to PagerSavepoint.nOrig, and
** * The bit corresponding to the page-number is not set in
** PagerSavepoint.pInSavepoint.
*/
static int subjRequiresPage(PgHdr *pPg){
Pager *pPager = pPg->pPager;
PagerSavepoint *p;
Pgno pgno = pPg->pgno;
int i;
for(i=0; i<pPager->nSavepoint; i++){
p = &pPager->aSavepoint[i];
if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){
return 1;
}
}
return 0;
}
#ifdef SQLITE_DEBUG
/*
** Return true if the page is already in the journal file.
*/
static int pageInJournal(Pager *pPager, PgHdr *pPg){
return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno);
}
#endif
/*
** Read a 32-bit integer from the given file descriptor. Store the integer
** that is read in *pRes. Return SQLITE_OK if everything worked, or an
** error code is something goes wrong.
**
** All values are stored on disk as big-endian.
*/
static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){
unsigned char ac[4];
int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset);
if( rc==SQLITE_OK ){
*pRes = sqlite3Get4byte(ac);
}
return rc;
}
/*
** Write a 32-bit integer into a string buffer in big-endian byte order.
*/
#define put32bits(A,B) sqlite3Put4byte((u8*)A,B)
/*
** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK
** on success or an error code is something goes wrong.
*/
static int write32bits(sqlite3_file *fd, i64 offset, u32 val){
char ac[4];
put32bits(ac, val);
return sqlite3OsWrite(fd, ac, 4, offset);
}
/*
** Unlock the database file to level eLock, which must be either NO_LOCK
** or SHARED_LOCK. Regardless of whether or not the call to xUnlock()
** succeeds, set the Pager.eLock variable to match the (attempted) new lock.
**
** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is
** called, do not modify it. See the comment above the #define of
** UNKNOWN_LOCK for an explanation of this.
*/
static int pagerUnlockDb(Pager *pPager, int eLock){
int rc = SQLITE_OK;
assert( !pPager->exclusiveMode || pPager->eLock==eLock );
assert( eLock==NO_LOCK || eLock==SHARED_LOCK );
assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 );
if( isOpen(pPager->fd) ){
assert( pPager->eLock>=eLock );
rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock);
if( pPager->eLock!=UNKNOWN_LOCK ){
pPager->eLock = (u8)eLock;
}
IOTRACE(("UNLOCK %p %d\n", pPager, eLock))
}
return rc;
}
/*
** Lock the database file to level eLock, which must be either SHARED_LOCK,
** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the
** Pager.eLock variable to the new locking state.
**
** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is
** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK.
** See the comment above the #define of UNKNOWN_LOCK for an explanation
** of this.
*/
static int pagerLockDb(Pager *pPager, int eLock){
int rc = SQLITE_OK;
assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK );
if( pPager->eLock<eLock || pPager->eLock==UNKNOWN_LOCK ){
rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock);
if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){
pPager->eLock = (u8)eLock;
IOTRACE(("LOCK %p %d\n", pPager, eLock))
}
}
return rc;
}
/*
** This function determines whether or not the atomic-write or
** atomic-batch-write optimizations can be used with this pager. The
** atomic-write optimization can be used if:
**
** (a) the value returned by OsDeviceCharacteristics() indicates that
** a database page may be written atomically, and
** (b) the value returned by OsSectorSize() is less than or equal
** to the page size.
**
** If it can be used, then the value returned is the size of the journal
** file when it contains rollback data for exactly one page.
**
** The atomic-batch-write optimization can be used if OsDeviceCharacteristics()
** returns a value with the SQLITE_IOCAP_BATCH_ATOMIC bit set. -1 is
** returned in this case.
**
** If neither optimization can be used, 0 is returned.
*/
static int jrnlBufferSize(Pager *pPager){
assert( !MEMDB );
#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \
|| defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE)
int dc; /* Device characteristics */
assert( isOpen(pPager->fd) );
dc = sqlite3OsDeviceCharacteristics(pPager->fd);
#else
UNUSED_PARAMETER(pPager);
#endif
#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE
if( pPager->dbSize>0 && (dc&SQLITE_IOCAP_BATCH_ATOMIC) ){
return -1;
}
#endif
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
{
int nSector = pPager->sectorSize;
int szPage = pPager->pageSize;
assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));
if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){
return 0;
}
}
return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager);
#endif
return 0;
}
/*
** If SQLITE_CHECK_PAGES is defined then we do some sanity checking
** on the cache using a hash function. This is used for testing
** and debugging only.
*/
#ifdef SQLITE_CHECK_PAGES
/*
** Return a 32-bit hash of the page data for pPage.
*/
static u32 pager_datahash(int nByte, unsigned char *pData){
u32 hash = 0;
int i;
for(i=0; i<nByte; i++){
hash = (hash*1039) + pData[i];
}
return hash;
}
static u32 pager_pagehash(PgHdr *pPage){
return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData);
}
static void pager_set_pagehash(PgHdr *pPage){
pPage->pageHash = pager_pagehash(pPage);
}
/*
** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES
** is defined, and NDEBUG is not defined, an assert() statement checks
** that the page is either dirty or still matches the calculated page-hash.
*/
#define CHECK_PAGE(x) checkPage(x)
static void checkPage(PgHdr *pPg){
Pager *pPager = pPg->pPager;
assert( pPager->eState!=PAGER_ERROR );
assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) );
}
#else
#define pager_datahash(X,Y) 0
#define pager_pagehash(X) 0
#define pager_set_pagehash(X)
#define CHECK_PAGE(x)
#endif /* SQLITE_CHECK_PAGES */
/*
** When this is called the journal file for pager pPager must be open.
** This function attempts to read a master journal file name from the
** end of the file and, if successful, copies it into memory supplied
** by the caller. See comments above writeMasterJournal() for the format
** used to store a master journal file name at the end of a journal file.
**
** zMaster must point to a buffer of at least nMaster bytes allocated by
** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is
** enough space to write the master journal name). If the master journal
** name in the journal is longer than nMaster bytes (including a
** nul-terminator), then this is handled as if no master journal name
** were present in the journal.
**
** If a master journal file name is present at the end of the journal
** file, then it is copied into the buffer pointed to by zMaster. A
** nul-terminator byte is appended to the buffer following the master
** journal file name.
**
** If it is determined that no master journal file name is present
** zMaster[0] is set to 0 and SQLITE_OK returned.
**
** If an error occurs while reading from the journal file, an SQLite
** error code is returned.
*/
static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){
int rc; /* Return code */
u32 len; /* Length in bytes of master journal name */
i64 szJ; /* Total size in bytes of journal file pJrnl */
u32 cksum; /* MJ checksum value read from journal */
u32 u; /* Unsigned loop counter */
unsigned char aMagic[8]; /* A buffer to hold the magic header */
zMaster[0] = '\0';
if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ))
|| szJ<16
|| SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len))
|| len>=nMaster
|| len>szJ-16
|| len==0
|| SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum))
|| SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8))
|| memcmp(aMagic, aJournalMagic, 8)
|| SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len))
){
return rc;
}
/* See if the checksum matches the master journal name */
for(u=0; u<len; u++){
cksum -= zMaster[u];
}
if( cksum ){
/* If the checksum doesn't add up, then one or more of the disk sectors
** containing the master journal filename is corrupted. This means
** definitely roll back, so just return SQLITE_OK and report a (nul)
** master-journal filename.
*/
len = 0;
}
zMaster[len] = '\0';
return SQLITE_OK;
}
/*
** Return the offset of the sector boundary at or immediately
** following the value in pPager->journalOff, assuming a sector
** size of pPager->sectorSize bytes.
**
** i.e for a sector size of 512:
**
** Pager.journalOff Return value
** ---------------------------------------
** 0 0
** 512 512
** 100 512
** 2000 2048
**
*/
static i64 journalHdrOffset(Pager *pPager){
i64 offset = 0;
i64 c = pPager->journalOff;
if( c ){
offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager);
}
assert( offset%JOURNAL_HDR_SZ(pPager)==0 );
assert( offset>=c );
assert( (offset-c)<JOURNAL_HDR_SZ(pPager) );
return offset;
}
/*
** The journal file must be open when this function is called.
**
** This function is a no-op if the journal file has not been written to
** within the current transaction (i.e. if Pager.journalOff==0).
**
** If doTruncate is non-zero or the Pager.journalSizeLimit variable is
** set to 0, then truncate the journal file to zero bytes in size. Otherwise,
** zero the 28-byte header at the start of the journal file. In either case,
** if the pager is not in no-sync mode, sync the journal file immediately
** after writing or truncating it.
**
** If Pager.journalSizeLimit is set to a positive, non-zero value, and
** following the truncation or zeroing described above the size of the
** journal file in bytes is larger than this value, then truncate the
** journal file to Pager.journalSizeLimit bytes. The journal file does
** not need to be synced following this operation.
**
** If an IO error occurs, abandon processing and return the IO error code.
** Otherwise, return SQLITE_OK.
*/
static int zeroJournalHdr(Pager *pPager, int doTruncate){
int rc = SQLITE_OK; /* Return code */
assert( isOpen(pPager->jfd) );
assert( !sqlite3JournalIsInMemory(pPager->jfd) );
if( pPager->journalOff ){
const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */
IOTRACE(("JZEROHDR %p\n", pPager))
if( doTruncate || iLimit==0 ){
rc = sqlite3OsTruncate(pPager->jfd, 0);
}else{
static const char zeroHdr[28] = {0};
rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0);
}
if( rc==SQLITE_OK && !pPager->noSync ){
rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags);
}
/* At this point the transaction is committed but the write lock
** is still held on the file. If there is a size limit configured for
** the persistent journal and the journal file currently consumes more
** space than that limit allows for, truncate it now. There is no need
** to sync the file following this operation.
*/
if( rc==SQLITE_OK && iLimit>0 ){
i64 sz;
rc = sqlite3OsFileSize(pPager->jfd, &sz);
if( rc==SQLITE_OK && sz>iLimit ){
rc = sqlite3OsTruncate(pPager->jfd, iLimit);
}
}
}
return rc;
}
/*
** The journal file must be open when this routine is called. A journal
** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the
** current location.
**
** The format for the journal header is as follows:
** - 8 bytes: Magic identifying journal format.
** - 4 bytes: Number of records in journal, or -1 no-sync mode is on.
** - 4 bytes: Random number used for page hash.
** - 4 bytes: Initial database page count.
** - 4 bytes: Sector size used by the process that wrote this journal.
** - 4 bytes: Database page size.
**
** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space.
*/
static int writeJournalHdr(Pager *pPager){
int rc = SQLITE_OK; /* Return code */
char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */
u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */
u32 nWrite; /* Bytes of header sector written */
int ii; /* Loop counter */
assert( isOpen(pPager->jfd) ); /* Journal file must be open. */
if( nHeader>JOURNAL_HDR_SZ(pPager) ){
nHeader = JOURNAL_HDR_SZ(pPager);
}
/* If there are active savepoints and any of them were created
** since the most recent journal header was written, update the
** PagerSavepoint.iHdrOffset fields now.
*/
for(ii=0; ii<pPager->nSavepoint; ii++){
if( pPager->aSavepoint[ii].iHdrOffset==0 ){
pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff;
}
}
pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager);
/*
** Write the nRec Field - the number of page records that follow this
** journal header. Normally, zero is written to this value at this time.
** After the records are added to the journal (and the journal synced,
** if in full-sync mode), the zero is overwritten with the true number
** of records (see syncJournal()).
**
** A faster alternative is to write 0xFFFFFFFF to the nRec field. When
** reading the journal this value tells SQLite to assume that the
** rest of the journal file contains valid page records. This assumption
** is dangerous, as if a failure occurred whilst writing to the journal
** file it may contain some garbage data. There are two scenarios
** where this risk can be ignored:
**
** * When the pager is in no-sync mode. Corruption can follow a
** power failure in this case anyway.
**
** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees
** that garbage data is never appended to the journal file.
*/
assert( isOpen(pPager->fd) || pPager->noSync );
if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY)
|| (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND)
){
memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic));
put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff);
}else{
memset(zHeader, 0, sizeof(aJournalMagic)+4);
}
/* The random check-hash initializer */
sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
/* The initial database size */
put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize);
/* The assumed sector size for this process */
put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize);
/* The page size */
put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize);
/* Initializing the tail of the buffer is not necessary. Everything
** works find if the following memset() is omitted. But initializing
** the memory prevents valgrind from complaining, so we are willing to
** take the performance hit.
*/
memset(&zHeader[sizeof(aJournalMagic)+20], 0,
nHeader-(sizeof(aJournalMagic)+20));
/* In theory, it is only necessary to write the 28 bytes that the
** journal header consumes to the journal file here. Then increment the
** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next
** record is written to the following sector (leaving a gap in the file
** that will be implicitly filled in by the OS).
**
** However it has been discovered that on some systems this pattern can
** be significantly slower than contiguously writing data to the file,
** even if that means explicitly writing data to the block of
** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what
** is done.
**
** The loop is required here in case the sector-size is larger than the
** database page size. Since the zHeader buffer is only Pager.pageSize
** bytes in size, more than one call to sqlite3OsWrite() may be required
** to populate the entire journal header sector.
*/
for(nWrite=0; rc==SQLITE_OK&&nWrite<JOURNAL_HDR_SZ(pPager); nWrite+=nHeader){
IOTRACE(("JHDR %p %lld %d\n", pPager, pPager->journalHdr, nHeader))
rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff);
assert( pPager->journalHdr <= pPager->journalOff );
pPager->journalOff += nHeader;
}
return rc;
}
/*
** The journal file must be open when this is called. A journal header file
** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal
** file. The current location in the journal file is given by
** pPager->journalOff. See comments above function writeJournalHdr() for
** a description of the journal header format.
**
** If the header is read successfully, *pNRec is set to the number of
** page records following this header and *pDbSize is set to the size of the
** database before the transaction began, in pages. Also, pPager->cksumInit
** is set to the value read from the journal header. SQLITE_OK is returned
** in this case.
**
** If the journal header file appears to be corrupted, SQLITE_DONE is
** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes
** cannot be read from the journal file an error code is returned.
*/
static int readJournalHdr(
Pager *pPager, /* Pager object */
int isHot,
i64 journalSize, /* Size of the open journal file in bytes */
u32 *pNRec, /* OUT: Value read from the nRec field */
u32 *pDbSize /* OUT: Value of original database size field */
){
int rc; /* Return code */
unsigned char aMagic[8]; /* A buffer to hold the magic header */
i64 iHdrOff; /* Offset of journal header being read */
assert( isOpen(pPager->jfd) ); /* Journal file must be open. */
/* Advance Pager.journalOff to the start of the next sector. If the
** journal file is too small for there to be a header stored at this
** point, return SQLITE_DONE.
*/
pPager->journalOff = journalHdrOffset(pPager);
if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){
return SQLITE_DONE;
}
iHdrOff = pPager->journalOff;
/* Read in the first 8 bytes of the journal header. If they do not match
** the magic string found at the start of each journal header, return
** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise,
** proceed.
*/
if( isHot || iHdrOff!=pPager->journalHdr ){
rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff);
if( rc ){
return rc;
}
if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){
return SQLITE_DONE;
}
}
/* Read the first three 32-bit fields of the journal header: The nRec
** field, the checksum-initializer and the database size at the start
** of the transaction. Return an error code if anything goes wrong.
*/
if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec))
|| SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit))
|| SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize))
){
return rc;
}
if( pPager->journalOff==0 ){
u32 iPageSize; /* Page-size field of journal header */
u32 iSectorSize; /* Sector-size field of journal header */
/* Read the page-size and sector-size journal header fields. */
if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize))
|| SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize))
){
return rc;
}
/* Versions of SQLite prior to 3.5.8 set the page-size field of the
** journal header to zero. In this case, assume that the Pager.pageSize
** variable is already set to the correct page size.
*/
if( iPageSize==0 ){
iPageSize = pPager->pageSize;
}
/* Check that the values read from the page-size and sector-size fields
** are within range. To be 'in range', both values need to be a power
** of two greater than or equal to 512 or 32, and not greater than their
** respective compile time maximum limits.
*/
if( iPageSize<512 || iSectorSize<32
|| iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE
|| ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0
){
/* If the either the page-size or sector-size in the journal-header is
** invalid, then the process that wrote the journal-header must have
** crashed before the header was synced. In this case stop reading
** the journal file here.
*/
return SQLITE_DONE;
}
/* Update the page-size to match the value read from the journal.
** Use a testcase() macro to make sure that malloc failure within
** PagerSetPagesize() is tested.
*/
rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1);
testcase( rc!=SQLITE_OK );
/* Update the assumed sector-size to match the value used by
** the process that created this journal. If this journal was
** created by a process other than this one, then this routine
** is being called from within pager_playback(). The local value
** of Pager.sectorSize is restored at the end of that routine.
*/
pPager->sectorSize = iSectorSize;
}
pPager->journalOff += JOURNAL_HDR_SZ(pPager);
return rc;
}
/*
** Write the supplied master journal name into the journal file for pager
** pPager at the current location. The master journal name must be the last
** thing written to a journal file. If the pager is in full-sync mode, the
** journal file descriptor is advanced to the next sector boundary before
** anything is written. The format is:
**
** + 4 bytes: PAGER_MJ_PGNO.
** + N bytes: Master journal filename in utf-8.
** + 4 bytes: N (length of master journal name in bytes, no nul-terminator).
** + 4 bytes: Master journal name checksum.
** + 8 bytes: aJournalMagic[].
**
** The master journal page checksum is the sum of the bytes in the master
** journal name, where each byte is interpreted as a signed 8-bit integer.
**
** If zMaster is a NULL pointer (occurs for a single database transaction),
** this call is a no-op.
*/
static int writeMasterJournal(Pager *pPager, const char *zMaster){
int rc; /* Return code */
int nMaster; /* Length of string zMaster */
i64 iHdrOff; /* Offset of header in journal file */
i64 jrnlSize; /* Size of journal file on disk */
u32 cksum = 0; /* Checksum of string zMaster */
assert( pPager->setMaster==0 );
assert( !pagerUseWal(pPager) );
if( !zMaster
|| pPager->journalMode==PAGER_JOURNALMODE_MEMORY
|| !isOpen(pPager->jfd)
){
return SQLITE_OK;
}
pPager->setMaster = 1;
assert( pPager->journalHdr <= pPager->journalOff );
/* Calculate the length in bytes and the checksum of zMaster */
for(nMaster=0; zMaster[nMaster]; nMaster++){
cksum += zMaster[nMaster];
}
/* If in full-sync mode, advance to the next disk sector before writing
** the master journal name. This is in case the previous page written to
** the journal has already been synced.
*/
if( pPager->fullSync ){
pPager->journalOff = journalHdrOffset(pPager);
}
iHdrOff = pPager->journalOff;
/* Write the master journal data to the end of the journal file. If
** an error occurs, return the error code to the caller.
*/
if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager))))
|| (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4)))
|| (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster)))
|| (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum)))
|| (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8,
iHdrOff+4+nMaster+8)))
){
return rc;
}
pPager->journalOff += (nMaster+20);
/* If the pager is in peristent-journal mode, then the physical
** journal-file may extend past the end of the master-journal name
** and 8 bytes of magic data just written to the file. This is
** dangerous because the code to rollback a hot-journal file
** will not be able to find the master-journal name to determine
** whether or not the journal is hot.
**
** Easiest thing to do in this scenario is to truncate the journal
** file to the required size.
*/
if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize))
&& jrnlSize>pPager->journalOff
){
rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff);
}
return rc;
}
/*
** Discard the entire contents of the in-memory page-cache.
*/
static void pager_reset(Pager *pPager){
pPager->iDataVersion++;
sqlite3BackupRestart(pPager->pBackup);
sqlite3PcacheClear(pPager->pPCache);
}
/*
** Return the pPager->iDataVersion value
*/
u32 sqlite3PagerDataVersion(Pager *pPager){
return pPager->iDataVersion;
}
/*
** Free all structures in the Pager.aSavepoint[] array and set both
** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal
** if it is open and the pager is not in exclusive mode.
*/
static void releaseAllSavepoints(Pager *pPager){
int ii; /* Iterator for looping through Pager.aSavepoint */
for(ii=0; ii<pPager->nSavepoint; ii++){
sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
}
if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){
sqlite3OsClose(pPager->sjfd);
}
sqlite3_free(pPager->aSavepoint);
pPager->aSavepoint = 0;
pPager->nSavepoint = 0;
pPager->nSubRec = 0;
}
/*
** Set the bit number pgno in the PagerSavepoint.pInSavepoint
** bitvecs of all open savepoints. Return SQLITE_OK if successful
** or SQLITE_NOMEM if a malloc failure occurs.
*/
static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
int ii; /* Loop counter */
int rc = SQLITE_OK; /* Result code */
for(ii=0; ii<pPager->nSavepoint; ii++){
PagerSavepoint *p = &pPager->aSavepoint[ii];
if( pgno<=p->nOrig ){
rc |= sqlite3BitvecSet(p->pInSavepoint, pgno);
testcase( rc==SQLITE_NOMEM );
assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
}
}
return rc;
}
/*
** This function is a no-op if the pager is in exclusive mode and not
** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN
** state.
**
** If the pager is not in exclusive-access mode, the database file is
** completely unlocked. If the file is unlocked and the file-system does
** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is
** closed (if it is open).
**
** If the pager is in ERROR state when this function is called, the
** contents of the pager cache are discarded before switching back to
** the OPEN state. Regardless of whether the pager is in exclusive-mode
** or not, any journal file left in the file-system will be treated
** as a hot-journal and rolled back the next time a read-transaction
** is opened (by this or by any other connection).
*/
static void pager_unlock(Pager *pPager){
assert( pPager->eState==PAGER_READER
|| pPager->eState==PAGER_OPEN
|| pPager->eState==PAGER_ERROR
);
sqlite3BitvecDestroy(pPager->pInJournal);
pPager->pInJournal = 0;
releaseAllSavepoints(pPager);
if( pagerUseWal(pPager) ){
assert( !isOpen(pPager->jfd) );
sqlite3WalEndReadTransaction(pPager->pWal);
pPager->eState = PAGER_OPEN;
}else if( !pPager->exclusiveMode ){
int rc; /* Error code returned by pagerUnlockDb() */
int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0;
/* If the operating system support deletion of open files, then
** close the journal file when dropping the database lock. Otherwise
** another connection with journal_mode=delete might delete the file
** out from under us.
*/
assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 );
assert( (PAGER_JOURNALMODE_OFF & 5)!=1 );
assert( (PAGER_JOURNALMODE_WAL & 5)!=1 );
assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 );
assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 );
assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 );
if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN)
|| 1!=(pPager->journalMode & 5)
){
sqlite3OsClose(pPager->jfd);
}
/* If the pager is in the ERROR state and the call to unlock the database
** file fails, set the current lock to UNKNOWN_LOCK. See the comment
** above the #define for UNKNOWN_LOCK for an explanation of why this
** is necessary.
*/
rc = pagerUnlockDb(pPager, NO_LOCK);
if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){
pPager->eLock = UNKNOWN_LOCK;
}
/* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here
** without clearing the error code. This is intentional - the error
** code is cleared and the cache reset in the block below.
*/
assert( pPager->errCode || pPager->eState!=PAGER_ERROR );
pPager->changeCountDone = 0;
pPager->eState = PAGER_OPEN;
}
/* If Pager.errCode is set, the contents of the pager cache cannot be
** trusted. Now that there are no outstanding references to the pager,
** it can safely move back to PAGER_OPEN state. This happens in both
** normal and exclusive-locking mode.
*/
assert( pPager->errCode==SQLITE_OK || !MEMDB );
if( pPager->errCode ){
if( pPager->tempFile==0 ){
pager_reset(pPager);
pPager->changeCountDone = 0;
pPager->eState = PAGER_OPEN;
}else{
pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER);
}
if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
pPager->errCode = SQLITE_OK;
setGetterMethod(pPager);
}
pPager->journalOff = 0;
pPager->journalHdr = 0;
pPager->setMaster = 0;
}
/*
** This function is called whenever an IOERR or FULL error that requires
** the pager to transition into the ERROR state may ahve occurred.
** The first argument is a pointer to the pager structure, the second
** the error-code about to be returned by a pager API function. The
** value returned is a copy of the second argument to this function.
**
** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the
** IOERR sub-codes, the pager enters the ERROR state and the error code
** is stored in Pager.errCode. While the pager remains in the ERROR state,
** all major API calls on the Pager will immediately return Pager.errCode.
**
** The ERROR state indicates that the contents of the pager-cache
** cannot be trusted. This state can be cleared by completely discarding
** the contents of the pager-cache. If a transaction was active when
** the persistent error occurred, then the rollback journal may need
** to be replayed to restore the contents of the database file (as if
** it were a hot-journal).
*/
static int pager_error(Pager *pPager, int rc){
int rc2 = rc & 0xff;
assert( rc==SQLITE_OK || !MEMDB );
assert(
pPager->errCode==SQLITE_FULL ||
pPager->errCode==SQLITE_OK ||
(pPager->errCode & 0xff)==SQLITE_IOERR
);
if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){
pPager->errCode = rc;
pPager->eState = PAGER_ERROR;
setGetterMethod(pPager);
}
return rc;
}
static int pager_truncate(Pager *pPager, Pgno nPage);
/*
** The write transaction open on pPager is being committed (bCommit==1)
** or rolled back (bCommit==0).
**
** Return TRUE if and only if all dirty pages should be flushed to disk.
**
** Rules:
**
** * For non-TEMP databases, always sync to disk. This is necessary
** for transactions to be durable.
**
** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing
** file has been created already (via a spill on pagerStress()) and
** when the number of dirty pages in memory exceeds 25% of the total
** cache size.
*/
static int pagerFlushOnCommit(Pager *pPager, int bCommit){
if( pPager->tempFile==0 ) return 1;
if( !bCommit ) return 0;
if( !isOpen(pPager->fd) ) return 0;
return (sqlite3PCachePercentDirty(pPager->pPCache)>=25);
}
/*
** This routine ends a transaction. A transaction is usually ended by
** either a COMMIT or a ROLLBACK operation. This routine may be called
** after rollback of a hot-journal, or if an error occurs while opening
** the journal file or writing the very first journal-header of a
** database transaction.
**
** This routine is never called in PAGER_ERROR state. If it is called
** in PAGER_NONE or PAGER_SHARED state and the lock held is less
** exclusive than a RESERVED lock, it is a no-op.
**
** Otherwise, any active savepoints are released.
**
** If the journal file is open, then it is "finalized". Once a journal
** file has been finalized it is not possible to use it to roll back a
** transaction. Nor will it be considered to be a hot-journal by this
** or any other database connection. Exactly how a journal is finalized
** depends on whether or not the pager is running in exclusive mode and
** the current journal-mode (Pager.journalMode value), as follows:
**
** journalMode==MEMORY
** Journal file descriptor is simply closed. This destroys an
** in-memory journal.
**
** journalMode==TRUNCATE
** Journal file is truncated to zero bytes in size.
**
** journalMode==PERSIST
** The first 28 bytes of the journal file are zeroed. This invalidates
** the first journal header in the file, and hence the entire journal
** file. An invalid journal file cannot be rolled back.
**
** journalMode==DELETE
** The journal file is closed and deleted using sqlite3OsDelete().
**
** If the pager is running in exclusive mode, this method of finalizing
** the journal file is never used. Instead, if the journalMode is
** DELETE and the pager is in exclusive mode, the method described under
** journalMode==PERSIST is used instead.
**
** After the journal is finalized, the pager moves to PAGER_READER state.
** If running in non-exclusive rollback mode, the lock on the file is
** downgraded to a SHARED_LOCK.
**
** SQLITE_OK is returned if no error occurs. If an error occurs during
** any of the IO operations to finalize the journal file or unlock the
** database then the IO error code is returned to the user. If the
** operation to finalize the journal file fails, then the code still
** tries to unlock the database file if not in exclusive mode. If the
** unlock operation fails as well, then the first error code related
** to the first error encountered (the journal finalization one) is
** returned.
*/
static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
int rc = SQLITE_OK; /* Error code from journal finalization operation */
int rc2 = SQLITE_OK; /* Error code from db file unlock operation */
/* Do nothing if the pager does not have an open write transaction
** or at least a RESERVED lock. This function may be called when there
** is no write-transaction active but a RESERVED or greater lock is
** held under two circumstances:
**
** 1. After a successful hot-journal rollback, it is called with
** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK.
**
** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE
** lock switches back to locking_mode=normal and then executes a
** read-transaction, this function is called with eState==PAGER_READER
** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed.
*/
assert( assert_pager_state(pPager) );
assert( pPager->eState!=PAGER_ERROR );
if( pPager->eState<PAGER_WRITER_LOCKED && pPager->eLock<RESERVED_LOCK ){
return SQLITE_OK;
}
releaseAllSavepoints(pPager);
assert( isOpen(pPager->jfd) || pPager->pInJournal==0
|| (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
);
if( isOpen(pPager->jfd) ){
assert( !pagerUseWal(pPager) );
/* Finalize the journal file. */
if( sqlite3JournalIsInMemory(pPager->jfd) ){
/* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */
sqlite3OsClose(pPager->jfd);
}else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){
if( pPager->journalOff==0 ){
rc = SQLITE_OK;
}else{
rc = sqlite3OsTruncate(pPager->jfd, 0);
if( rc==SQLITE_OK && pPager->fullSync ){
/* Make sure the new file size is written into the inode right away.
** Otherwise the journal might resurrect following a power loss and
** cause the last transaction to roll back. See
** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773
*/
rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags);
}
}
pPager->journalOff = 0;
}else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
|| (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
){
rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile);
pPager->journalOff = 0;
}else{
/* This branch may be executed with Pager.journalMode==MEMORY if
** a hot-journal was just rolled back. In this case the journal
** file should be closed and deleted. If this connection writes to
** the database file, it will do so using an in-memory journal.
*/
int bDelete = !pPager->tempFile;
assert( sqlite3JournalIsInMemory(pPager->jfd)==0 );
assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE
|| pPager->journalMode==PAGER_JOURNALMODE_MEMORY
|| pPager->journalMode==PAGER_JOURNALMODE_WAL
);
sqlite3OsClose(pPager->jfd);
if( bDelete ){
rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync);
}
}
}
#ifdef SQLITE_CHECK_PAGES
sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash);
if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){
PgHdr *p = sqlite3PagerLookup(pPager, 1);
if( p ){
p->pageHash = 0;
sqlite3PagerUnrefNotNull(p);
}
}
#endif
sqlite3BitvecDestroy(pPager->pInJournal);
pPager->pInJournal = 0;
pPager->nRec = 0;
if( rc==SQLITE_OK ){
if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){
sqlite3PcacheCleanAll(pPager->pPCache);
}else{
sqlite3PcacheClearWritable(pPager->pPCache);
}
sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
}
if( pagerUseWal(pPager) ){
/* Drop the WAL write-lock, if any. Also, if the connection was in
** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE
** lock held on the database file.
*/
rc2 = sqlite3WalEndWriteTransaction(pPager->pWal);
assert( rc2==SQLITE_OK );
}else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){
/* This branch is taken when committing a transaction in rollback-journal
** mode if the database file on disk is larger than the database image.
** At this point the journal has been finalized and the transaction
** successfully committed, but the EXCLUSIVE lock is still held on the
** file. So it is safe to truncate the database file to its minimum
** required size. */
assert( pPager->eLock==EXCLUSIVE_LOCK );
rc = pager_truncate(pPager, pPager->dbSize);
}
if( rc==SQLITE_OK && bCommit ){
rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0);
if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
}
if( !pPager->exclusiveMode
&& (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0))
){
rc2 = pagerUnlockDb(pPager, SHARED_LOCK);
pPager->changeCountDone = 0;
}
pPager->eState = PAGER_READER;
pPager->setMaster = 0;
return (rc==SQLITE_OK?rc2:rc);
}
/*
** Execute a rollback if a transaction is active and unlock the
** database file.
**
** If the pager has already entered the ERROR state, do not attempt
** the rollback at this time. Instead, pager_unlock() is called. The
** call to pager_unlock() will discard all in-memory pages, unlock
** the database file and move the pager back to OPEN state. If this
** means that there is a hot-journal left in the file-system, the next
** connection to obtain a shared lock on the pager (which may be this one)
** will roll it back.
**
** If the pager has not already entered the ERROR state, but an IO or
** malloc error occurs during a rollback, then this will itself cause
** the pager to enter the ERROR state. Which will be cleared by the
** call to pager_unlock(), as described above.
*/
static void pagerUnlockAndRollback(Pager *pPager){
if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){
assert( assert_pager_state(pPager) );
if( pPager->eState>=PAGER_WRITER_LOCKED ){
sqlite3BeginBenignMalloc();
sqlite3PagerRollback(pPager);
sqlite3EndBenignMalloc();
}else if( !pPager->exclusiveMode ){
assert( pPager->eState==PAGER_READER );
pager_end_transaction(pPager, 0, 0);
}
}
pager_unlock(pPager);
}
/*
** Parameter aData must point to a buffer of pPager->pageSize bytes
** of data. Compute and return a checksum based ont the contents of the
** page of data and the current value of pPager->cksumInit.
**
** This is not a real checksum. It is really just the sum of the
** random initial value (pPager->cksumInit) and every 200th byte
** of the page data, starting with byte offset (pPager->pageSize%200).
** Each byte is interpreted as an 8-bit unsigned integer.
**
** Changing the formula used to compute this checksum results in an
** incompatible journal file format.
**
** If journal corruption occurs due to a power failure, the most likely
** scenario is that one end or the other of the record will be changed.
** It is much less likely that the two ends of the journal record will be
** correct and the middle be corrupt. Thus, this "checksum" scheme,
** though fast and simple, catches the mostly likely kind of corruption.
*/
static u32 pager_cksum(Pager *pPager, const u8 *aData){
u32 cksum = pPager->cksumInit; /* Checksum value to return */
int i = pPager->pageSize-200; /* Loop counter */
while( i>0 ){
cksum += aData[i];
i -= 200;
}
return cksum;
}
/*
** Report the current page size and number of reserved bytes back
** to the codec.
*/
#ifdef SQLITE_HAS_CODEC
static void pagerReportSize(Pager *pPager){
if( pPager->xCodecSizeChng ){
pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize,
(int)pPager->nReserve);
}
}
#else
# define pagerReportSize(X) /* No-op if we do not support a codec */
#endif
#ifdef SQLITE_HAS_CODEC
/*
** Make sure the number of reserved bits is the same in the destination
** pager as it is in the source. This comes up when a VACUUM changes the
** number of reserved bits to the "optimal" amount.
*/
void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){
if( pDest->nReserve!=pSrc->nReserve ){
pDest->nReserve = pSrc->nReserve;
pagerReportSize(pDest);
}
}
#endif
/*
** Read a single page from either the journal file (if isMainJrnl==1) or
** from the sub-journal (if isMainJrnl==0) and playback that page.
** The page begins at offset *pOffset into the file. The *pOffset
** value is increased to the start of the next page in the journal.
**
** The main rollback journal uses checksums - the statement journal does
** not.
**
** If the page number of the page record read from the (sub-)journal file
** is greater than the current value of Pager.dbSize, then playback is
** skipped and SQLITE_OK is returned.
**
** If pDone is not NULL, then it is a record of pages that have already
** been played back. If the page at *pOffset has already been played back
** (if the corresponding pDone bit is set) then skip the playback.
** Make sure the pDone bit corresponding to the *pOffset page is set
** prior to returning.
**
** If the page record is successfully read from the (sub-)journal file
** and played back, then SQLITE_OK is returned. If an IO error occurs
** while reading the record from the (sub-)journal file or while writing
** to the database file, then the IO error code is returned. If data
** is successfully read from the (sub-)journal file but appears to be
** corrupted, SQLITE_DONE is returned. Data is considered corrupted in
** two circumstances:
**
** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or
** * If the record is being rolled back from the main journal file
** and the checksum field does not match the record content.
**
** Neither of these two scenarios are possible during a savepoint rollback.
**
** If this is a savepoint rollback, then memory may have to be dynamically
** allocated by this function. If this is the case and an allocation fails,
** SQLITE_NOMEM is returned.
*/
static int pager_playback_one_page(
Pager *pPager, /* The pager being played back */
i64 *pOffset, /* Offset of record to playback */
Bitvec *pDone, /* Bitvec of pages already played back */
int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */
int isSavepnt /* True for a savepoint rollback */
){
int rc;
PgHdr *pPg; /* An existing page in the cache */
Pgno pgno; /* The page number of a page in journal */
u32 cksum; /* Checksum used for sanity checking */
char *aData; /* Temporary storage for the page */
sqlite3_file *jfd; /* The file descriptor for the journal file */
int isSynced; /* True if journal page is synced */
#ifdef SQLITE_HAS_CODEC
/* The jrnlEnc flag is true if Journal pages should be passed through
** the codec. It is false for pure in-memory journals. */
const int jrnlEnc = (isMainJrnl || pPager->subjInMemory==0);
#endif
assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */
assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */
assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */
aData = pPager->pTmpSpace;
assert( aData ); /* Temp storage must have already been allocated */
assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) );
/* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction
** or savepoint rollback done at the request of the caller) or this is
** a hot-journal rollback. If it is a hot-journal rollback, the pager
** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback
** only reads from the main journal, not the sub-journal.
*/
assert( pPager->eState>=PAGER_WRITER_CACHEMOD
|| (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK)
);
assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl );
/* Read the page number and page data from the journal or sub-journal
** file. Return an error code to the caller if an IO error occurs.
*/
jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
rc = read32bits(jfd, *pOffset, &pgno);
if( rc!=SQLITE_OK ) return rc;
rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4);
if( rc!=SQLITE_OK ) return rc;
*pOffset += pPager->pageSize + 4 + isMainJrnl*4;
/* Sanity checking on the page. This is more important that I originally
** thought. If a power failure occurs while the journal is being written,
** it could cause invalid data to be written into the journal. We need to
** detect this invalid data (with high probability) and ignore it.
*/
if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){
assert( !isSavepnt );
return SQLITE_DONE;
}
if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){
return SQLITE_OK;
}
if( isMainJrnl ){
rc = read32bits(jfd, (*pOffset)-4, &cksum);
if( rc ) return rc;
if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){
return SQLITE_DONE;
}
}
/* If this page has already been played back before during the current
** rollback, then don't bother to play it back again.
*/
if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){
return rc;
}
/* When playing back page 1, restore the nReserve setting
*/
if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){
pPager->nReserve = ((u8*)aData)[20];
pagerReportSize(pPager);
}
/* If the pager is in CACHEMOD state, then there must be a copy of this
** page in the pager cache. In this case just update the pager cache,
** not the database file. The page is left marked dirty in this case.
**
** An exception to the above rule: If the database is in no-sync mode
** and a page is moved during an incremental vacuum then the page may
** not be in the pager cache. Later: if a malloc() or IO error occurs
** during a Movepage() call, then the page may not be in the cache
** either. So the condition described in the above paragraph is not
** assert()able.
**
** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the
** pager cache if it exists and the main file. The page is then marked
** not dirty. Since this code is only executed in PAGER_OPEN state for
** a hot-journal rollback, it is guaranteed that the page-cache is empty
** if the pager is in OPEN state.
**
** Ticket #1171: The statement journal might contain page content that is
** different from the page content at the start of the transaction.
** This occurs when a page is changed prior to the start of a statement
** then changed again within the statement. When rolling back such a
** statement we must not write to the original database unless we know
** for certain that original page contents are synced into the main rollback
** journal. Otherwise, a power loss might leave modified data in the
** database file without an entry in the rollback journal that can
** restore the database to its original form. Two conditions must be
** met before writing to the database files. (1) the database must be
** locked. (2) we know that the original page content is fully synced
** in the main journal either because the page is not in cache or else
** the page is marked as needSync==0.
**
** 2008-04-14: When attempting to vacuum a corrupt database file, it
** is possible to fail a statement on a database that does not yet exist.
** Do not attempt to write if database file has never been opened.
*/
if( pagerUseWal(pPager) ){
pPg = 0;
}else{
pPg = sqlite3PagerLookup(pPager, pgno);
}
assert( pPg || !MEMDB );
assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile );
PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
(isMainJrnl?"main-journal":"sub-journal")
));
if( isMainJrnl ){
isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr);
}else{
isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC));
}
if( isOpen(pPager->fd)
&& (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
&& isSynced
){
i64 ofst = (pgno-1)*(i64)pPager->pageSize;
testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 );
assert( !pagerUseWal(pPager) );
/* Write the data read from the journal back into the database file.
** This is usually safe even for an encrypted database - as the data
** was encrypted before it was written to the journal file. The exception
** is if the data was just read from an in-memory sub-journal. In that
** case it must be encrypted here before it is copied into the database
** file. */
#ifdef SQLITE_HAS_CODEC
if( !jrnlEnc ){
CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData);
rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst);
CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT);
}else
#endif
rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst);
if( pgno>pPager->dbFileSize ){
pPager->dbFileSize = pgno;
}
if( pPager->pBackup ){
#ifdef SQLITE_HAS_CODEC
if( jrnlEnc ){
CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT);
sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT,aData);
}else
#endif
sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
}
}else if( !isMainJrnl && pPg==0 ){
/* If this is a rollback of a savepoint and data was not written to
** the database and the page is not in-memory, there is a potential
** problem. When the page is next fetched by the b-tree layer, it
** will be read from the database file, which may or may not be
** current.
**
** There are a couple of different ways this can happen. All are quite
** obscure. When running in synchronous mode, this can only happen
** if the page is on the free-list at the start of the transaction, then
** populated, then moved using sqlite3PagerMovepage().
**
** The solution is to add an in-memory page to the cache containing
** the data just read from the sub-journal. Mark the page as dirty
** and if the pager requires a journal-sync, then mark the page as
** requiring a journal-sync before it is written.
*/
assert( isSavepnt );
assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 );
pPager->doNotSpill |= SPILLFLAG_ROLLBACK;
rc = sqlite3PagerGet(pPager, pgno, &pPg, 1);
assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 );
pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK;
if( rc!=SQLITE_OK ) return rc;
sqlite3PcacheMakeDirty(pPg);
}
if( pPg ){
/* No page should ever be explicitly rolled back that is in use, except
** for page 1 which is held in use in order to keep the lock on the
** database active. However such a page may be rolled back as a result
** of an internal error resulting in an automatic call to
** sqlite3PagerRollback().
*/
void *pData;
pData = pPg->pData;
memcpy(pData, (u8*)aData, pPager->pageSize);
pPager->xReiniter(pPg);
/* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But
** that call was dangerous and had no detectable benefit since the cache
** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so
** has been removed. */
pager_set_pagehash(pPg);
/* If this was page 1, then restore the value of Pager.dbFileVers.
** Do this before any decoding. */
if( pgno==1 ){
memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers));
}
/* Decode the page just read from disk */
#if SQLITE_HAS_CODEC
if( jrnlEnc ){ CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); }
#endif
sqlite3PcacheRelease(pPg);
}
return rc;
}
/*
** Parameter zMaster is the name of a master journal file. A single journal
** file that referred to the master journal file has just been rolled back.
** This routine checks if it is possible to delete the master journal file,
** and does so if it is.
**
** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not
** available for use within this function.
**
** When a master journal file is created, it is populated with the names
** of all of its child journals, one after another, formatted as utf-8
** encoded text. The end of each child journal file is marked with a
** nul-terminator byte (0x00). i.e. the entire contents of a master journal
** file for a transaction involving two databases might be:
**
** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00"
**
** A master journal file may only be deleted once all of its child
** journals have been rolled back.
**
** This function reads the contents of the master-journal file into
** memory and loops through each of the child journal names. For
** each child journal, it checks if:
**
** * if the child journal exists, and if so
** * if the child journal contains a reference to master journal
** file zMaster
**
** If a child journal can be found that matches both of the criteria
** above, this function returns without doing anything. Otherwise, if
** no such child journal can be found, file zMaster is deleted from
** the file-system using sqlite3OsDelete().
**
** If an IO error within this function, an error code is returned. This
** function allocates memory by calling sqlite3Malloc(). If an allocation
** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors
** occur, SQLITE_OK is returned.
**
** TODO: This function allocates a single block of memory to load
** the entire contents of the master journal file. This could be
** a couple of kilobytes or so - potentially larger than the page
** size.
*/
static int pager_delmaster(Pager *pPager, const char *zMaster){
sqlite3_vfs *pVfs = pPager->pVfs;
int rc; /* Return code */
sqlite3_file *pMaster; /* Malloc'd master-journal file descriptor */
sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */
char *zMasterJournal = 0; /* Contents of master journal file */
i64 nMasterJournal; /* Size of master journal file */
char *zJournal; /* Pointer to one journal within MJ file */
char *zMasterPtr; /* Space to hold MJ filename from a journal file */
int nMasterPtr; /* Amount of space allocated to zMasterPtr[] */
/* Allocate space for both the pJournal and pMaster file descriptors.
** If successful, open the master journal file for reading.
*/
pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2);
pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile);
if( !pMaster ){
rc = SQLITE_NOMEM_BKPT;
}else{
const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL);
rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0);
}
if( rc!=SQLITE_OK ) goto delmaster_out;
/* Load the entire master journal file into space obtained from
** sqlite3_malloc() and pointed to by zMasterJournal. Also obtain
** sufficient space (in zMasterPtr) to hold the names of master
** journal files extracted from regular rollback-journals.
*/
rc = sqlite3OsFileSize(pMaster, &nMasterJournal);
if( rc!=SQLITE_OK ) goto delmaster_out;
nMasterPtr = pVfs->mxPathname+1;
zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1);
if( !zMasterJournal ){
rc = SQLITE_NOMEM_BKPT;
goto delmaster_out;
}
zMasterPtr = &zMasterJournal[nMasterJournal+1];
rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0);
if( rc!=SQLITE_OK ) goto delmaster_out;
zMasterJournal[nMasterJournal] = 0;
zJournal = zMasterJournal;
while( (zJournal-zMasterJournal)<nMasterJournal ){
int exists;
rc = sqlite3OsAccess(pVfs, zJournal, SQLITE_ACCESS_EXISTS, &exists);
if( rc!=SQLITE_OK ){
goto delmaster_out;
}
if( exists ){
/* One of the journals pointed to by the master journal exists.
** Open it and check if it points at the master journal. If
** so, return without deleting the master journal file.
*/
int c;
int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL);
rc = sqlite3OsOpen(pVfs, zJournal, pJournal, flags, 0);
if( rc!=SQLITE_OK ){
goto delmaster_out;
}
rc = readMasterJournal(pJournal, zMasterPtr, nMasterPtr);
sqlite3OsClose(pJournal);
if( rc!=SQLITE_OK ){
goto delmaster_out;
}
c = zMasterPtr[0]!=0 && strcmp(zMasterPtr, zMaster)==0;
if( c ){
/* We have a match. Do not delete the master journal file. */
goto delmaster_out;
}
}
zJournal += (sqlite3Strlen30(zJournal)+1);
}
sqlite3OsClose(pMaster);
rc = sqlite3OsDelete(pVfs, zMaster, 0);
delmaster_out:
sqlite3_free(zMasterJournal);
if( pMaster ){
sqlite3OsClose(pMaster);
assert( !isOpen(pJournal) );
sqlite3_free(pMaster);
}
return rc;
}
/*
** This function is used to change the actual size of the database
** file in the file-system. This only happens when committing a transaction,
** or rolling back a transaction (including rolling back a hot-journal).
**
** If the main database file is not open, or the pager is not in either
** DBMOD or OPEN state, this function is a no-op. Otherwise, the size
** of the file is changed to nPage pages (nPage*pPager->pageSize bytes).
** If the file on disk is currently larger than nPage pages, then use the VFS
** xTruncate() method to truncate it.
**
** Or, it might be the case that the file on disk is smaller than
** nPage pages. Some operating system implementations can get confused if
** you try to truncate a file to some size that is larger than it
** currently is, so detect this case and write a single zero byte to
** the end of the new file instead.
**
** If successful, return SQLITE_OK. If an IO error occurs while modifying
** the database file, return the error code to the caller.
*/
static int pager_truncate(Pager *pPager, Pgno nPage){
int rc = SQLITE_OK;
assert( pPager->eState!=PAGER_ERROR );
assert( pPager->eState!=PAGER_READER );
if( isOpen(pPager->fd)
&& (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
){
i64 currentSize, newSize;
int szPage = pPager->pageSize;
assert( pPager->eLock==EXCLUSIVE_LOCK );
/* TODO: Is it safe to use Pager.dbFileSize here? */
rc = sqlite3OsFileSize(pPager->fd, &currentSize);
newSize = szPage*(i64)nPage;
if( rc==SQLITE_OK && currentSize!=newSize ){
if( currentSize>newSize ){
rc = sqlite3OsTruncate(pPager->fd, newSize);
}else if( (currentSize+szPage)<=newSize ){
char *pTmp = pPager->pTmpSpace;
memset(pTmp, 0, szPage);
testcase( (newSize-szPage) == currentSize );
testcase( (newSize-szPage) > currentSize );
rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage);
}
if( rc==SQLITE_OK ){
pPager->dbFileSize = nPage;
}
}
}
return rc;
}
/*
** Return a sanitized version of the sector-size of OS file pFile. The
** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE.
*/
int sqlite3SectorSize(sqlite3_file *pFile){
int iRet = sqlite3OsSectorSize(pFile);
if( iRet<32 ){
iRet = 512;
}else if( iRet>MAX_SECTOR_SIZE ){
assert( MAX_SECTOR_SIZE>=512 );
iRet = MAX_SECTOR_SIZE;
}
return iRet;
}
/*
** Set the value of the Pager.sectorSize variable for the given
** pager based on the value returned by the xSectorSize method
** of the open database file. The sector size will be used
** to determine the size and alignment of journal header and
** master journal pointers within created journal files.
**
** For temporary files the effective sector size is always 512 bytes.
**
** Otherwise, for non-temporary files, the effective sector size is
** the value returned by the xSectorSize() method rounded up to 32 if
** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it
** is greater than MAX_SECTOR_SIZE.
**
** If the file has the SQLITE_IOCAP_POWERSAFE_OVERWRITE property, then set
** the effective sector size to its minimum value (512). The purpose of
** pPager->sectorSize is to define the "blast radius" of bytes that
** might change if a crash occurs while writing to a single byte in
** that range. But with POWERSAFE_OVERWRITE, the blast radius is zero
** (that is what POWERSAFE_OVERWRITE means), so we minimize the sector
** size. For backwards compatibility of the rollback journal file format,
** we cannot reduce the effective sector size below 512.
*/
static void setSectorSize(Pager *pPager){
assert( isOpen(pPager->fd) || pPager->tempFile );
if( pPager->tempFile
|| (sqlite3OsDeviceCharacteristics(pPager->fd) &
SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0
){
/* Sector size doesn't matter for temporary files. Also, the file
** may not have been opened yet, in which case the OsSectorSize()
** call will segfault. */
pPager->sectorSize = 512;
}else{
pPager->sectorSize = sqlite3SectorSize(pPager->fd);
}
}
/*
** Playback the journal and thus restore the database file to
** the state it was in before we started making changes.
**
** The journal file format is as follows:
**
** (1) 8 byte prefix. A copy of aJournalMagic[].
** (2) 4 byte big-endian integer which is the number of valid page records
** in the journal. If this value is 0xffffffff, then compute the
** number of page records from the journal size.
** (3) 4 byte big-endian integer which is the initial value for the
** sanity checksum.
** (4) 4 byte integer which is the number of pages to truncate the
** database to during a rollback.
** (5) 4 byte big-endian integer which is the sector size. The header
** is this many bytes in size.
** (6) 4 byte big-endian integer which is the page size.
** (7) zero padding out to the next sector size.
** (8) Zero or more pages instances, each as follows:
** + 4 byte page number.
** + pPager->pageSize bytes of data.
** + 4 byte checksum
**
** When we speak of the journal header, we mean the first 7 items above.
** Each entry in the journal is an instance of the 8th item.
**
** Call the value from the second bullet "nRec". nRec is the number of
** valid page entries in the journal. In most cases, you can compute the
** value of nRec from the size of the journal file. But if a power
** failure occurred while the journal was being written, it could be the
** case that the size of the journal file had already been increased but
** the extra entries had not yet made it safely to disk. In such a case,
** the value of nRec computed from the file size would be too large. For
** that reason, we always use the nRec value in the header.
**
** If the nRec value is 0xffffffff it means that nRec should be computed
** from the file size. This value is used when the user selects the
** no-sync option for the journal. A power failure could lead to corruption
** in this case. But for things like temporary table (which will be
** deleted when the power is restored) we don't care.
**
** If the file opened as the journal file is not a well-formed
** journal file then all pages up to the first corrupted page are rolled
** back (or no pages if the journal header is corrupted). The journal file
** is then deleted and SQLITE_OK returned, just as if no corruption had
** been encountered.
**
** If an I/O or malloc() error occurs, the journal-file is not deleted
** and an error code is returned.
**
** The isHot parameter indicates that we are trying to rollback a journal
** that might be a hot journal. Or, it could be that the journal is
** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE.
** If the journal really is hot, reset the pager cache prior rolling
** back any content. If the journal is merely persistent, no reset is
** needed.
*/
static int pager_playback(Pager *pPager, int isHot){
sqlite3_vfs *pVfs = pPager->pVfs;
i64 szJ; /* Size of the journal file in bytes */
u32 nRec; /* Number of Records in the journal */
u32 u; /* Unsigned loop counter */
Pgno mxPg = 0; /* Size of the original file in pages */
int rc; /* Result code of a subroutine */
int res = 1; /* Value returned by sqlite3OsAccess() */
char *zMaster = 0; /* Name of master journal file if any */
int needPagerReset; /* True to reset page prior to first page rollback */
int nPlayback = 0; /* Total number of pages restored from journal */
u32 savedPageSize = pPager->pageSize;
/* Figure out how many records are in the journal. Abort early if
** the journal is empty.
*/
assert( isOpen(pPager->jfd) );
rc = sqlite3OsFileSize(pPager->jfd, &szJ);
if( rc!=SQLITE_OK ){
goto end_playback;
}
/* Read the master journal name from the journal, if it is present.
** If a master journal file name is specified, but the file is not
** present on disk, then the journal is not hot and does not need to be
** played back.
**
** TODO: Technically the following is an error because it assumes that
** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that
** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c,
** mxPathname is 512, which is the same as the minimum allowable value
** for pageSize.
*/
zMaster = pPager->pTmpSpace;
rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
if( rc==SQLITE_OK && zMaster[0] ){
rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res);
}
zMaster = 0;
if( rc!=SQLITE_OK || !res ){
goto end_playback;
}
pPager->journalOff = 0;
needPagerReset = isHot;
/* This loop terminates either when a readJournalHdr() or
** pager_playback_one_page() call returns SQLITE_DONE or an IO error
** occurs.
*/
while( 1 ){
/* Read the next journal header from the journal file. If there are
** not enough bytes left in the journal file for a complete header, or
** it is corrupted, then a process must have failed while writing it.
** This indicates nothing more needs to be rolled back.
*/
rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
}
goto end_playback;
}
/* If nRec is 0xffffffff, then this journal was created by a process
** working in no-sync mode. This means that the rest of the journal
** file consists of pages, there are no more journal headers. Compute
** the value of nRec based on this assumption.
*/
if( nRec==0xffffffff ){
assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) );
nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager));
}
/* If nRec is 0 and this rollback is of a transaction created by this
** process and if this is the final header in the journal, then it means
** that this part of the journal was being filled but has not yet been
** synced to disk. Compute the number of pages based on the remaining
** size of the file.
**
** The third term of the test was added to fix ticket #2565.
** When rolling back a hot journal, nRec==0 always means that the next
** chunk of the journal contains zero pages to be rolled back. But
** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in
** the journal, it means that the journal might contain additional
** pages that need to be rolled back and that the number of pages
** should be computed based on the journal file size.
*/
if( nRec==0 && !isHot &&
pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){
nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager));
}
/* If this is the first header read from the journal, truncate the
** database file back to its original size.
*/
if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){
rc = pager_truncate(pPager, mxPg);
if( rc!=SQLITE_OK ){
goto end_playback;
}
pPager->dbSize = mxPg;
}
/* Copy original pages out of the journal and back into the
** database file and/or page cache.
*/
for(u=0; u<nRec; u++){
if( needPagerReset ){
pager_reset(pPager);
needPagerReset = 0;
}
rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0);
if( rc==SQLITE_OK ){
nPlayback++;
}else{
if( rc==SQLITE_DONE ){
pPager->journalOff = szJ;
break;
}else if( rc==SQLITE_IOERR_SHORT_READ ){
/* If the journal has been truncated, simply stop reading and
** processing the journal. This might happen if the journal was
** not completely written and synced prior to a crash. In that
** case, the database should have never been written in the
** first place so it is OK to simply abandon the rollback. */
rc = SQLITE_OK;
goto end_playback;
}else{