| /* |
| ** 2017-10-20 |
| ** |
| ** The author disclaims copyright to this source code. In place of |
| ** a legal notice, here is a blessing: |
| ** |
| ** May you do good and not evil. |
| ** May you find forgiveness for yourself and forgive others. |
| ** May you share freely, never taking more than you give. |
| ** |
| ****************************************************************************** |
| ** |
| ** This file implements a VFS shim that allows an SQLite database to be |
| ** appended onto the end of some other file, such as an executable. |
| ** |
| ** A special record must appear at the end of the file that identifies the |
| ** file as an appended database and provides an offset to page 1. For |
| ** best performance page 1 should be located at a disk page boundary, though |
| ** that is not required. |
| ** |
| ** When opening a database using this VFS, the connection might treat |
| ** the file as an ordinary SQLite database, or it might treat is as a |
| ** database appended onto some other file. Here are the rules: |
| ** |
| ** (1) When opening a new empty file, that file is treated as an ordinary |
| ** database. |
| ** |
| ** (2) When opening a file that begins with the standard SQLite prefix |
| ** string "SQLite format 3", that file is treated as an ordinary |
| ** database. |
| ** |
| ** (3) When opening a file that ends with the appendvfs trailer string |
| ** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended |
| ** database. |
| ** |
| ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is |
| ** set, then a new database is appended to the already existing file. |
| ** |
| ** (5) Otherwise, SQLITE_CANTOPEN is returned. |
| ** |
| ** To avoid unnecessary complications with the PENDING_BYTE, the size of |
| ** the file containing the database is limited to 1GB. This VFS will refuse |
| ** to read or write past the 1GB mark. This restriction might be lifted in |
| ** future versions. For now, if you need a large database, then keep the |
| ** database in a separate file. |
| ** |
| ** If the file being opened is not an appended database, then this shim is |
| ** a pass-through into the default underlying VFS. |
| **/ |
| #include "sqlite3ext.h" |
| SQLITE_EXTENSION_INIT1 |
| #include <string.h> |
| #include <assert.h> |
| |
| /* The append mark at the end of the database is: |
| ** |
| ** Start-Of-SQLite3-NNNNNNNN |
| ** 123456789 123456789 12345 |
| ** |
| ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is |
| ** the offset to page 1. |
| */ |
| #define APND_MARK_PREFIX "Start-Of-SQLite3-" |
| #define APND_MARK_PREFIX_SZ 17 |
| #define APND_MARK_SIZE 25 |
| |
| /* |
| ** Maximum size of the combined prefix + database + append-mark. This |
| ** must be less than 0x40000000 to avoid locking issues on Windows. |
| */ |
| #define APND_MAX_SIZE (65536*15259) |
| |
| /* |
| ** Forward declaration of objects used by this utility |
| */ |
| typedef struct sqlite3_vfs ApndVfs; |
| typedef struct ApndFile ApndFile; |
| |
| /* Access to a lower-level VFS that (might) implement dynamic loading, |
| ** access to randomness, etc. |
| */ |
| #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) |
| #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) |
| |
| /* An open file */ |
| struct ApndFile { |
| sqlite3_file base; /* IO methods */ |
| sqlite3_int64 iPgOne; /* File offset to page 1 */ |
| sqlite3_int64 iMark; /* Start of the append-mark */ |
| }; |
| |
| /* |
| ** Methods for ApndFile |
| */ |
| static int apndClose(sqlite3_file*); |
| static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); |
| static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); |
| static int apndTruncate(sqlite3_file*, sqlite3_int64 size); |
| static int apndSync(sqlite3_file*, int flags); |
| static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); |
| static int apndLock(sqlite3_file*, int); |
| static int apndUnlock(sqlite3_file*, int); |
| static int apndCheckReservedLock(sqlite3_file*, int *pResOut); |
| static int apndFileControl(sqlite3_file*, int op, void *pArg); |
| static int apndSectorSize(sqlite3_file*); |
| static int apndDeviceCharacteristics(sqlite3_file*); |
| static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); |
| static int apndShmLock(sqlite3_file*, int offset, int n, int flags); |
| static void apndShmBarrier(sqlite3_file*); |
| static int apndShmUnmap(sqlite3_file*, int deleteFlag); |
| static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); |
| static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); |
| |
| /* |
| ** Methods for ApndVfs |
| */ |
| static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); |
| static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); |
| static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); |
| static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); |
| static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); |
| static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); |
| static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); |
| static void apndDlClose(sqlite3_vfs*, void*); |
| static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); |
| static int apndSleep(sqlite3_vfs*, int microseconds); |
| static int apndCurrentTime(sqlite3_vfs*, double*); |
| static int apndGetLastError(sqlite3_vfs*, int, char *); |
| static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); |
| static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); |
| static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); |
| static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); |
| |
| static sqlite3_vfs apnd_vfs = { |
| 3, /* iVersion (set when registered) */ |
| 0, /* szOsFile (set when registered) */ |
| 1024, /* mxPathname */ |
| 0, /* pNext */ |
| "apndvfs", /* zName */ |
| 0, /* pAppData (set when registered) */ |
| apndOpen, /* xOpen */ |
| apndDelete, /* xDelete */ |
| apndAccess, /* xAccess */ |
| apndFullPathname, /* xFullPathname */ |
| apndDlOpen, /* xDlOpen */ |
| apndDlError, /* xDlError */ |
| apndDlSym, /* xDlSym */ |
| apndDlClose, /* xDlClose */ |
| apndRandomness, /* xRandomness */ |
| apndSleep, /* xSleep */ |
| apndCurrentTime, /* xCurrentTime */ |
| apndGetLastError, /* xGetLastError */ |
| apndCurrentTimeInt64, /* xCurrentTimeInt64 */ |
| apndSetSystemCall, /* xSetSystemCall */ |
| apndGetSystemCall, /* xGetSystemCall */ |
| apndNextSystemCall /* xNextSystemCall */ |
| }; |
| |
| static const sqlite3_io_methods apnd_io_methods = { |
| 3, /* iVersion */ |
| apndClose, /* xClose */ |
| apndRead, /* xRead */ |
| apndWrite, /* xWrite */ |
| apndTruncate, /* xTruncate */ |
| apndSync, /* xSync */ |
| apndFileSize, /* xFileSize */ |
| apndLock, /* xLock */ |
| apndUnlock, /* xUnlock */ |
| apndCheckReservedLock, /* xCheckReservedLock */ |
| apndFileControl, /* xFileControl */ |
| apndSectorSize, /* xSectorSize */ |
| apndDeviceCharacteristics, /* xDeviceCharacteristics */ |
| apndShmMap, /* xShmMap */ |
| apndShmLock, /* xShmLock */ |
| apndShmBarrier, /* xShmBarrier */ |
| apndShmUnmap, /* xShmUnmap */ |
| apndFetch, /* xFetch */ |
| apndUnfetch /* xUnfetch */ |
| }; |
| |
| |
| |
| /* |
| ** Close an apnd-file. |
| */ |
| static int apndClose(sqlite3_file *pFile){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xClose(pFile); |
| } |
| |
| /* |
| ** Read data from an apnd-file. |
| */ |
| static int apndRead( |
| sqlite3_file *pFile, |
| void *zBuf, |
| int iAmt, |
| sqlite_int64 iOfst |
| ){ |
| ApndFile *p = (ApndFile *)pFile; |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); |
| } |
| |
| /* |
| ** Add the append-mark onto the end of the file. |
| */ |
| static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ |
| int i; |
| unsigned char a[APND_MARK_SIZE]; |
| memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); |
| for(i=0; i<8; i++){ |
| a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; |
| } |
| return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark); |
| } |
| |
| /* |
| ** Write data to an apnd-file. |
| */ |
| static int apndWrite( |
| sqlite3_file *pFile, |
| const void *zBuf, |
| int iAmt, |
| sqlite_int64 iOfst |
| ){ |
| int rc; |
| ApndFile *p = (ApndFile *)pFile; |
| pFile = ORIGFILE(pFile); |
| if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL; |
| rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); |
| if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){ |
| sqlite3_int64 sz = 0; |
| rc = pFile->pMethods->xFileSize(pFile, &sz); |
| if( rc==SQLITE_OK ){ |
| p->iMark = sz - APND_MARK_SIZE; |
| if( iOfst + iAmt + p->iPgOne > p->iMark ){ |
| p->iMark = p->iPgOne + iOfst + iAmt; |
| rc = apndWriteMark(p, pFile); |
| } |
| } |
| } |
| return rc; |
| } |
| |
| /* |
| ** Truncate an apnd-file. |
| */ |
| static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ |
| int rc; |
| ApndFile *p = (ApndFile *)pFile; |
| pFile = ORIGFILE(pFile); |
| rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); |
| if( rc==SQLITE_OK ){ |
| p->iMark = p->iPgOne+size; |
| rc = apndWriteMark(p, pFile); |
| } |
| return rc; |
| } |
| |
| /* |
| ** Sync an apnd-file. |
| */ |
| static int apndSync(sqlite3_file *pFile, int flags){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xSync(pFile, flags); |
| } |
| |
| /* |
| ** Return the current file-size of an apnd-file. |
| */ |
| static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ |
| ApndFile *p = (ApndFile *)pFile; |
| int rc; |
| pFile = ORIGFILE(p); |
| rc = pFile->pMethods->xFileSize(pFile, pSize); |
| if( rc==SQLITE_OK && p->iPgOne ){ |
| *pSize -= p->iPgOne + APND_MARK_SIZE; |
| } |
| return rc; |
| } |
| |
| /* |
| ** Lock an apnd-file. |
| */ |
| static int apndLock(sqlite3_file *pFile, int eLock){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xLock(pFile, eLock); |
| } |
| |
| /* |
| ** Unlock an apnd-file. |
| */ |
| static int apndUnlock(sqlite3_file *pFile, int eLock){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xUnlock(pFile, eLock); |
| } |
| |
| /* |
| ** Check if another file-handle holds a RESERVED lock on an apnd-file. |
| */ |
| static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xCheckReservedLock(pFile, pResOut); |
| } |
| |
| /* |
| ** File control method. For custom operations on an apnd-file. |
| */ |
| static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ |
| ApndFile *p = (ApndFile *)pFile; |
| int rc; |
| pFile = ORIGFILE(pFile); |
| rc = pFile->pMethods->xFileControl(pFile, op, pArg); |
| if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ |
| *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg); |
| } |
| return rc; |
| } |
| |
| /* |
| ** Return the sector-size in bytes for an apnd-file. |
| */ |
| static int apndSectorSize(sqlite3_file *pFile){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xSectorSize(pFile); |
| } |
| |
| /* |
| ** Return the device characteristic flags supported by an apnd-file. |
| */ |
| static int apndDeviceCharacteristics(sqlite3_file *pFile){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xDeviceCharacteristics(pFile); |
| } |
| |
| /* Create a shared memory file mapping */ |
| static int apndShmMap( |
| sqlite3_file *pFile, |
| int iPg, |
| int pgsz, |
| int bExtend, |
| void volatile **pp |
| ){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); |
| } |
| |
| /* Perform locking on a shared-memory segment */ |
| static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xShmLock(pFile,offset,n,flags); |
| } |
| |
| /* Memory barrier operation on shared memory */ |
| static void apndShmBarrier(sqlite3_file *pFile){ |
| pFile = ORIGFILE(pFile); |
| pFile->pMethods->xShmBarrier(pFile); |
| } |
| |
| /* Unmap a shared memory segment */ |
| static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xShmUnmap(pFile,deleteFlag); |
| } |
| |
| /* Fetch a page of a memory-mapped file */ |
| static int apndFetch( |
| sqlite3_file *pFile, |
| sqlite3_int64 iOfst, |
| int iAmt, |
| void **pp |
| ){ |
| ApndFile *p = (ApndFile *)pFile; |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); |
| } |
| |
| /* Release a memory-mapped page */ |
| static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ |
| ApndFile *p = (ApndFile *)pFile; |
| pFile = ORIGFILE(pFile); |
| return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); |
| } |
| |
| /* |
| ** Check to see if the file is an ordinary SQLite database file. |
| */ |
| static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ |
| int rc; |
| char zHdr[16]; |
| static const char aSqliteHdr[] = "SQLite format 3"; |
| if( sz<512 ) return 0; |
| rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0); |
| if( rc ) return 0; |
| return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0; |
| } |
| |
| /* |
| ** Try to read the append-mark off the end of a file. Return the |
| ** start of the appended database if the append-mark is present. If |
| ** there is no append-mark, return -1; |
| */ |
| static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ |
| int rc, i; |
| sqlite3_int64 iMark; |
| unsigned char a[APND_MARK_SIZE]; |
| |
| if( sz<=APND_MARK_SIZE ) return -1; |
| rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); |
| if( rc ) return -1; |
| if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; |
| iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56; |
| for(i=1; i<8; i++){ |
| iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i); |
| } |
| return iMark; |
| } |
| |
| /* |
| ** Open an apnd file handle. |
| */ |
| static int apndOpen( |
| sqlite3_vfs *pVfs, |
| const char *zName, |
| sqlite3_file *pFile, |
| int flags, |
| int *pOutFlags |
| ){ |
| ApndFile *p; |
| sqlite3_file *pSubFile; |
| sqlite3_vfs *pSubVfs; |
| int rc; |
| sqlite3_int64 sz; |
| pSubVfs = ORIGVFS(pVfs); |
| if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ |
| return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); |
| } |
| p = (ApndFile*)pFile; |
| memset(p, 0, sizeof(*p)); |
| pSubFile = ORIGFILE(pFile); |
| p->base.pMethods = &apnd_io_methods; |
| rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); |
| if( rc ) goto apnd_open_done; |
| rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); |
| if( rc ){ |
| pSubFile->pMethods->xClose(pSubFile); |
| goto apnd_open_done; |
| } |
| if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ |
| memmove(pFile, pSubFile, pSubVfs->szOsFile); |
| return SQLITE_OK; |
| } |
| p->iMark = 0; |
| p->iPgOne = apndReadMark(sz, pFile); |
| if( p->iPgOne>0 ){ |
| return SQLITE_OK; |
| } |
| if( (flags & SQLITE_OPEN_CREATE)==0 ){ |
| pSubFile->pMethods->xClose(pSubFile); |
| rc = SQLITE_CANTOPEN; |
| } |
| p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; |
| apnd_open_done: |
| if( rc ) pFile->pMethods = 0; |
| return rc; |
| } |
| |
| /* |
| ** All other VFS methods are pass-thrus. |
| */ |
| static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ |
| return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); |
| } |
| static int apndAccess( |
| sqlite3_vfs *pVfs, |
| const char *zPath, |
| int flags, |
| int *pResOut |
| ){ |
| return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); |
| } |
| static int apndFullPathname( |
| sqlite3_vfs *pVfs, |
| const char *zPath, |
| int nOut, |
| char *zOut |
| ){ |
| return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); |
| } |
| static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ |
| return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); |
| } |
| static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ |
| ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); |
| } |
| static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ |
| return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); |
| } |
| static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ |
| ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); |
| } |
| static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ |
| return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); |
| } |
| static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ |
| return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); |
| } |
| static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ |
| return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); |
| } |
| static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ |
| return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); |
| } |
| static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ |
| return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); |
| } |
| static int apndSetSystemCall( |
| sqlite3_vfs *pVfs, |
| const char *zName, |
| sqlite3_syscall_ptr pCall |
| ){ |
| return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); |
| } |
| static sqlite3_syscall_ptr apndGetSystemCall( |
| sqlite3_vfs *pVfs, |
| const char *zName |
| ){ |
| return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); |
| } |
| static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ |
| return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); |
| } |
| |
| |
| #ifdef _WIN32 |
| __declspec(dllexport) |
| #endif |
| /* |
| ** This routine is called when the extension is loaded. |
| ** Register the new VFS. |
| */ |
| int sqlite3_appendvfs_init( |
| sqlite3 *db, |
| char **pzErrMsg, |
| const sqlite3_api_routines *pApi |
| ){ |
| int rc = SQLITE_OK; |
| sqlite3_vfs *pOrig; |
| SQLITE_EXTENSION_INIT2(pApi); |
| (void)pzErrMsg; |
| (void)db; |
| pOrig = sqlite3_vfs_find(0); |
| apnd_vfs.iVersion = pOrig->iVersion; |
| apnd_vfs.pAppData = pOrig; |
| apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); |
| rc = sqlite3_vfs_register(&apnd_vfs, 0); |
| #ifdef APPENDVFS_TEST |
| if( rc==SQLITE_OK ){ |
| rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); |
| } |
| #endif |
| if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; |
| return rc; |
| } |