/*

** 2014 August 30

**

** The author disclaims copyright to this source code. In place of

** a legal notice, here is a blessing:

**

** May you do good and not evil.

** May you find forgiveness for yourself and forgive others.

** May you share freely, never taking more than you give.

**

*************************************************************************

**

**

** OVERVIEW

**

** The RBU extension requires that the RBU update be packaged as an

** SQLite database. The tables it expects to find are described in

** sqlite3rbu.h. Essentially, for each table xyz in the target database

** that the user wishes to write to, a corresponding data_xyz table is

** created in the RBU database and populated with one row for each row to

** update, insert or delete from the target table.

**

** The update proceeds in three stages:

**

** 1) The database is updated. The modified database pages are written

** to a *-oal file. A *-oal file is just like a *-wal file, except

** that it is named "<database>-oal" instead of "<database>-wal".

** Because regular SQLite clients do not look for file named

** "<database>-oal", they go on using the original database in

** rollback mode while the *-oal file is being generated.

**

** During this stage RBU does not update the database by writing

** directly to the target tables. Instead it creates "imposter"

** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses

** to update each b-tree individually. All updates required by each

** b-tree are completed before moving on to the next, and all

** updates are done in sorted key order.

**

** 2) The "<database>-oal" file is moved to the equivalent "<database>-wal"

** location using a call to rename(2). Before doing this the RBU

** module takes an EXCLUSIVE lock on the database file, ensuring

** that there are no other active readers.

**

** Once the EXCLUSIVE lock is released, any other database readers

** detect the new *-wal file and read the database in wal mode. At

** this point they see the new version of the database - including

** the updates made as part of the RBU update.

**

** 3) The new *-wal file is checkpointed. This proceeds in the same way

** as a regular database checkpoint, except that a single frame is

** checkpointed each time sqlite3rbu_step() is called. If the RBU

** handle is closed before the entire *-wal file is checkpointed,

** the checkpoint progress is saved in the RBU database and the

** checkpoint can be resumed by another RBU client at some point in

** the future.

**

** POTENTIAL PROBLEMS

**

** The rename() call might not be portable. And RBU is not currently

** syncing the directory after renaming the file.

**

** When state is saved, any commit to the *-oal file and the commit to

** the RBU update database are not atomic. So if the power fails at the

** wrong moment they might get out of sync. As the main database will be

** committed before the RBU update database this will likely either just

** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE

** constraint violations).

**

** If some client does modify the target database mid RBU update, or some

** other error occurs, the RBU extension will keep throwing errors. It's

** not really clear how to get out of this state. The system could just

** by delete the RBU update database and *-oal file and have the device

** download the update again and start over.

**

** At present, for an UPDATE, both the new.* and old.* records are

** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all

** fields are collected. This means we're probably writing a lot more

** data to disk when saving the state of an ongoing update to the RBU

** update database than is strictly necessary.

**

*/

#include <assert.h>

#include <string.h>

#include <stdio.h>

#include "sqlite3.h"

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)

#include "sqlite3rbu.h"

#if defined(_WIN32_WCE)

#include "windows.h"

#endif

/* Maximum number of prepared UPDATE statements held by this module */

#define SQLITE_RBU_UPDATE_CACHESIZE 16

/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM

** to enable checksum verification.

*/

#ifndef RBU_ENABLE_DELTA_CKSUM

# define RBU_ENABLE_DELTA_CKSUM 0

#endif

/*

** Swap two objects of type TYPE.

*/

#if !defined(SQLITE_AMALGAMATION)

# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}

#endif

/*

** Name of the URI option that causes RBU to take an exclusive lock as

** part of the incremental checkpoint operation.

*/

#define RBU_EXCLUSIVE_CHECKPOINT "rbu_exclusive_checkpoint"

/*

** The rbu_state table is used to save the state of a partially applied

** update so that it can be resumed later. The table consists of integer

** keys mapped to values as follows:

**

** RBU_STATE_STAGE:

** May be set to integer values 1, 2, 4 or 5. As follows:

** 1: the *-rbu file is currently under construction.

** 2: the *-rbu file has been constructed, but not yet moved

** to the *-wal path.

** 4: the checkpoint is underway.

** 5: the rbu update has been checkpointed.

**

** RBU_STATE_TBL:

** Only valid if STAGE==1. The target database name of the table

** currently being written.

**

** RBU_STATE_IDX:

** Only valid if STAGE==1. The target database name of the index

** currently being written, or NULL if the main table is currently being

** updated.

**

** RBU_STATE_ROW:

** Only valid if STAGE==1. Number of rows already processed for the current

** table/index.

**

** RBU_STATE_PROGRESS:

** Trbul number of sqlite3rbu_step() calls made so far as part of this

** rbu update.

**

** RBU_STATE_CKPT:

** Valid if STAGE==4. The 64-bit checksum associated with the wal-index

** header created by recovering the *-wal file. This is used to detect

** cases when another client appends frames to the *-wal file in the

** middle of an incremental checkpoint (an incremental checkpoint cannot

** be continued if this happens).

**

** RBU_STATE_COOKIE:

** Valid if STAGE==1. The current change-counter cookie value in the

** target db file.

**

** RBU_STATE_OALSZ:

** Valid if STAGE==1. The size in bytes of the *-oal file.

**

** RBU_STATE_DATATBL:

** Only valid if STAGE==1. The RBU database name of the table

** currently being read.

*/

#define RBU_STATE_STAGE 1

#define RBU_STATE_TBL 2

#define RBU_STATE_IDX 3

#define RBU_STATE_ROW 4

#define RBU_STATE_PROGRESS 5

#define RBU_STATE_CKPT 6

#define RBU_STATE_COOKIE 7

#define RBU_STATE_OALSZ 8

#define RBU_STATE_PHASEONESTEP 9

#define RBU_STATE_DATATBL 10

#define RBU_STAGE_OAL 1

#define RBU_STAGE_MOVE 2

#define RBU_STAGE_CAPTURE 3

#define RBU_STAGE_CKPT 4

#define RBU_STAGE_DONE 5

#define RBU_CREATE_STATE \

"CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)"

typedef struct RbuFrame RbuFrame;

typedef struct RbuObjIter RbuObjIter;

typedef struct RbuState RbuState;

typedef struct RbuSpan RbuSpan;

typedef struct rbu_vfs rbu_vfs;

typedef struct rbu_file rbu_file;

typedef struct RbuUpdateStmt RbuUpdateStmt;

#if !defined(SQLITE_AMALGAMATION)

typedef unsigned int u32;

typedef unsigned short u16;

typedef unsigned char u8;

typedef sqlite3_int64 i64;

#endif

/*

** These values must match the values defined in wal.c for the equivalent

** locks. These are not magic numbers as they are part of the SQLite file

** format.

*/

#define WAL_LOCK_WRITE 0

#define WAL_LOCK_CKPT 1

#define WAL_LOCK_READ0 3

#define SQLITE_FCNTL_RBUCNT 5149216

/*

** A structure to store values read from the rbu_state table in memory.

*/

struct RbuState {

int eStage;

char *zTbl;

char *zDataTbl;

char *zIdx;

i64 iWalCksum;

int nRow;

i64 nProgress;

u32 iCookie;

i64 iOalSz;

i64 nPhaseOneStep;

};

struct RbuUpdateStmt {

char *zMask; /* Copy of update mask used with pUpdate */

sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */

RbuUpdateStmt *pNext;

};

struct RbuSpan {

const char *zSpan;

int nSpan;

};

/*

** An iterator of this type is used to iterate through all objects in

** the target database that require updating. For each such table, the

** iterator visits, in order:

**

** * the table itself,

** * each index of the table (zero or more points to visit), and

** * a special "cleanup table" state.

**

** abIndexed:

** If the table has no indexes on it, abIndexed is set to NULL. Otherwise,

** it points to an array of flags nTblCol elements in size. The flag is

** set for each column that is either a part of the PK or a part of an

** index. Or clear otherwise.

**

** If there are one or more partial indexes on the table, all fields of

** this array set set to 1. This is because in that case, the module has

** no way to tell which fields will be required to add and remove entries

** from the partial indexes.

**

*/

struct RbuObjIter {

sqlite3_stmt *pTblIter; /* Iterate through tables */

sqlite3_stmt *pIdxIter; /* Index iterator */

int nTblCol; /* Size of azTblCol[] array */

char **azTblCol; /* Array of unquoted target column names */

char **azTblType; /* Array of target column types */

int *aiSrcOrder; /* src table col -> target table col */

u8 *abTblPk; /* Array of flags, set on target PK columns */

u8 *abNotNull; /* Array of flags, set on NOT NULL columns */

u8 *abIndexed; /* Array of flags, set on indexed & PK cols */

int eType; /* Table type - an RBU_PK_XXX value */

/* Output variables. zTbl==0 implies EOF. */

int bCleanup; /* True in "cleanup" state */

const char *zTbl; /* Name of target db table */

const char *zDataTbl; /* Name of rbu db table (or null) */

const char *zIdx; /* Name of target db index (or null) */

int iTnum; /* Root page of current object */

int iPkTnum; /* If eType==EXTERNAL, root of PK index */

int bUnique; /* Current index is unique */

int nIndex; /* Number of aux. indexes on table zTbl */

/* Statements created by rbuObjIterPrepareAll() */

int nCol; /* Number of columns in current object */

sqlite3_stmt *pSelect; /* Source data */

sqlite3_stmt *pInsert; /* Statement for INSERT operations */

sqlite3_stmt *pDelete; /* Statement for DELETE ops */

sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */

int nIdxCol;

RbuSpan *aIdxCol;

char *zIdxSql;

/* Last UPDATE used (for PK b-tree updates only), or NULL. */

RbuUpdateStmt *pRbuUpdate;

};

/*

** Values for RbuObjIter.eType

**

** 0: Table does not exist (error)

** 1: Table has an implicit rowid.

** 2: Table has an explicit IPK column.

** 3: Table has an external PK index.

** 4: Table is WITHOUT ROWID.

** 5: Table is a virtual table.

*/

#define RBU_PK_NOTABLE 0

#define RBU_PK_NONE 1

#define RBU_PK_IPK 2

#define RBU_PK_EXTERNAL 3

#define RBU_PK_WITHOUT_ROWID 4

#define RBU_PK_VTAB 5

/*

** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs

** one of the following operations.

*/

#define RBU_INSERT 1 /* Insert on a main table b-tree */

#define RBU_DELETE 2 /* Delete a row from a main table b-tree */

#define RBU_REPLACE 3 /* Delete and then insert a row */

#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */

#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */

#define RBU_UPDATE 6 /* Update a row in a main table b-tree */

/*

** A single step of an incremental checkpoint - frame iWalFrame of the wal

** file should be copied to page iDbPage of the database file.

*/

struct RbuFrame {

u32 iDbPage;

u32 iWalFrame;

};

/*

** RBU handle.

**

** nPhaseOneStep:

** If the RBU database contains an rbu_count table, this value is set to

** a running estimate of the number of b-tree operations required to

** finish populating the *-oal file. This allows the sqlite3_bp_progress()

** API to calculate the permyriadage progress of populating the *-oal file

** using the formula:

**

** permyriadage = (10000 * nProgress) / nPhaseOneStep

**

** nPhaseOneStep is initialized to the sum of:

**

** nRow * (nIndex + 1)

**

** for all source tables in the RBU database, where nRow is the number

** of rows in the source table and nIndex the number of indexes on the

** corresponding target database table.

**

** This estimate is accurate if the RBU update consists entirely of

** INSERT operations. However, it is inaccurate if:

**

** * the RBU update contains any UPDATE operations. If the PK specified

** for an UPDATE operation does not exist in the target table, then

** no b-tree operations are required on index b-trees. Or if the

** specified PK does exist, then (nIndex*2) such operations are

** required (one delete and one insert on each index b-tree).

**

** * the RBU update contains any DELETE operations for which the specified

** PK does not exist. In this case no operations are required on index

** b-trees.

**

** * the RBU update contains REPLACE operations. These are similar to

** UPDATE operations.

**

** nPhaseOneStep is updated to account for the conditions above during the

** first pass of each source table. The updated nPhaseOneStep value is

** stored in the rbu_state table if the RBU update is suspended.

*/

struct sqlite3rbu {

int eStage; /* Value of RBU_STATE_STAGE field */

sqlite3 *dbMain; /* target database handle */

sqlite3 *dbRbu; /* rbu database handle */

char *zTarget; /* Path to target db */

char *zRbu; /* Path to rbu db */

char *zState; /* Path to state db (or NULL if zRbu) */

char zStateDb[5]; /* Db name for state ("stat" or "main") */

int rc; /* Value returned by last rbu_step() call */

char *zErrmsg; /* Error message if rc!=SQLITE_OK */

int nStep; /* Rows processed for current object */

int nProgress; /* Rows processed for all objects */

RbuObjIter objiter; /* Iterator for skipping through tbl/idx */

const char *zVfsName; /* Name of automatically created rbu vfs */

rbu_file *pTargetFd; /* File handle open on target db */

int nPagePerSector; /* Pages per sector for pTargetFd */

i64 iOalSz;

i64 nPhaseOneStep;

/* The following state variables are used as part of the incremental

** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding

** function rbuSetupCheckpoint() for details. */

u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */

u32 mLock;

int nFrame; /* Entries in aFrame[] array */

int nFrameAlloc; /* Allocated size of aFrame[] array */

RbuFrame *aFrame;

int pgsz;

u8 *aBuf;

i64 iWalCksum;

i64 szTemp; /* Current size of all temp files in use */

i64 szTempLimit; /* Total size limit for temp files */

/* Used in RBU vacuum mode only */

int nRbu; /* Number of RBU VFS in the stack */

rbu_file *pRbuFd; /* Fd for main db of dbRbu */

};

/*

** An rbu VFS is implemented using an instance of this structure.

**

** Variable pRbu is only non-NULL for automatically created RBU VFS objects.

** It is NULL for RBU VFS objects created explicitly using

** sqlite3rbu_create_vfs(). It is used to track the total amount of temp

** space used by the RBU handle.

*/

struct rbu_vfs {

sqlite3_vfs base; /* rbu VFS shim methods */

sqlite3_vfs *pRealVfs; /* Underlying VFS */

sqlite3_mutex *mutex; /* Mutex to protect pMain */

sqlite3rbu *pRbu; /* Owner RBU object */

rbu_file *pMain; /* List of main db files */

rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */

};

/*

** Each file opened by an rbu VFS is represented by an instance of

** the following structure.

**

** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable

** "sz" is set to the current size of the database file.

*/

struct rbu_file {

sqlite3_file base; /* sqlite3_file methods */

sqlite3_file *pReal; /* Underlying file handle */

rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */

sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */

i64 sz; /* Size of file in bytes (temp only) */

int openFlags; /* Flags this file was opened with */

u32 iCookie; /* Cookie value for main db files */

u8 iWriteVer; /* "write-version" value for main db files */

u8 bNolock; /* True to fail EXCLUSIVE locks */

int nShm; /* Number of entries in apShm[] array */

char **apShm; /* Array of mmap'd *-shm regions */

char *zDel; /* Delete this when closing file */

const char *zWal; /* Wal filename for this main db file */

rbu_file *pWalFd; /* Wal file descriptor for this main db */

rbu_file *pMainNext; /* Next MAIN_DB file */

rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */

};

/*

** True for an RBU vacuum handle, or false otherwise.

*/

#define rbuIsVacuum(p) ((p)->zTarget==0)

/*************************************************************************

** The following three functions, found below:

**

** rbuDeltaGetInt()

** rbuDeltaChecksum()

** rbuDeltaApply()

**

** are lifted from the fossil source code (http://fossil-scm.org). They

** are used to implement the scalar SQL function rbu_fossil_delta().

*/

/*

** Read bytes from *pz and convert them into a positive integer. When

** finished, leave *pz pointing to the first character past the end of

** the integer. The *pLen parameter holds the length of the string

** in *pz and is decremented once for each character in the integer.

*/

static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){

static const signed char zValue[] = {

-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,

-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,

-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,

0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,

-1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,

25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36,

-1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,

52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1,

};

unsigned int v = 0;

int c;

unsigned char *z = (unsigned char*)*pz;

unsigned char *zStart = z;

while( (c = zValue[0x7f&*(z++)])>=0 ){

v = (v<<6) + c;

}

z--;

*pLen -= z - zStart;

*pz = (char*)z;

return v;

}

#if RBU_ENABLE_DELTA_CKSUM

/*

** Compute a 32-bit checksum on the N-byte buffer. Return the result.

*/

static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){

const unsigned char *z = (const unsigned char *)zIn;

unsigned sum0 = 0;

unsigned sum1 = 0;

unsigned sum2 = 0;

unsigned sum3 = 0;

while(N >= 16){

sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);

sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);

sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);

sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);

z += 16;

N -= 16;

}

while(N >= 4){

sum0 += z[0];

sum1 += z[1];

sum2 += z[2];

sum3 += z[3];

z += 4;

N -= 4;

}

sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);

switch(N){

case 3: sum3 += (z[2] << 8);

case 2: sum3 += (z[1] << 16);

case 1: sum3 += (z[0] << 24);

default: ;

}

return sum3;

}

#endif

/*

** Apply a delta.

**

** The output buffer should be big enough to hold the whole output

** file and a NUL terminator at the end. The delta_output_size()

** routine will determine this size for you.

**

** The delta string should be null-terminated. But the delta string

** may contain embedded NUL characters (if the input and output are

** binary files) so we also have to pass in the length of the delta in

** the lenDelta parameter.

**

** This function returns the size of the output file in bytes (excluding

** the final NUL terminator character). Except, if the delta string is

** malformed or intended for use with a source file other than zSrc,

** then this routine returns -1.

**

** Refer to the delta_create() documentation above for a description

** of the delta file format.

*/

static int rbuDeltaApply(

const char *zSrc, /* The source or pattern file */

int lenSrc, /* Length of the source file */

const char *zDelta, /* Delta to apply to the pattern */

int lenDelta, /* Length of the delta */

char *zOut /* Write the output into this preallocated buffer */

){

unsigned int limit;

unsigned int total = 0;

#if RBU_ENABLE_DELTA_CKSUM

char *zOrigOut = zOut;

#endif

limit = rbuDeltaGetInt(&zDelta, &lenDelta);

if( *zDelta!='\n' ){

/* ERROR: size integer not terminated by "\n" */

return -1;

}

zDelta++; lenDelta--;

while( *zDelta && lenDelta>0 ){

unsigned int cnt, ofst;

cnt = rbuDeltaGetInt(&zDelta, &lenDelta);

switch( zDelta[0] ){

case '@': {

zDelta++; lenDelta--;

ofst = rbuDeltaGetInt(&zDelta, &lenDelta);

if( lenDelta>0 && zDelta[0]!=',' ){

/* ERROR: copy command not terminated by ',' */

return -1;

}

zDelta++; lenDelta--;

total += cnt;

if( total>limit ){

/* ERROR: copy exceeds output file size */

return -1;

}

if( (int)(ofst+cnt) > lenSrc ){

/* ERROR: copy extends past end of input */

return -1;

}

memcpy(zOut, &zSrc[ofst], cnt);

zOut += cnt;

break;

}

case ':': {

zDelta++; lenDelta--;

total += cnt;

if( total>limit ){

/* ERROR: insert command gives an output larger than predicted */

return -1;

}

if( (int)cnt>lenDelta ){

/* ERROR: insert count exceeds size of delta */

return -1;

}

memcpy(zOut, zDelta, cnt);

zOut += cnt;

zDelta += cnt;

lenDelta -= cnt;

break;

}

case ';': {

zDelta++; lenDelta--;

zOut[0] = 0;

#if RBU_ENABLE_DELTA_CKSUM

if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){

/* ERROR: bad checksum */

return -1;

}

#endif

if( total!=limit ){

/* ERROR: generated size does not match predicted size */

return -1;

}

return total;

}

default: {

/* ERROR: unknown delta operator */

return -1;

}

}

}

/* ERROR: unterminated delta */

return -1;

}

static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){

int size;

size = rbuDeltaGetInt(&zDelta, &lenDelta);

if( *zDelta!='\n' ){

/* ERROR: size integer not terminated by "\n" */

return -1;

}

return size;

}

/*

** End of code taken from fossil.

*************************************************************************/

/*

** Implementation of SQL scalar function rbu_fossil_delta().

**

** This function applies a fossil delta patch to a blob. Exactly two

** arguments must be passed to this function. The first is the blob to

** patch and the second the patch to apply. If no error occurs, this

** function returns the patched blob.

*/

static void rbuFossilDeltaFunc(

sqlite3_context *context,

int argc,

sqlite3_value **argv

){

const char *aDelta;

int nDelta;

const char *aOrig;

int nOrig;

int nOut;

int nOut2;

char *aOut;

assert( argc==2 );

nOrig = sqlite3_value_bytes(argv[0]);

aOrig = (const char*)sqlite3_value_blob(argv[0]);

nDelta = sqlite3_value_bytes(argv[1]);

aDelta = (const char*)sqlite3_value_blob(argv[1]);

/* Figure out the size of the output */

nOut = rbuDeltaOutputSize(aDelta, nDelta);

if( nOut<0 ){

sqlite3_result_error(context, "corrupt fossil delta", -1);

return;

}

aOut = sqlite3_malloc(nOut+1);

if( aOut==0 ){

sqlite3_result_error_nomem(context);

}else{

nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);

if( nOut2!=nOut ){

sqlite3_free(aOut);

sqlite3_result_error(context, "corrupt fossil delta", -1);

}else{

sqlite3_result_blob(context, aOut, nOut, sqlite3_free);

}

}

}

/*

** Prepare the SQL statement in buffer zSql against database handle db.

** If successful, set *ppStmt to point to the new statement and return

** SQLITE_OK.

**

** Otherwise, if an error does occur, set *ppStmt to NULL and return

** an SQLite error code. Additionally, set output variable *pzErrmsg to

** point to a buffer containing an error message. It is the responsibility

** of the caller to (eventually) free this buffer using sqlite3_free().

*/

static int prepareAndCollectError(

sqlite3 *db,

sqlite3_stmt **ppStmt,

char **pzErrmsg,

const char *zSql

){

int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);

if( rc!=SQLITE_OK ){

*pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));

*ppStmt = 0;

}

return rc;

}

/*

** Reset the SQL statement passed as the first argument. Return a copy

** of the value returned by sqlite3_reset().

**

** If an error has occurred, then set *pzErrmsg to point to a buffer

** containing an error message. It is the responsibility of the caller

** to eventually free this buffer using sqlite3_free().

*/

static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){

int rc = sqlite3_reset(pStmt);

if( rc!=SQLITE_OK ){

*pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt)));

}

return rc;

}

/*

** Unless it is NULL, argument zSql points to a buffer allocated using

** sqlite3_malloc containing an SQL statement. This function prepares the SQL

** statement against database db and frees the buffer. If statement

** compilation is successful, *ppStmt is set to point to the new statement

** handle and SQLITE_OK is returned.

**

** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code

** returned. In this case, *pzErrmsg may also be set to point to an error

** message. It is the responsibility of the caller to free this error message

** buffer using sqlite3_free().

**

** If argument zSql is NULL, this function assumes that an OOM has occurred.

** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.

*/

static int prepareFreeAndCollectError(

sqlite3 *db,

sqlite3_stmt **ppStmt,

char **pzErrmsg,

char *zSql

){

int rc;

assert( *pzErrmsg==0 );

if( zSql==0 ){

rc = SQLITE_NOMEM;

*ppStmt = 0;

}else{

rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql);

sqlite3_free(zSql);

}

return rc;

}

/*

** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated

** by an earlier call to rbuObjIterCacheTableInfo().

*/

static void rbuObjIterFreeCols(RbuObjIter *pIter){

int i;

for(i=0; i<pIter->nTblCol; i++){

sqlite3_free(pIter->azTblCol[i]);

sqlite3_free(pIter->azTblType[i]);

}

sqlite3_free(pIter->azTblCol);

pIter->azTblCol = 0;

pIter->azTblType = 0;

pIter->aiSrcOrder = 0;

pIter->abTblPk = 0;

pIter->abNotNull = 0;

pIter->nTblCol = 0;

pIter->eType = 0; /* Invalid value */

}

/*

** Finalize all statements and free all allocations that are specific to

** the current object (table/index pair).

*/

static void rbuObjIterClearStatements(RbuObjIter *pIter){

RbuUpdateStmt *pUp;

sqlite3_finalize(pIter->pSelect);

sqlite3_finalize(pIter->pInsert);

sqlite3_finalize(pIter->pDelete);

sqlite3_finalize(pIter->pTmpInsert);

pUp = pIter->pRbuUpdate;

while( pUp ){

RbuUpdateStmt *pTmp = pUp->pNext;

sqlite3_finalize(pUp->pUpdate);

sqlite3_free(pUp);

pUp = pTmp;

}

sqlite3_free(pIter->aIdxCol);

sqlite3_free(pIter->zIdxSql);

pIter->pSelect = 0;

pIter->pInsert = 0;

pIter->pDelete = 0;

pIter->pRbuUpdate = 0;

pIter->pTmpInsert = 0;

pIter->nCol = 0;

pIter->nIdxCol = 0;

pIter->aIdxCol = 0;

pIter->zIdxSql = 0;

}

/*

** Clean up any resources allocated as part of the iterator object passed

** as the only argument.

*/

static void rbuObjIterFinalize(RbuObjIter *pIter){

rbuObjIterClearStatements(pIter);

sqlite3_finalize(pIter->pTblIter);

sqlite3_finalize(pIter->pIdxIter);

rbuObjIterFreeCols(pIter);

memset(pIter, 0, sizeof(RbuObjIter));

}

/*

** Advance the iterator to the next position.

**

** If no error occurs, SQLITE_OK is returned and the iterator is left

** pointing to the next entry. Otherwise, an error code and message is

** left in the RBU handle passed as the first argument. A copy of the

** error code is returned.

*/

static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){

int rc = p->rc;

if( rc==SQLITE_OK ){

/* Free any SQLite statements used while processing the previous object */

rbuObjIterClearStatements(pIter);

if( pIter->zIdx==0 ){

rc = sqlite3_exec(p->dbMain,

"DROP TRIGGER IF EXISTS temp.rbu_insert_tr;"

"DROP TRIGGER IF EXISTS temp.rbu_update1_tr;"

"DROP TRIGGER IF EXISTS temp.rbu_update2_tr;"

"DROP TRIGGER IF EXISTS temp.rbu_delete_tr;"

, 0, 0, &p->zErrmsg

);

}

if( rc==SQLITE_OK ){

if( pIter->bCleanup ){

rbuObjIterFreeCols(pIter);

pIter->bCleanup = 0;

rc = sqlite3_step(pIter->pTblIter);

if( rc!=SQLITE_ROW ){

rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg);

pIter->zTbl = 0;

}else{

pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0);

pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1);

rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM;

}

}else{

if( pIter->zIdx==0 ){

sqlite3_stmt *pIdx = pIter->pIdxIter;

rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC);

}

if( rc==SQLITE_OK ){

rc = sqlite3_step(pIter->pIdxIter);

if( rc!=SQLITE_ROW ){

rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg);

pIter->bCleanup = 1;

pIter->zIdx = 0;

}else{

pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);

pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1);

pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2);

rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM;

}

}

}

}

}

if( rc!=SQLITE_OK ){

rbuObjIterFinalize(pIter);

p->rc = rc;

}

return rc;

}

/*

** The implementation of the rbu_target_name() SQL function. This function

** accepts one or two arguments. The first argument is the name of a table -

** the name of a table in the RBU database. The second, if it is present, is 1

** for a view or 0 for a table.

**

** For a non-vacuum RBU handle, if the table name matches the pattern:

**

** data[0-9]_<name>

**

** where <name> is any sequence of 1 or more characters, <name> is returned.

** Otherwise, if the only argument does not match the above pattern, an SQL

** NULL is returned.

**

** "data_t1" -> "t1"

** "data0123_t2" -> "t2"

** "dataAB_t3" -> NULL

**

** For an rbu vacuum handle, a copy of the first argument is returned if

** the second argument is either missing or 0 (not a view).

*/

static void rbuTargetNameFunc(

sqlite3_context *pCtx,

int argc,

sqlite3_value **argv

){

sqlite3rbu *p = sqlite3_user_data(pCtx);

const char *zIn;

assert( argc==1 || argc==2 );

zIn = (const char*)sqlite3_value_text(argv[0]);

if( zIn ){

if( rbuIsVacuum(p) ){

assert( argc==2 || argc==1 );

if( argc==1 || 0==sqlite3_value_int(argv[1]) ){

sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);

}

}else{

if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){

int i;

for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);

if( zIn[i]=='_' && zIn[i+1] ){

sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC);

}

}

}

}

}

/*

** Initialize the iterator structure passed as the second argument.

**

** If no error occurs, SQLITE_OK is returned and the iterator is left

** pointing to the first entry. Otherwise, an error code and message is

** left in the RBU handle passed as the first argument. A copy of the

** error code is returned.

*/

static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){

int rc;

memset(pIter, 0, sizeof(RbuObjIter));

rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg,

sqlite3_mprintf(

"SELECT rbu_target_name(name, type='view') AS target, name "

"FROM sqlite_schema "

"WHERE type IN ('table', 'view') AND target IS NOT NULL "

" %s "

"ORDER BY name"

, rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));

if( rc==SQLITE_OK ){

rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,

"SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "

" FROM main.sqlite_schema "

" WHERE type='index' AND tbl_name = ?"

);

}

pIter->bCleanup = 1;

p->rc = rc;