In LEMON, fix a bug in the text formatter introduced by the previous
commit. Also add the new "%token_class" directive for defining symbolic
names that stand any one of a collection of tokens.
FossilOrigin-Name: da7890ca6b1d8e511377a469047120220e8c3b2d
diff --git a/manifest b/manifest
index edfa17a..b56fe09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\suse\ssprintf(),\sstrcpy()\sor\sstrcat()\sin\sthe\simplementation\sof\sthe\nlemon\sparser\sgenerator\stool,\sto\savoid\scompiler\swarnings\sin\sOpenBSD.
-D 2014-01-10T23:21:00.243
+C In\sLEMON,\sfix\sa\sbug\sin\sthe\stext\sformatter\sintroduced\sby\sthe\sprevious\ncommit.\s\sAlso\sadd\sthe\snew\s"%token_class"\sdirective\sfor\sdefining\ssymbolic\nnames\sthat\sstand\sany\sone\sof\sa\scollection\sof\stokens.
+D 2014-01-11T03:06:18.172
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -1109,7 +1109,7 @@
F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
-F tool/lemon.c 04f60c891f6c07643221b068bbc2621be46c039a
+F tool/lemon.c 4a3d4a579c5dff6a42785e97d1f2b59789f3b8dd
F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75
F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
@@ -1148,7 +1148,7 @@
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 7f1e7ae313c7625ef2623d78883dce776eecca30
-R 153a0201d8a4a180b2a188d3abd169a6
+P e43c522dde01e134f1adc94f534d2b3eda74afc2
+R 498b2eca2a4d9bdf3c22bba6f4386191
U drh
-Z c49dfd6bd62bb08d757d02ffce1c49c4
+Z 303f39885598d1865b7eb401105ea9ea
diff --git a/manifest.uuid b/manifest.uuid
index fdbe371..2d1234c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e43c522dde01e134f1adc94f534d2b3eda74afc2
\ No newline at end of file
+da7890ca6b1d8e511377a469047120220e8c3b2d
\ No newline at end of file
diff --git a/tool/lemon.c b/tool/lemon.c
index 191ab0d..25787ab 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -71,12 +71,15 @@
char *zBuf, /* The buffer to which text is added */
int *pnUsed, /* Slots of the buffer used so far */
const char *zIn, /* Text to add */
- int nIn /* Bytes of text to add. -1 to use strlen() */
+ int nIn, /* Bytes of text to add. -1 to use strlen() */
+ int iWidth /* Field width. Negative to left justify */
){
if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){}
+ while( iWidth>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth--; }
if( nIn==0 ) return;
memcpy(&zBuf[*pnUsed], zIn, nIn);
*pnUsed += nIn;
+ while( (-iWidth)>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth++; }
zBuf[*pnUsed] = 0;
}
static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
@@ -87,15 +90,22 @@
str[0] = 0;
for(i=j=0; (c = zFormat[i])!=0; i++){
if( c=='%' ){
- lemon_addtext(str, &nUsed, &zFormat[j], i-j);
+ int iWidth = 0;
+ lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
c = zFormat[++i];
+ if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){
+ if( c=='-' ) i++;
+ while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0';
+ if( c=='-' ) iWidth = -iWidth;
+ c = zFormat[i];
+ }
if( c=='d' ){
int v = va_arg(ap, int);
if( v<0 ){
- lemon_addtext(str, &nUsed, "-", 1);
+ lemon_addtext(str, &nUsed, "-", 1, iWidth);
v = -v;
}else if( v==0 ){
- lemon_addtext(str, &nUsed, "0", 1);
+ lemon_addtext(str, &nUsed, "0", 1, iWidth);
}
k = 0;
while( v>0 ){
@@ -103,17 +113,17 @@
zTemp[sizeof(zTemp)-k] = (v%10) + '0';
v /= 10;
}
- lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k);
+ lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth);
}else if( c=='s' ){
z = va_arg(ap, const char*);
- lemon_addtext(str, &nUsed, z, -1);
+ lemon_addtext(str, &nUsed, z, -1, iWidth);
}else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){
i += 2;
k = va_arg(ap, int);
z = va_arg(ap, const char*);
- lemon_addtext(str, &nUsed, z, k);
+ lemon_addtext(str, &nUsed, z, k, iWidth);
}else if( c=='%' ){
- lemon_addtext(str, &nUsed, "%", 1);
+ lemon_addtext(str, &nUsed, "%", 1, 0);
}else{
fprintf(stderr, "illegal format\n");
exit(1);
@@ -121,7 +131,7 @@
j = i+1;
}
}
- lemon_addtext(str, &nUsed, &zFormat[j], i-j);
+ lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
return nUsed;
}
static int lemon_sprintf(char *str, const char *format, ...){
@@ -1538,12 +1548,15 @@
}
/* Count and index the symbols of the grammar */
- lem.nsymbol = Symbol_count();
Symbol_new("{default}");
+ lem.nsymbol = Symbol_count();
lem.symbols = Symbol_arrayof();
- for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
- qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp);
- for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
+ for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
+ qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp);
+ for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
+ while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; }
+ assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 );
+ lem.nsymbol = i - 1;
for(i=1; isupper(lem.symbols[i]->name[0]); i++);
lem.nterminal = i;
@@ -2031,7 +2044,9 @@
WAITING_FOR_DESTRUCTOR_SYMBOL,
WAITING_FOR_DATATYPE_SYMBOL,
WAITING_FOR_FALLBACK_ID,
- WAITING_FOR_WILDCARD_ID
+ WAITING_FOR_WILDCARD_ID,
+ WAITING_FOR_CLASS_ID,
+ WAITING_FOR_CLASS_TOKEN
};
struct pstate {
char *filename; /* Name of the input file */
@@ -2041,6 +2056,7 @@
struct lemon *gp; /* Global state vector */
enum e_state state; /* The state of the parser */
struct symbol *fallback; /* The fallback token */
+ struct symbol *tkclass; /* Token class symbol */
struct symbol *lhs; /* Left-hand side of current rule */
const char *lhsalias; /* Alias for the LHS */
int nrhs; /* Number of right-hand side symbols seen */
@@ -2345,6 +2361,8 @@
psp->state = WAITING_FOR_FALLBACK_ID;
}else if( strcmp(x,"wildcard")==0 ){
psp->state = WAITING_FOR_WILDCARD_ID;
+ }else if( strcmp(x,"token_class")==0 ){
+ psp->state = WAITING_FOR_CLASS_ID;
}else{
ErrorMsg(psp->filename,psp->tokenlineno,
"Unknown declaration keyword: \"%%%s\".",x);
@@ -2513,6 +2531,40 @@
}
}
break;
+ case WAITING_FOR_CLASS_ID:
+ if( !islower(x[0]) ){
+ ErrorMsg(psp->filename, psp->tokenlineno,
+ "%%token_class must be followed by an identifier: ", x);
+ psp->errorcnt++;
+ psp->state = RESYNC_AFTER_DECL_ERROR;
+ }else if( Symbol_find(x) ){
+ ErrorMsg(psp->filename, psp->tokenlineno,
+ "Symbol \"%s\" already used", x);
+ psp->errorcnt++;
+ psp->state = RESYNC_AFTER_DECL_ERROR;
+ }else{
+ psp->tkclass = Symbol_new(x);
+ psp->tkclass->type = MULTITERMINAL;
+ psp->state = WAITING_FOR_CLASS_TOKEN;
+ }
+ break;
+ case WAITING_FOR_CLASS_TOKEN:
+ if( x[0]=='.' ){
+ psp->state = WAITING_FOR_DECL_OR_RULE;
+ }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){
+ struct symbol *msp = psp->tkclass;
+ msp->nsubsym++;
+ msp->subsym = (struct symbol **) realloc(msp->subsym,
+ sizeof(struct symbol*)*msp->nsubsym);
+ if( !isupper(x[0]) ) x++;
+ msp->subsym[msp->nsubsym-1] = Symbol_new(x);
+ }else{
+ ErrorMsg(psp->filename, psp->tokenlineno,
+ "%%token_class argument \"%s\" should be a token", x);
+ psp->errorcnt++;
+ psp->state = RESYNC_AFTER_DECL_ERROR;
+ }
+ break;
case RESYNC_AFTER_RULE_ERROR:
/* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE;
** break; */
@@ -2867,11 +2919,13 @@
printf(" ::=");
for(i=0; i<rp->nrhs; i++){
sp = rp->rhs[i];
- printf(" %s", sp->name);
if( sp->type==MULTITERMINAL ){
+ printf(" %s", sp->subsym[0]->name);
for(j=1; j<sp->nsubsym; j++){
printf("|%s", sp->subsym[j]->name);
}
+ }else{
+ printf(" %s", sp->name);
}
/* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */
}
@@ -2893,11 +2947,13 @@
if( i==cfp->dot ) fprintf(fp," *");
if( i==rp->nrhs ) break;
sp = rp->rhs[i];
- fprintf(fp," %s", sp->name);
if( sp->type==MULTITERMINAL ){
+ fprintf(fp," %s", sp->subsym[0]->name);
for(j=1; j<sp->nsubsym; j++){
fprintf(fp,"|%s",sp->subsym[j]->name);
}
+ }else{
+ fprintf(fp," %s", sp->name);
}
}
}
@@ -3644,9 +3700,11 @@
fprintf(out,"%s ::=", rp->lhs->name);
for(j=0; j<rp->nrhs; j++){
struct symbol *sp = rp->rhs[j];
- fprintf(out," %s", sp->name);
- if( sp->type==MULTITERMINAL ){
+ if( sp->type!=MULTITERMINAL ){
+ fprintf(out," %s", sp->name);
+ }else{
int k;
+ fprintf(out," %s", sp->subsym[0]->name);
for(k=1; k<sp->nsubsym; k++){
fprintf(out,"|%s",sp->subsym[k]->name);
}
@@ -4114,7 +4172,8 @@
if( in ){
int nextChar;
for(i=1; i<lemp->nterminal && fgets(line,LINESIZE,in); i++){
- lemon_sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+ lemon_sprintf(pattern,"#define %s%-30s %3d\n",
+ prefix,lemp->symbols[i]->name,i);
if( strcmp(line,pattern) ) break;
}
nextChar = fgetc(in);
@@ -4127,7 +4186,7 @@
out = file_open(lemp,".h","wb");
if( out ){
for(i=1; i<lemp->nterminal; i++){
- fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+ fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i);
}
fclose(out);
}
@@ -4497,11 +4556,15 @@
return sp;
}
-/* Compare two symbols for working purposes
+/* Compare two symbols for sorting purposes. Return negative,
+** zero, or positive if a is less then, equal to, or greater
+** than b.
**
** Symbols that begin with upper case letters (terminals or tokens)
** must sort before symbols that begin with lower case letters
-** (non-terminals). Other than that, the order does not matter.
+** (non-terminals). And MULTITERMINAL symbols (created using the
+** %token_class directive) must sort at the very end. Other than
+** that, the order does not matter.
**
** We find experimentally that leaving the symbols in their original
** order (the order they appeared in the grammar file) gives the
@@ -4509,12 +4572,11 @@
*/
int Symbolcmpp(const void *_a, const void *_b)
{
- const struct symbol **a = (const struct symbol **) _a;
- const struct symbol **b = (const struct symbol **) _b;
- int i1 = (**a).index + 10000000*((**a).name[0]>'Z');
- int i2 = (**b).index + 10000000*((**b).name[0]>'Z');
- assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 );
- return i1-i2;
+ const struct symbol *a = *(const struct symbol **) _a;
+ const struct symbol *b = *(const struct symbol **) _b;
+ int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1;
+ int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1;
+ return i1==i2 ? a->index - b->index : i1 - i2;
}
/* There is one instance of the following structure for each
