inc/tokenizer.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	tokenizer
Functions
tokenizer *	tokenizerNew (char *fileName)
tokenizer *	tokenizerOnLineFile (struct lineFile *lf)
void	tokenizerFree (struct tokenizer **pTkz)
void	tokenizerReuse (struct tokenizer *tkz)
int	tokenizerLineCount (struct tokenizer *tkz)
char *	tokenizerFileName (struct tokenizer *tkz)
char *	tokenizerNext (struct tokenizer *tkz)
void	tokenizerErrAbort (struct tokenizer *tkz, char *format,...)
void	tokenizerNotEnd (struct tokenizer *tkz)
void	tokenizerMustHaveNext (struct tokenizer *tkz)
void	tokenizerMustMatch (struct tokenizer *tkz, char *string)

---

Function Documentation

void tokenizerErrAbort	(	struct tokenizer *	tkz,
		char *	format,
			...
	)

Definition at line 174 of file tokenizer.c.

References tokenizer::curLine, errAbort(), tokenizerFileName(), tokenizerLineCount(), and vaWarn().

Referenced by asParseColArraySpec(), asParseColSymSpec(), asParseTableDef(), asParseTokens(), findLowType(), and tokenizerMustMatch().

{
va_list args;
va_start(args, format);
vaWarn(format, args);
errAbort("line %d of %s:\n%s", 
        tokenizerLineCount(tkz), tokenizerFileName(tkz), tkz->curLine);
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* tokenizerFileName

(

struct tokenizer *

tkz

)

Definition at line 55 of file tokenizer.c.

References lineFile::fileName, and tokenizer::lf.

Referenced by tokenizerErrAbort(), and tokenizerNext().

{
return tkz->lf->fileName;
}

Here is the caller graph for this function:

void tokenizerFree

(

struct tokenizer **

pTkz

)

Definition at line 31 of file tokenizer.c.

References freeMem(), freez(), tokenizer::lf, lineFileClose(), and tokenizer::string.

Referenced by asParseLineFile().

{
struct tokenizer *tkz;
if ((tkz = *pTkz) != NULL)
    {
    freeMem(tkz->string);
    lineFileClose(&tkz->lf);
    freez(pTkz);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

int tokenizerLineCount

(

struct tokenizer *

tkz

)

Definition at line 49 of file tokenizer.c.

References tokenizer::lf, and lineFile::lineIx.

Referenced by tokenizerErrAbort().

{
return tkz->lf->lineIx;
}

Here is the caller graph for this function:

void tokenizerMustHaveNext

(

struct tokenizer *

tkz

)

Definition at line 192 of file tokenizer.c.

References errAbort(), and tokenizerNext().

Referenced by asParseColArraySpec(), asParseColDef(), asParseColSymSpec(), asParseTableDef(), and tokenizerMustMatch().

{
if (tokenizerNext(tkz) == NULL)
    errAbort("Unexpected end of file");
}

Here is the call graph for this function:

Here is the caller graph for this function:

void tokenizerMustMatch	(	struct tokenizer *	tkz,
		char *	string
	)

Definition at line 199 of file tokenizer.c.

References sameWord, tokenizer::string, tokenizerErrAbort(), and tokenizerMustHaveNext().

Referenced by asParseColArraySpec(), asParseColDef(), asParseColSymSpec(), and asParseTableDef().

{
if (sameWord(tkz->string, string))
    tokenizerMustHaveNext(tkz);
else
    tokenizerErrAbort(tkz, "Expecting %s got %s", string, tkz->string);
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct tokenizer* tokenizerNew

(

char *

fileName

)

[read]

Definition at line 25 of file tokenizer.c.

References lineFileOpen(), tokenizerOnLineFile(), and TRUE.

{
return tokenizerOnLineFile(lineFileOpen(fileName, TRUE));
}

Here is the call graph for this function:

char* tokenizerNext

(

struct tokenizer *

tkz

)

Definition at line 61 of file tokenizer.c.

References tokenizer::curLine, tokenizer::eof, errAbort(), FALSE, tokenizer::lf, lineFileNext(), tokenizer::linePt, tokenizer::reuse, skipLeadingSpaces(), tokenizer::string, stringIn, tokenizerFileName(), TRUE, tokenizer::uncommentC, and tokenizer::uncommentShell.

Referenced by asParseTokens(), and tokenizerMustHaveNext().

{
char *start, *end;
char c, *s;
int size;
if (tkz->reuse)
    {
    tkz->reuse = FALSE;
    return tkz->string;
    }
for (;;)        /* Skip over white space and comments. */
    {
    int lineSize;
    s = start = skipLeadingSpaces(tkz->linePt);
    if ((c = start[0]) != 0)
        {
        if (tkz->uncommentC && c == '/')
             {
             if (start[1] == '/')
                 ;  /* Keep going in loop effectively ignoring rest of line. */
             else if (start[1] == '*')
                 {
                 start += 2;
                 for (;;)
                     {
                     char *end = stringIn("*/", start);
                     if (end != NULL)
                          {
                          tkz->linePt = end+2;
                          break;
                          }
                     if (!lineFileNext(tkz->lf, &tkz->curLine, &lineSize))
                          errAbort("End of file (%s) in comment", tokenizerFileName(tkz));
                     start = tkz->curLine;
                     }
                 continue;
                 }
             else
                 break;
             }
        else if (tkz->uncommentShell && c == '#')
             ;  /* Keep going in loop effectively ignoring rest of line. */
        else
            break;      /* Got something real. */
        }
    if (!lineFileNext(tkz->lf, &tkz->curLine, &lineSize))
        {
        tkz->eof = TRUE;
        return NULL;
        }
    tkz->linePt = tkz->curLine;
    }
if (isalnum(c) || (c == '_'))
    {
    for (;;)
        {
        s++;
        if (!(isalnum(*s) || (*s == '_')))
            break;
        }
    end = s;
    }
else if (c == '"' || c == '\'')
    {
    char quot = c;
    if (tkz->leaveQuotes)
        start = s++;
    else
        start = ++s;
    for (;;)
        {
        c = *s;
        if (c == quot)
            {
            if (s[-1] == '\\')
                {
                if (s >= start+2 && s[-2] == '\\')
                    break;
                }
            else
                break;
            }
        else if (c == 0)
            {
            break;
            }
        ++s;
        }
    end = s;
    if (c != 0)
        ++s;
    if (tkz->leaveQuotes)
        end += 1;
    }
else
    {
    end = ++s;
    }
tkz->linePt = s;
size = end - start;
if (size >= tkz->sAlloc)
    {
    tkz->sAlloc = size+128;
    tkz->string = needMoreMem(tkz->string, 0, tkz->sAlloc);
    }
memcpy(tkz->string, start, size);
tkz->string[size] = 0;
return tkz->string;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void tokenizerNotEnd

(

struct tokenizer *

tkz

)

Definition at line 185 of file tokenizer.c.

References tokenizer::eof, and errAbort().

{
if (tkz->eof)
    errAbort("Unexpected end of file");
}

Here is the call graph for this function:

struct tokenizer* tokenizerOnLineFile

(

struct lineFile *

lf

)

[read]

Definition at line 13 of file tokenizer.c.

References AllocVar, tokenizer::curLine, tokenizer::lf, tokenizer::linePt, needMem(), tokenizer::sAlloc, and tokenizer::string.

Referenced by asParseLineFile(), and tokenizerNew().

{
struct tokenizer *tkz;
AllocVar(tkz);
tkz->sAlloc = 128;
tkz->string = needMem(tkz->sAlloc);
tkz->lf = lf;
tkz->curLine = tkz->linePt = "";
return tkz;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void tokenizerReuse

(

struct tokenizer *

tkz

)

Definition at line 43 of file tokenizer.c.

References tokenizer::reuse, and TRUE.

{
tkz->reuse = TRUE;
}

---