00001 /* tokenizer - A tokenizer structure that will chop up file into 00002 * tokens. It is aware of quoted strings and otherwise tends to return 00003 * white-space or punctuated-separated words, with punctuation in 00004 * a separate token. This is used by autoSql. */ 00005 00006 #ifndef TOKENIZER_H 00007 #define TOKENIZER_H 00008 00009 struct tokenizer 00010 /* This handles reading in tokens. */ 00011 { 00012 bool reuse; /* True if want to reuse this token. */ 00013 bool eof; /* True at end of file. */ 00014 struct lineFile *lf; /* Underlying file. */ 00015 char *curLine; /* Current line of text. */ 00016 char *linePt; /* Start position within current line. */ 00017 char *string; /* String value of token */ 00018 int sSize; /* Size of string. */ 00019 int sAlloc; /* Allocated string size. */ 00020 /* Some variables set after tokenizerNew to control details of 00021 * parsing. */ 00022 bool leaveQuotes; /* Leave quotes in string. */ 00023 bool uncommentC; /* Take out C (and C++) style comments. */ 00024 bool uncommentShell; /* Take out # style comments. */ 00025 }; 00026 00027 struct tokenizer *tokenizerNew(char *fileName); 00028 /* Return a new tokenizer. */ 00029 00030 struct tokenizer *tokenizerOnLineFile(struct lineFile *lf); 00031 /* Create a new tokenizer on open lineFile. */ 00032 00033 void tokenizerFree(struct tokenizer **pTkz); 00034 /* Tear down a tokenizer. */ 00035 00036 void tokenizerReuse(struct tokenizer *tkz); 00037 /* Reuse token. */ 00038 00039 int tokenizerLineCount(struct tokenizer *tkz); 00040 /* Return line of current token. */ 00041 00042 char *tokenizerFileName(struct tokenizer *tkz); 00043 /* Return name of file. */ 00044 00045 char *tokenizerNext(struct tokenizer *tkz); 00046 /* Return token's next string (also available as tkz->string) or 00047 * NULL at EOF. */ 00048 00049 void tokenizerErrAbort(struct tokenizer *tkz, char *format, ...); 00050 /* Print error message followed by file and line number and 00051 * abort. */ 00052 00053 void tokenizerNotEnd(struct tokenizer *tkz); 00054 /* Squawk if at end. */ 00055 00056 void tokenizerMustHaveNext(struct tokenizer *tkz); 00057 /* Get next token, which must be there. */ 00058 00059 void tokenizerMustMatch(struct tokenizer *tkz, char *string); 00060 /* Require next token to match string. Return next token 00061 * if it does, otherwise abort. */ 00062 00063 #endif /* TOKENIZER_H */ 00064
1.5.2