lib/linefile.c File Reference

#include "common.h"
#include "hash.h"
#include <fcntl.h>
#include "dystring.h"
#include "errabort.h"
#include "linefile.h"
#include "pipeline.h"
#include <signal.h>

Include dependency graph for linefile.c:

Go to the source code of this file.

Functions
char *	getFileNameFromHdrSig (char *m)
static char **	getDecompressor (char *fileName)
static void	metaDataAdd (struct lineFile *lf, char *line)
static void	metaDataFree (struct lineFile *lf)
void	lineFileSetMetaDataOutput (struct lineFile *lf, FILE *f)
void	lineFileSetUniqueMetaData (struct lineFile *lf)
static char *	headerBytes (char *fileName, int numbytes)
lineFile *	lineFileDecompress (char *fileName, bool zTerm)
lineFile *	lineFileDecompressFd (char *name, bool zTerm, int fd)
lineFile *	lineFileDecompressMem (bool zTerm, char *mem, long size)
lineFile *	lineFileAttach (char *fileName, bool zTerm, int fd)
lineFile *	lineFileOnString (char *name, bool zTerm, char *s)
void	lineFileExpandBuf (struct lineFile *lf, int newSize)
lineFile *	lineFileStdin (bool zTerm)
lineFile *	lineFileMayOpen (char *fileName, bool zTerm)
lineFile *	lineFileOpen (char *fileName, bool zTerm)
void	lineFileReuse (struct lineFile *lf)
void	lineFileSeek (struct lineFile *lf, off_t offset, int whence)
int	lineFileLongNetRead (int fd, char *buf, int size)
static void	determineNlType (struct lineFile *lf, char *buf, int bufSize)
boolean	lineFileNext (struct lineFile *lf, char **retStart, int *retSize)
void	lineFileVaAbort (struct lineFile *lf, char *format, va_list args)
void	lineFileAbort (struct lineFile *lf, char *format,...)
void	lineFileUnexpectedEnd (struct lineFile *lf)
void	lineFileNeedNext (struct lineFile *lf, char **retStart, int *retSize)
void	lineFileClose (struct lineFile **pLf)
void	lineFileCloseList (struct lineFile **pList)
void	lineFileExpectWords (struct lineFile *lf, int expecting, int got)
void	lineFileExpectAtLeast (struct lineFile *lf, int expecting, int got)
void	lineFileShort (struct lineFile *lf)
boolean	lineFileNextReal (struct lineFile *lf, char **retStart)
int	lineFileChopNext (struct lineFile *lf, char *words[], int maxWords)
int	lineFileChopCharNext (struct lineFile *lf, char sep, char *words[], int maxWords)
int	lineFileChopNextTab (struct lineFile *lf, char *words[], int maxWords)
boolean	lineFileNextCharRow (struct lineFile *lf, char sep, char *words[], int wordCount)
boolean	lineFileNextRow (struct lineFile *lf, char *words[], int wordCount)
boolean	lineFileNextRowTab (struct lineFile *lf, char *words[], int wordCount)
int	lineFileNeedFullNum (struct lineFile *lf, char *words[], int wordIx)
int	lineFileNeedNum (struct lineFile *lf, char *words[], int wordIx)
double	lineFileNeedDouble (struct lineFile *lf, char *words[], int wordIx)
void	lineFileSkip (struct lineFile *lf, int lineCount)
char *	lineFileSkipToLineStartingWith (struct lineFile *lf, char *start, int maxCount)
boolean	lineFileParseHttpHeader (struct lineFile *lf, char **hdr, boolean *chunked, int *contentLength)
dyString *	lineFileSlurpHttpBody (struct lineFile *lf, boolean chunked, int contentLength)
Variables
static char const	rcsid [] = "$Id: linefile.c,v 1.54 2007/03/23 08:12:42 kent Exp $"

---

Function Documentation

static void determineNlType	(	struct lineFile *	lf,
		char *	buf,
		int	bufSize
	)			[static]

Definition at line 280 of file linefile.c.

References nlt_dos, nlt_mac, nlt_undet, nlt_unix, and lineFile::nlType.

Referenced by lineFileNext().

{
char *c = buf;
if (bufSize==0) return;
if (lf->nlType != nlt_undet) return;  /* if already determined just exit */
lf->nlType = nlt_unix;  /* start with default of unix lf type */
while (c < buf+bufSize)
    {
    if (*c=='\r')
        {
        lf->nlType = nlt_mac;
        if (++c < buf+bufSize) 
            if (*c == '\n') 
                lf->nlType = nlt_dos;
        return;
        }
    if (*(c++) == '\n')
        {
        return;
        }
    }
}

Here is the caller graph for this function:

static char** getDecompressor

(

char *

fileName

)

[static]

Definition at line 33 of file linefile.c.

References endsWith().

Referenced by lineFileDecompress(), lineFileDecompressFd(), lineFileDecompressMem(), and lineFileMayOpen().

{
static char *GZ_READ[] = {"gzip", "-dc", NULL};
static char *Z_READ[] = {"compress", "-dc", NULL};
static char *BZ2_READ[] = {"bzip2", "-dc", NULL};

if (endsWith(fileName, ".gz"))
    return GZ_READ;
else if (endsWith(fileName, ".Z"))
    return Z_READ;
else if (endsWith(fileName, ".bz2"))
    return BZ2_READ;
else
    return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* getFileNameFromHdrSig

(

char *

m

)

Definition at line 18 of file linefile.c.

References cloneString(), safef(), and startsWith().

Referenced by lineFileDecompress(), and lineFileDecompressMem().

{
char buf[20];
char *ext=NULL;
if (startsWith("\x1f\x8b",m)) ext = "gz";
else if (startsWith("\x1f\x9d\x90",m)) ext = "Z";
else if (startsWith("BZ",m)) ext = "bz2";
if (ext==NULL) 
    return NULL;
safef(buf, sizeof(buf), "somefile.%s", ext);
return cloneString(buf);
}   

Here is the call graph for this function:

Here is the caller graph for this function:

static char* headerBytes	(	char *	fileName,
		int	numbytes
	)			[static]

Definition at line 98 of file linefile.c.

References freez(), and needMem().

Referenced by lineFileDecompress().

{
int fd,bytesread=0;
char *result = NULL;
if ((fd = open(fileName, O_RDONLY)) >= 0)
    {
    result=needMem(numbytes+1);
    if ((bytesread=read(fd,result,numbytes)) < numbytes) 
        freez(&result);  /* file too short? can read numbytes */
    else
        result[numbytes]=0;
    close(fd);
    }
return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileAbort	(	struct lineFile *	lf,
		char *	format,
			...
	)

Definition at line 472 of file linefile.c.

References lineFileVaAbort().

{
va_list args;
va_start(args, format);
lineFileVaAbort(lf, format, args);
va_end(args);
}

Here is the call graph for this function:

struct lineFile* lineFileAttach	(	char *	fileName,
		bool	zTerm,
		int	fd
	)			[read]

Definition at line 168 of file linefile.c.

References AllocVar, lineFile::buf, lineFile::bufSize, cloneString(), lineFile::fd, lineFile::fileName, needMem(), and lineFile::zTerm.

Referenced by lineFileDecompress(), lineFileDecompressFd(), lineFileDecompressMem(), lineFileMayOpen(), lineFileStdin(), netHttpLineFileMayOpen(), netLineFileMayOpen(), netLineFileMayOpenCatchError(), netUrlHead(), and pipelineLineFile().

{
struct lineFile *lf;
AllocVar(lf);
lf->fileName = cloneString(fileName);
lf->fd = fd;
lf->bufSize = 64*1024;
lf->zTerm = zTerm;
lf->buf = needMem(lf->bufSize+1);
return lf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileChopCharNext	(	struct lineFile *	lf,
		char	sep,
		char *	words[],
		int	maxWords
	)

Definition at line 585 of file linefile.c.

References chopByChar(), and lineFileNext().

Referenced by lineFileNextCharRow().

{
int lineSize, wordCount;
char *line;

while (lineFileNext(lf, &line, &lineSize))
    {
    if (line[0] == '#')
        continue;
    wordCount = chopByChar(line, sep, words, maxWords);
    if (wordCount != 0)
        return wordCount;
    }
return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileChopNext	(	struct lineFile *	lf,
		char *	words[],
		int	maxWords
	)

Definition at line 568 of file linefile.c.

References chopByWhite(), and lineFileNext().

Referenced by lineFileNextRow().

{
int lineSize, wordCount;
char *line;

while (lineFileNext(lf, &line, &lineSize))
    {
    if (line[0] == '#')
        continue;
    wordCount = chopByWhite(line, words, maxWords);
    if (wordCount != 0)
        return wordCount;
    }
return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileChopNextTab	(	struct lineFile *	lf,
		char *	words[],
		int	maxWords
	)

Definition at line 603 of file linefile.c.

References chopByChar(), and lineFileNext().

Referenced by lineFileNextRowTab().

{
int lineSize, wordCount;
char *line;

while (lineFileNext(lf, &line, &lineSize))
    {
    if (line[0] == '#')
        continue;
    wordCount = chopByChar(line, '\t', words, maxWords);
    if (wordCount != 0)
        return wordCount;
    }
return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileClose

(

struct lineFile **

pLf

)

Definition at line 494 of file linefile.c.

References lineFile::buf, lineFile::fd, lineFile::fileName, freeMem(), freez(), metaDataFree(), pipelineFree(), pipelineWait(), and lineFile::pl.

Referenced by aliLines(), bigBlat(), blastFileFree(), blastFileReadAll(), cgiFromFile(), chainReadAllWithMeta(), chainReadUsedSwap(), closePipeline(), convertEpsToPdf(), countWordsInFile(), dnaLoadStackFree(), doDetailLine(), dtdParse(), faReadAllSeqMixable(), faReadNamedSeq(), gapCalcFromFile(), gapCalcFromString(), genoFindDirect(), gffFileAdd(), gfPcrInputLoadAll(), hashNameIntFile(), hashTwoColumnFile(), hashWordsInFile(), htmlCookieFileRead(), lineFileCloseList(), loadPsls(), mafFileFree(), mafNextWithPos(), maskFromOut(), netUrlHead(), phyloOpenTree(), pslLoadAll(), quotedPrintableDecode(), raFoldIn(), raReadAll(), raReadSingle(), readPartHeaderMB(), readPslToBinKeeper(), searchOneIndex(), slNameLoadReal(), spacedColumnFromFile(), tokenizerFree(), trixOpen(), twoBitSpecNewFile(), and xAliLoadAll().

{
struct lineFile *lf;
if ((lf = *pLf) != NULL)
    {
    if (lf->pl != NULL)
        {
        pipelineWait(lf->pl);
        pipelineFree(&lf->pl);
        }
    else if (lf->fd > 0 && lf->fd != fileno(stdin))
        {
        close(lf->fd);
        freeMem(lf->buf);
        }
    freeMem(lf->fileName);
    metaDataFree(lf);
    freez(pLf);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileCloseList

(

struct lineFile **

pList

)

Definition at line 516 of file linefile.c.

References lineFileClose(), and lineFile::next.

{
struct lineFile *el, *next;

for (el = *pList; el != NULL; el = next)
    {
    next = el->next;
    lineFileClose(&el);
    }
*pList = NULL;
}

Here is the call graph for this function:

struct lineFile* lineFileDecompress	(	char *	fileName,
		bool	zTerm
	)			[read]

Definition at line 117 of file linefile.c.

References freez(), getDecompressor(), getFileNameFromHdrSig(), headerBytes(), lineFileAttach(), pipelineFd(), pipelineOpen1(), pipelineRead, and lineFile::pl.

Referenced by lineFileMayOpen().

{
struct pipeline *pl;
struct lineFile *lf;
char *testName = NULL;
char *testbytes = NULL;    /* the header signatures for .gz .bz2, .Z are all 2 or 3 bytes only */
if (fileName==NULL)
  return NULL;
testbytes=headerBytes(fileName,3);
if (!testbytes)
    return NULL;  /* avoid error from pipeline */
testName=getFileNameFromHdrSig(testbytes);
freez(&testbytes);
if (!testName)
    return NULL;  /* avoid error from pipeline */
pl = pipelineOpen1(getDecompressor(fileName), pipelineRead, fileName, NULL);
lf = lineFileAttach(fileName, zTerm, pipelineFd(pl));
lf->pl = pl;
return lf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* lineFileDecompressFd	(	char *	name,
		bool	zTerm,
		int	fd
	)			[read]

Definition at line 139 of file linefile.c.

References getDecompressor(), lineFileAttach(), pipelineFd(), pipelineOpenFd1(), pipelineRead, and lineFile::pl.

Referenced by netLineFileMayOpen(), and netLineFileMayOpenCatchError().

{
struct pipeline *pl;
struct lineFile *lf;
pl = pipelineOpenFd1(getDecompressor(name), pipelineRead, fd, STDERR_FILENO);
lf = lineFileAttach(name, zTerm, pipelineFd(pl));
lf->pl = pl;
return lf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* lineFileDecompressMem	(	bool	zTerm,
		char *	mem,
		long	size
	)			[read]

Definition at line 152 of file linefile.c.

References lineFile::fileName, getDecompressor(), getFileNameFromHdrSig(), lineFileAttach(), pipelineFd(), pipelineOpenMem1(), pipelineRead, and lineFile::pl.

{
struct pipeline *pl;
struct lineFile *lf;
char *fileName = getFileNameFromHdrSig(mem);
if (fileName==NULL)
  return NULL;
pl = pipelineOpenMem1(getDecompressor(fileName), pipelineRead, mem, size, STDERR_FILENO);
lf = lineFileAttach(fileName, zTerm, pipelineFd(pl));
lf->pl = pl;
return lf;
}

Here is the call graph for this function:

void lineFileExpandBuf	(	struct lineFile *	lf,
		int	newSize
	)

Definition at line 196 of file linefile.c.

References lineFile::buf, lineFile::bufSize, lineFile::bytesInBuf, and needMoreMem().

Referenced by lineFileNext().

{
assert(newSize > lf->bufSize);
lf->buf = needMoreMem(lf->buf, lf->bytesInBuf, newSize);
lf->bufSize = newSize;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileExpectAtLeast	(	struct lineFile *	lf,
		int	expecting,
		int	got
	)

Definition at line 537 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFile::lineIx.

{
if (got < expecting)
    errAbort("Expecting at least %d words line %d of %s got %d", 
            expecting, lf->lineIx, lf->fileName, got);
}

Here is the call graph for this function:

void lineFileExpectWords	(	struct lineFile *	lf,
		int	expecting,
		int	got
	)

Definition at line 529 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFile::lineIx.

Referenced by lineFileNextCharRow(), lineFileNextRow(), lineFileNextRowTab(), and mafNextWithPos().

{
if (expecting != got)
    errAbort("Expecting %d words line %d of %s got %d", 
            expecting, lf->lineIx, lf->fileName, got);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileLongNetRead	(	int	fd,
		char *	buf,
		int	size
	)

Definition at line 262 of file linefile.c.

Referenced by lineFileNext().

{
int oneSize, totalRead = 0;

while (size > 0)
    {
    oneSize = read(fd, buf, size);
    if (oneSize <= 0)
        break;
    totalRead += oneSize;
    buf += oneSize;
    size -= oneSize;
    }
return totalRead;
}

Here is the caller graph for this function:

struct lineFile* lineFileMayOpen	(	char *	fileName,
		bool	zTerm
	)			[read]

Definition at line 211 of file linefile.c.

References lineFile::fd, getDecompressor(), lineFileAttach(), lineFileDecompress(), lineFileStdin(), and sameString.

Referenced by lineFileOpen(), mafMayOpen(), and raFoldIn().

{
if (sameString(fileName, "stdin"))
    return lineFileStdin(zTerm);
else if (getDecompressor(fileName) != NULL)
    return lineFileDecompress(fileName, zTerm);
else
    {
    int fd = open(fileName, O_RDONLY);
    if (fd == -1)
        return NULL;
    return lineFileAttach(fileName, zTerm, fd);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

double lineFileNeedDouble	(	struct lineFile *	lf,
		char *	words[],
		int	wordIx
	)

Definition at line 688 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFile::lineIx.

Referenced by hpfNext().

{
char *valEnd;
char *val = words[wordIx];
double doubleValue;

doubleValue = strtod(val, &valEnd);
if ((*val == '\0') || (*valEnd != '\0'))
    errAbort("Expecting double field %d line %d of %s, got %s",
        wordIx+1, lf->lineIx, lf->fileName, val);
return doubleValue;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileNeedFullNum	(	struct lineFile *	lf,
		char *	words[],
		int	wordIx
	)

Definition at line 660 of file linefile.c.

References errAbort(), lineFile::fileName, lineFileNeedNum(), and lineFile::lineIx.

{
char *c;
for (c = words[wordIx]; *c; c++)
    {
    if (*c == '-' || isdigit(*c))
        /* NOTE: embedded '-' will be caught by lineFileNeedNum */
        continue;
    errAbort("Expecting number field %d line %d of %s, got %s", 
            wordIx+1, lf->lineIx, lf->fileName, words[wordIx]);
    }
return lineFileNeedNum(lf, words, wordIx);
}

Here is the call graph for this function:

void lineFileNeedNext	(	struct lineFile *	lf,
		char **	retStart,
		int *	retSize
	)

Definition at line 487 of file linefile.c.

References lineFileNext(), and lineFileUnexpectedEnd().

Referenced by axtReadWithPos(), hpfNext(), and mafMayOpen().

{
if (!lineFileNext(lf, retStart, retSize))
    lineFileUnexpectedEnd(lf);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int lineFileNeedNum	(	struct lineFile *	lf,
		char *	words[],
		int	wordIx
	)

Definition at line 676 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFile::lineIx.

Referenced by axtReadWithPos(), axtScoreSchemeReadLf(), chainReadBlocks(), chainReadChainLine(), hashNameIntFile(), hpfNext(), lineFileNeedFullNum(), mafNextWithPos(), and readPslToBinKeeper().

{
char *ascii = words[wordIx];
char c = ascii[0];
if (c != '-' && !isdigit(c))
    errAbort("Expecting number field %d line %d of %s, got %s", 
        wordIx+1, lf->lineIx, lf->fileName, ascii);
return atoi(ascii);
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean lineFileNext	(	struct lineFile *	lf,
		char **	retStart,
		int *	retSize
	)

Definition at line 304 of file linefile.c.

References lineFile::buf, lineFile::bufOffsetInFile, lineFile::bufSize, lineFile::bytesInBuf, determineNlType(), errAbort(), FALSE, lineFile::fd, lineFile::fileName, lineFile::lineEnd, lineFileExpandBuf(), lineFileLongNetRead(), lineFile::lineIx, lineFile::lineStart, metaDataAdd(), lineFile::metaOutput, nlt_dos, nlt_mac, nlt_undet, nlt_unix, lineFile::nlType, lineFile::reuse, and TRUE.

Referenced by axtReadWithPos(), axtScoreSchemeReadLf(), bfNextLine(), cgiFromFile(), convertEpsToPdf(), countWordsInFile(), dtdxTag(), eatComment(), emblLineGroup(), faMixedSpeedReadNext(), gffFileAdd(), hashWordsInFile(), lineFileChopCharNext(), lineFileChopNext(), lineFileChopNextTab(), lineFileNeedNext(), lineFileNextReal(), lineFileParseHttpHeader(), lineFileSkip(), lineFileSkipToLineStartingWith(), lineFileSlurpHttpBody(), maskFromOut(), netUrlHead(), nextLine(), phyloReadTree(), pslxFileOpen(), pslxFileOpenWithMetaConfig(), quotedPrintableDecode(), raFoldInOneRetName(), raNextRecord(), readAllLines(), readPartHeaderMB(), spacedColumnFromLineFile(), tokenizerNext(), trixOpen(), and trixSearchWordResults().

{
char *buf = lf->buf;
int bytesInBuf = lf->bytesInBuf;
int endIx = lf->lineEnd;
boolean gotLf = FALSE;
int newStart;

if (lf->reuse)
    {
    lf->reuse = FALSE;
    if (retSize != NULL)
        *retSize = lf->lineEnd - lf->lineStart;
    *retStart = buf + lf->lineStart;
    if (lf->metaOutput && *retStart[0] == '#') 
        metaDataAdd(lf, *retStart); 
    return TRUE;
    }

determineNlType(lf, buf+endIx, bytesInBuf);

/* Find next end of line in buffer. */
switch(lf->nlType)
    {
    case nlt_unix:
    case nlt_dos:
        for (endIx = lf->lineEnd; endIx < bytesInBuf; ++endIx)
            {
            if (buf[endIx] == '\n')
                {
                gotLf = TRUE;
                endIx += 1;
                break;
                }
            }
        break;
    case nlt_mac:
        for (endIx = lf->lineEnd; endIx < bytesInBuf; ++endIx)
            {
            if (buf[endIx] == '\r')
                {
                gotLf = TRUE;
                endIx += 1;
                break;
                }
            }
        break;
    case nlt_undet:
        break;
    }

/* If not in buffer read in a new buffer's worth. */
while (!gotLf)
    {
    int oldEnd = lf->lineEnd;
    int sizeLeft = bytesInBuf - oldEnd;
    int bufSize = lf->bufSize;
    int readSize = bufSize - sizeLeft;

    if (oldEnd > 0 && sizeLeft > 0)
        {
        memmove(buf, buf+oldEnd, sizeLeft);
        }
    lf->bufOffsetInFile += oldEnd;
    if (lf->fd >= 0)
        readSize = lineFileLongNetRead(lf->fd, buf+sizeLeft, readSize);
    else
        readSize = 0;

    if ((readSize == 0) && (endIx > oldEnd))
        {
        endIx = sizeLeft;
        buf[endIx] = 0;
        lf->bytesInBuf = newStart = lf->lineStart = 0;
        lf->lineEnd = endIx;
        ++lf->lineIx;
        if (retSize != NULL)
            *retSize = endIx - newStart;
        *retStart = buf + newStart;
        if (*retStart[0] == '#')
            metaDataAdd(lf, *retStart);
        return TRUE;
        }
    else if (readSize <= 0)
        {
        lf->bytesInBuf = lf->lineStart = lf->lineEnd = 0;
        return FALSE;
        }
    bytesInBuf = lf->bytesInBuf = readSize + sizeLeft;
    lf->lineEnd = 0;

    determineNlType(lf, buf+endIx, bytesInBuf);
        
    /* Look for next end of line.  */
    switch(lf->nlType)
        {
        case nlt_unix:
        case nlt_dos:
            for (endIx = sizeLeft; endIx <bytesInBuf; ++endIx)
                {
                if (buf[endIx] == '\n')
                    {
                    endIx += 1;
                    gotLf = TRUE;
                    break;
                    }
                }
            break;
        case nlt_mac:
            for (endIx = sizeLeft; endIx <bytesInBuf; ++endIx)
                {
                if (buf[endIx] == '\r')
                    {
                    endIx += 1;
                    gotLf = TRUE;
                    break;
                    }
                }
            break;
        case nlt_undet:
            break;
        }
    if (!gotLf && bytesInBuf == lf->bufSize)
        {
        if (bufSize >= 512*1024*1024)
            {
            errAbort("Line too long (more than %d chars) line %d of %s",
                lf->bufSize, lf->lineIx+1, lf->fileName);
            }
        else
            {
            lineFileExpandBuf(lf, bufSize*2);
            buf = lf->buf;
            }
        }
    }

if (lf->zTerm)
    {
    buf[endIx-1] = 0;
    if ((lf->nlType == nlt_dos) && (buf[endIx-2]=='\r'))
        {
        buf[endIx-2] = 0;
        }
    }

lf->lineStart = newStart = lf->lineEnd;
lf->lineEnd = endIx;
++lf->lineIx;
if (retSize != NULL)
    *retSize = endIx - newStart;
*retStart = buf + newStart;
if (*retStart[0] == '#')
    metaDataAdd(lf, *retStart);
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean lineFileNextCharRow	(	struct lineFile *	lf,
		char	sep,
		char *	words[],
		int	wordCount
	)

Definition at line 621 of file linefile.c.

References FALSE, lineFileChopCharNext(), lineFileExpectWords(), and TRUE.

{
int wordsRead;
wordsRead = lineFileChopCharNext(lf, sep, words, wordCount);
if (wordsRead == 0)
    return FALSE;
if (wordsRead < wordCount)
    lineFileExpectWords(lf, wordCount, wordsRead);
return TRUE;
}

Here is the call graph for this function:

boolean lineFileNextReal	(	struct lineFile *	lf,
		char **	retStart
	)

Definition at line 551 of file linefile.c.

References FALSE, lineFileNext(), skipLeadingSpaces(), and TRUE.

Referenced by dnaLoadNextFromStack(), dnaLoadStackNew(), dtdxTag(), gfWebConfigRead(), htmlCookieFileRead(), pslNext(), raFoldInOneRetName(), readTaggedNumLine(), slNameLoadReal(), and twoBitSpecNewFile().

{
char *s, c;
while (lineFileNext(lf, retStart, NULL))
    {
    s = skipLeadingSpaces(*retStart);
    c = s[0];
    if (c != 0 && c != '#')
        {
        return TRUE;
        }
    }
return FALSE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean lineFileNextRow	(	struct lineFile *	lf,
		char *	words[],
		int	wordCount
	)

Definition at line 634 of file linefile.c.

References FALSE, lineFileChopNext(), lineFileExpectWords(), and TRUE.

Referenced by readPslToBinKeeper().

{
int wordsRead;
wordsRead = lineFileChopNext(lf, words, wordCount);
if (wordsRead == 0)
    return FALSE;
if (wordsRead < wordCount)
    lineFileExpectWords(lf, wordCount, wordsRead);
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean lineFileNextRowTab	(	struct lineFile *	lf,
		char *	words[],
		int	wordCount
	)

Definition at line 647 of file linefile.c.

References FALSE, lineFileChopNextTab(), lineFileExpectWords(), and TRUE.

Referenced by loadPsls().

{
int wordsRead;
wordsRead = lineFileChopNextTab(lf, words, wordCount);
if (wordsRead == 0)
    return FALSE;
if (wordsRead < wordCount)
    lineFileExpectWords(lf, wordCount, wordsRead);
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* lineFileOnString	(	char *	name,
		bool	zTerm,
		char *	s
	)			[read]

Definition at line 181 of file linefile.c.

References AllocVar, lineFile::buf, lineFile::bufSize, lineFile::bytesInBuf, cloneString(), lineFile::fd, lineFile::fileName, and lineFile::zTerm.

Referenced by asParseText(), gapCalcFromString(), quotedPrintableDecode(), and readPartHeaderMB().

{
struct lineFile *lf;
AllocVar(lf);
lf->fileName = cloneString(name);
lf->fd = -1;
lf->bufSize = lf->bytesInBuf = strlen(s);
lf->zTerm = zTerm;
lf->buf = s;
return lf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* lineFileOpen	(	char *	fileName,
		bool	zTerm
	)			[read]

Definition at line 227 of file linefile.c.

References errAbort(), errno, and lineFileMayOpen().

Referenced by asParseFile(), axtScoreSchemeRead(), bigBlat(), blastFileOpenVerify(), cgiFromFile(), chainReadAllWithMeta(), chainReadUsedSwap(), convertEpsToPdf(), countWordsInFile(), dnaLoadStackNew(), dtdParse(), emblOpen(), faReadAllSeqMixable(), faReadNamedSeq(), gapCalcFromFile(), genoFindDirect(), gfClient(), gffFileAdd(), gfPcrInputLoadAll(), gfWebConfigRead(), hashNameIntFile(), hashTwoColumnFile(), hashWordsInFile(), htmlCookieFileRead(), loadPsls(), maskFromOut(), phyloOpenTree(), pslxFileOpen(), pslxFileOpenWithMetaConfig(), raReadAll(), raReadSingle(), readAllLines(), readPslToBinKeeper(), searchOneIndex(), slNameLoadReal(), spacedColumnFromFile(), tokenizerNew(), trixOpen(), twoBitSpecNewFile(), and xAliLoadAll().

{
struct lineFile *lf = lineFileMayOpen(fileName, zTerm);
if (lf == NULL)
    errAbort("Couldn't open %s , %s", fileName, strerror(errno));
return lf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean lineFileParseHttpHeader	(	struct lineFile *	lf,
		char **	hdr,
		boolean *	chunked,
		int *	contentLength
	)

Definition at line 729 of file linefile.c.

References cloneString(), dyStringAppendC(), dyStringAppendN(), dyStringClear, dyStringFree, FALSE, lineFile::fileName, lineFileNext(), lineFileReuse(), newDyString(), nextWord(), sameString, startsWith(), dyString::string, TRUE, and warn().

Referenced by netHttpGetMultiple().

                     : chunked" or if it has a contentLength. */
{
  struct dyString *header = newDyString(1024);
  char *line;
  int lineSize;

  if (chunked != NULL)
    *chunked = FALSE;
  if (contentLength != NULL)
    *contentLength = -1;
  dyStringClear(header);
  if (lineFileNext(lf, &line, &lineSize))
    {
      if (startsWith("HTTP/", line))
        {
        char *version, *code;
        dyStringAppendN(header, line, lineSize-1);
        dyStringAppendC(header, '\n');
        version = nextWord(&line);
        code = nextWord(&line);
        if (code == NULL)
            {
            warn("%s: Expecting HTTP/<version> <code> header line, got this: %s\n", lf->fileName, header->string);
            *hdr = cloneString(header->string);
            dyStringFree(&header);
            return FALSE;
            }
        if (!sameString(code, "200"))
            {
            warn("%s: Errored HTTP response header: %s %s %s\n", lf->fileName, version, code, line);
            *hdr = cloneString(header->string);
            dyStringFree(&header);
            return FALSE;
            }
        while (lineFileNext(lf, &line, &lineSize))
            {
            /* blank line means end of HTTP header */
            if ((line[0] == '\r' && line[1] == 0) || line[0] == 0)
                break;
            if (strstr(line, "Transfer-Encoding: chunked") && chunked != NULL)
                *chunked = TRUE;
            dyStringAppendN(header, line, lineSize-1);
            dyStringAppendC(header, '\n');
            if (strstr(line, "Content-Length:"))
              {
                code = nextWord(&line);
                code = nextWord(&line);
                if (contentLength != NULL)
                    *contentLength = atoi(code);
              }
            }
        }
      else
        {
          /* put the line back, don't put it in header/hdr */
          lineFileReuse(lf);
          warn("%s: Expecting HTTP/<version> <code> header line, got this: %s\n", lf->fileName, header->string);
          *hdr = cloneString(header->string);
          dyStringFree(&header);
          return FALSE;
        }
    }
  else
    {
      *hdr = cloneString(header->string);
      dyStringFree(&header);
      return FALSE;
    }

  *hdr = cloneString(header->string);
  dyStringFree(&header);
  return TRUE;
} /* lineFileParseHttpHeader */

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileReuse

(

struct lineFile *

lf

)

Definition at line 236 of file linefile.c.

References lineFile::reuse, and TRUE.

Referenced by bfSkipBlankLines(), blastFileNextQuery(), dnaLoadStackNew(), emblLineGroup(), faMixedSpeedReadNext(), lineFileParseHttpHeader(), lineFileSlurpHttpBody(), nextBlock(), nextBlockLine(), pslxFileOpen(), pslxFileOpenWithMetaConfig(), and raNextRecord().

{
lf->reuse = TRUE;
}

Here is the caller graph for this function:

void lineFileSeek	(	struct lineFile *	lf,
		off_t	offset,
		int	whence
	)

Definition at line 243 of file linefile.c.

References lineFile::bufOffsetInFile, lineFile::bytesInBuf, errnoAbort(), FALSE, lineFile::fd, lineFile::fileName, lineFile::lineEnd, lineFile::lineStart, lineFile::pl, lineFile::reuse, and SEEK_SET.

Referenced by mafRewind(), and trixSearchWordResults().

{
if (lf->pl != NULL)
    errnoAbort("Can't lineFileSeek on a compressed file: %s", lf->fileName);
lf->reuse = FALSE;
if (whence == SEEK_SET && offset >= lf->bufOffsetInFile 
        && offset < lf->bufOffsetInFile + lf->bytesInBuf)
    {
    lf->lineStart = lf->lineEnd = offset - lf->bufOffsetInFile;
    }
else
    {
    lf->lineStart = lf->lineEnd = lf->bytesInBuf = 0;
    if ((lf->bufOffsetInFile = lseek(lf->fd, offset, whence)) == -1)
        errnoAbort("Couldn't lineFileSeek %s", lf->fileName);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileSetMetaDataOutput	(	struct lineFile *	lf,
		FILE *	f
	)

Definition at line 78 of file linefile.c.

References AllocVar, metaOutput::metaFile, lineFile::metaOutput, metaOutput::next, and slAddHead.

Referenced by chainReadAllWithMeta(), and pslxFileOpenWithMetaConfig().

{
struct metaOutput *meta = NULL;
if (lf == NULL)
    return;
AllocVar(meta);
meta->next = NULL;
meta->metaFile = f;
slAddHead(&lf->metaOutput, meta);
}

Here is the caller graph for this function:

void lineFileSetUniqueMetaData

(

struct lineFile *

lf

)

Definition at line 91 of file linefile.c.

References hashNew, lineFile::isMetaUnique, lineFile::metaLines, and TRUE.

Referenced by pslxFileOpenWithMetaConfig().

{
lf->isMetaUnique = TRUE;
lf->metaLines = hashNew(8);
}

Here is the caller graph for this function:

void lineFileShort

(

struct lineFile *

lf

)

Definition at line 545 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFile::lineIx.

{
errAbort("Short line %d of %s", lf->lineIx, lf->fileName);
}

Here is the call graph for this function:

void lineFileSkip	(	struct lineFile *	lf,
		int	lineCount
	)

Definition at line 703 of file linefile.c.

References errAbort(), lineFile::fileName, and lineFileNext().

{
int i, lineSize;
char *line;

for (i=0; i<lineCount; ++i)
    {
    if (!lineFileNext(lf, &line, &lineSize))
        errAbort("Premature end of file in %s", lf->fileName);
    }
}

Here is the call graph for this function:

char* lineFileSkipToLineStartingWith	(	struct lineFile *	lf,
		char *	start,
		int	maxCount
	)

Definition at line 716 of file linefile.c.

References lineFileNext(), and startsWith().

Referenced by hpfNext(), and needLineStartingWith().

{
char *line;
while (lineFileNext(lf, &line, NULL) && --maxCount >= 0)
    {
    if (startsWith(start, line))
        return line;
    }
return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct dyString* lineFileSlurpHttpBody	(	struct lineFile *	lf,
		boolean	chunked,
		int	contentLength
	)			[read]

Definition at line 806 of file linefile.c.

References dyStringAppendC(), dyStringAppendN(), dyStringClear, lineFile::fileName, lineFileNext(), lineFileReuse(), newDyString(), nextWord(), startsWith(), dyString::string, dyString::stringSize, and warn().

Referenced by netHttpGetMultiple().

{
  struct dyString *body = newDyString(64*1024);
  char *line;
  int lineSize;

  dyStringClear(body);
  if (chunked)
    {
      /* Handle "Transfer-Encoding: chunked" body */
      /* Procedure from RFC2068 section 19.4.6 */
      char *csword;
      unsigned chunkSize = 0;
      unsigned size;
      do
        {
          /* Read line that has chunk size (in hex) as first word. */
          if (lineFileNext(lf, &line, NULL))
            csword = nextWord(&line);
          else break;
          if (sscanf(csword, "%x", &chunkSize) < 1)
            {
              warn("%s: chunked transfer-encoding chunk size parse error.\n",
                   lf->fileName);
              break;
            }
          /* If chunk size is 0, read in a blank line & then we're done. */
          if (chunkSize == 0)
            {
              lineFileNext(lf, &line, NULL);
              if (line == NULL || (line[0] != '\r' && line[0] != 0))
                warn("%s: chunked transfer-encoding: expected blank line, got %s\n",
                     lf->fileName, line);
              
              break;
            }
          /* Read (and save) lines until we have read in chunk. */
          for (size = 0;  size < chunkSize;  size += lineSize)
            {
              if (! lineFileNext(lf, &line, &lineSize))
                break;
              dyStringAppendN(body, line, lineSize-1);
              dyStringAppendC(body, '\n');
            }
          /* Read blank line - or extra CRLF inserted in the middle of the 
           * current line, in which case we need to trim it. */
          if (size > chunkSize)
            {
              body->stringSize -= (size - chunkSize);
              body->string[body->stringSize] = 0;
            }
          else if (size == chunkSize)
            {
              lineFileNext(lf, &line, NULL);
              if (line == NULL || (line[0] != '\r' && line[0] != 0))
                warn("%s: chunked transfer-encoding: expected blank line, got %s\n",
                     lf->fileName, line);
            }
        } while (chunkSize > 0);
      /* Try to read in next line.  If it's an HTTP header, put it back. */
      /* If there is a next line but it's not an HTTP header, it's a footer. */
      if (lineFileNext(lf, &line, NULL))
        {
          if (startsWith("HTTP/", line))
            lineFileReuse(lf);
          else
            {
              /* Got a footer -- keep reading until blank line */
              warn("%s: chunked transfer-encoding: got footer %s, discarding it.\n",
                   lf->fileName, line);
              while (lineFileNext(lf, &line, NULL))
                {
                  if ((line[0] == '\r' && line[1] == 0) || line[0] == 0)
                    break;
                  warn("discarding footer line: %s\n", line);
                }
            }
        }
    }
  else if (contentLength >= 0)
    {
      /* Read in known length */
      int size;
      for (size = 0;  size < contentLength;  size += lineSize)
        {
          if (! lineFileNext(lf, &line, &lineSize))
            break;
          dyStringAppendN(body, line, lineSize-1);
          dyStringAppendC(body, '\n');
        }
    }
  else
    {
      /* Read in to end of file (assume it's not a persistent connection) */
      while (lineFileNext(lf, &line, &lineSize))
        {
          dyStringAppendN(body, line, lineSize-1);
          dyStringAppendC(body, '\n');
        }
    }

  return(body);
} /* lineFileSlurpHttpBody */

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* lineFileStdin

(

bool

zTerm

)

[read]

Definition at line 205 of file linefile.c.

References lineFileAttach().

Referenced by lineFileMayOpen().

{
return lineFileAttach("stdin", zTerm, fileno(stdin));
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileUnexpectedEnd

(

struct lineFile *

lf

)

Definition at line 481 of file linefile.c.

References errAbort(), and lineFile::fileName.

Referenced by lineFileNeedNext(), and readTaggedNumLine().

{
errAbort("Unexpected end of file in %s", lf->fileName);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void lineFileVaAbort	(	struct lineFile *	lf,
		char *	format,
		va_list	args
	)

Definition at line 462 of file linefile.c.

References dyStringFree, dyStringNew, dyStringPrintf(), dyStringVaPrintf(), errAbort(), lineFile::fileName, lineFile::lineIx, and dyString::string.

Referenced by lineFileAbort().

{
struct dyString *dy = dyStringNew(0);
dyStringPrintf(dy,  "Error line %d of %s: ", lf->lineIx, lf->fileName);
dyStringVaPrintf(dy, format, args);
errAbort("%s", dy->string);
dyStringFree(&dy);
}

Here is the call graph for this function:

Here is the caller graph for this function:

static void metaDataAdd	(	struct lineFile *	lf,
		char *	line
	)			[static]

Definition at line 51 of file linefile.c.

References hashAdd(), hashLookup(), lineFile::isMetaUnique, metaOutput::metaFile, lineFile::metaLines, lineFile::metaOutput, and metaOutput::next.

Referenced by lineFileNext().

{
struct metaOutput *meta = NULL;

if (lf->isMetaUnique)
    {
    /* suppress repetition of comments */
    if (hashLookup(lf->metaLines, line))
        {
        return;
        }
    hashAdd(lf->metaLines, line, NULL);
    }
for (meta = lf->metaOutput ; meta != NULL ; meta = meta->next)
    if (line != NULL && meta->metaFile != NULL)
        fprintf(meta->metaFile,"%s\n", line);
}

Here is the call graph for this function:

Here is the caller graph for this function:

static void metaDataFree

(

struct lineFile *

lf

)

[static]

Definition at line 71 of file linefile.c.

References freeHash(), lineFile::isMetaUnique, and lineFile::metaLines.

Referenced by lineFileClose().

{
if (lf->isMetaUnique && lf->metaLines)
    freeHash(&lf->metaLines);
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

Variable Documentation

char const rcsid[] = "$Id: linefile.c,v 1.54 2007/03/23 08:12:42 kent Exp $" [static]

Definition at line 16 of file linefile.c.

---