lib/net.c File Reference

#include "common.h"
#include <signal.h>
#include <errno.h>
#include <string.h>
#include "internet.h"
#include "errabort.h"
#include "hash.h"
#include "net.h"
#include "linefile.h"
#include "base64.h"

Include dependency graph for net.c:

Go to the source code of this file.

Functions
static int	netStreamSocket ()
int	netConnect (char *hostName, int port)
int	netMustConnect (char *hostName, int port)
int	netMustConnectTo (char *hostName, char *portName)
int	netAcceptingSocketFrom (int port, int queueSize, char *host)
int	netAcceptingSocket (int port, int queueSize)
int	netAccept (int sd)
int	netAcceptFrom (int acceptor, unsigned char subnet[4])
FILE *	netFileFromSocket (int socket)
void	netBlockBrokenPipes ()
int	netReadAll (int sd, void *vBuf, size_t size)
int	netMustReadAll (int sd, void *vBuf, size_t size)
static void	notGoodSubnet (char *sns)
void	netParseSubnet (char *in, unsigned char out[4])
void	netParseUrl (char *url, struct netParsedUrl *parsed)
static boolean	readReadyWait (int sd, int microseconds)
dyString *	sendFtpCommand (int sd, char *cmd, boolean seeResult, boolean noTimeoutError)
int	parsePasvPort (char *rs)
int	netGetOpenFtp (char *url)
int	netHttpConnect (char *url, char *method, char *protocol, char *agent)
int	netOpenHttpExt (char *url, char *method, boolean end)
static int	netGetOpenHttp (char *url)
int	netUrlHead (char *url, struct hash *hash)
int	netUrlOpen (char *url)
dyString *	netSlurpFile (int sd)
dyString *	netSlurpUrl (char *url)
int	netSkipHttpHeaderLinesCatch (int sd, char *url)
boolean	netSkipHttpHeaderLines (int sd, char *url)
lineFile *	netLineFileMayOpenCatchError (char *url)
lineFile *	netLineFileMayOpen (char *url)
lineFile *	netLineFileOpen (char *url)
boolean	netSendString (int sd, char *s)
boolean	netSendLongString (int sd, char *s)
boolean	netSendHugeString (int sd, char *s)
char *	netGetString (int sd, char buf[256])
char *	netGetLongString (int sd)
char *	netGetHugeString (int sd)
char *	netRecieveString (int sd, char buf[256])
char *	netRecieveLongString (int sd)
char *	netRecieveHugeString (int sd)
lineFile *	netHttpLineFileMayOpen (char *url, struct netParsedUrl **npu)
void	netHttpGet (struct lineFile *lf, struct netParsedUrl *npu, boolean keepAlive)
int	netHttpGetMultiple (char *url, struct slName *queries, void *userData, void(*responseCB)(void *userData, char *req, char *hdr, struct dyString *body))
Variables
static char const	rcsid [] = "$Id: net.c,v 1.55 2006/10/06 22:17:38 hiram Exp $"
int	errno
static boolean	plumberInstalled = FALSE

---

Function Documentation

int netAccept

(

int

sd

)

Definition at line 108 of file net.c.

{
socklen_t fromLen;
return accept(sd, NULL, &fromLen);
}

int netAcceptFrom	(	int	acceptor,
		unsigned char	subnet[4]
	)

Definition at line 115 of file net.c.

References internetIpInSubnet(), internetUnpackIp(), and ZeroVar.

{
struct sockaddr_in sai;         /* Some system socket info. */
ZeroVar(&sai);
sai.sin_family = AF_INET;
for (;;)
    {
    socklen_t addrSize = sizeof(sai);
    int sd = accept(acceptor, (struct sockaddr *)&sai, &addrSize);
    if (sd >= 0)
        {
        if (subnet == NULL)
            return sd;
        else
            {
            unsigned char unpacked[4]; 
            internetUnpackIp(ntohl(sai.sin_addr.s_addr), unpacked);
            if (internetIpInSubnet(unpacked, subnet))
                {
                return sd;
                }
            else
                {
                close(sd);
                }
            }
        }
    }
}

Here is the call graph for this function:

int netAcceptingSocket	(	int	port,
		int	queueSize
	)

Definition at line 101 of file net.c.

References netAcceptingSocketFrom().

Referenced by startServer().

{
return netAcceptingSocketFrom(port, queueSize, NULL);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netAcceptingSocketFrom	(	int	port,
		int	queueSize,
		char *	host
	)

Definition at line 75 of file net.c.

References errno, internetFillInAddress(), netBlockBrokenPipes(), netStreamSocket(), and warn().

Referenced by netAcceptingSocket().

{
struct sockaddr_in sai;
int sd;
int flag = 1;

netBlockBrokenPipes();
if ((sd = netStreamSocket()) < 0)
    return sd;
if (!internetFillInAddress(host, port, &sai))
    return -1;
if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(int)))
    return -1;
if (bind(sd, (struct sockaddr*)&sai, sizeof(sai)) == -1)
    {
    warn("Couldn't bind socket to %d: %s", port, strerror(errno));
    close(sd);
    return -1;
    }
listen(sd, queueSize);
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void netBlockBrokenPipes

(

)

Definition at line 164 of file net.c.

References plumberInstalled, and TRUE.

Referenced by netAcceptingSocketFrom(), netReadAll(), and startServer().

{
if (!plumberInstalled)
    {
    signal(SIGPIPE, SIG_IGN);       /* Block broken pipe signals. */
    plumberInstalled = TRUE;
    }
}

Here is the caller graph for this function:

int netConnect	(	char *	hostName,
		int	port
	)

Definition at line 34 of file net.c.

References internetFillInAddress(), netStreamSocket(), and warn().

Referenced by gfConnect(), netHttpLineFileMayOpen(), and netMustConnect().

{
int sd, err;
struct sockaddr_in sai;         /* Some system socket info. */

if (hostName == NULL)
    {
    warn("NULL hostName in netConnect");
    return -1;
    }
if (!internetFillInAddress(hostName, port, &sai))
    return -1;
if ((sd = netStreamSocket()) < 0)
    return sd;
if ((err = connect(sd, (struct sockaddr*)&sai, sizeof(sai))) < 0)
   {
   warn("Couldn't connect to %s %d", hostName, port);
   close(sd);
   return err;
   }
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

FILE* netFileFromSocket

(

int

socket

)

Definition at line 149 of file net.c.

References errnoAbort().

{
FILE *f;
if ((socket = dup(socket)) < 0)
   errnoAbort("Couldn't dupe socket in netFileFromSocket");
f = fdopen(socket, "r+");
if (f == NULL)
   errnoAbort("Couldn't fdopen socket in netFileFromSocket");
return f;
}

Here is the call graph for this function:

char* netGetHugeString

(

int

sd

)

Definition at line 1038 of file net.c.

References needMem(), netReadAll(), UBYTE, and warn().

Referenced by netRecieveHugeString().

{
UBYTE b[4];
char *s = NULL;
unsigned long length = 0;
int sz, i;
sz = netReadAll(sd, b, 4);
if (sz == 0)
    return NULL;
if (sz < 4)
    {
    warn("Couldn't read huge string length");
    return NULL;
    }
for (i=0; i<4; ++i)
    {
    length <<= 8;
    length += b[i];
    }
s = needMem(length+1);
if (length > 0)
    {
    if (netReadAll(sd, s, length) < 0)
        {
        warn("Couldn't read huge string body");
        return NULL;
        }
    }
s[length] = 0;
return s;
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* netGetLongString

(

int

sd

)

Definition at line 1009 of file net.c.

References needMem(), netReadAll(), UBYTE, and warn().

Referenced by netRecieveLongString(), and queryServer().

{
UBYTE b[2];
char *s = NULL;
int length = 0;
int sz;
b[0] = b[1] = 0;
sz = netReadAll(sd, b, 2);
if (sz == 0)
    return NULL;
if (sz < 0)
    {
    warn("Couldn't read long string length");
    return NULL;
    }
length = (b[0]<<8) + b[1];
s = needMem(length+1);
if (length > 0)
    if (netReadAll(sd, s, length) < 0)
        {
        warn("Couldn't read long string body");
        return NULL;
        }
s[length] = 0;
return s;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netGetOpenFtp

(

char *

url

)

Definition at line 464 of file net.c.

References dyStringFree, errAbort(), FALSE, netParsedUrl::file, netParsedUrl::host, netMustConnect(), netParseUrl(), parsePasvPort(), netParsedUrl::password, netParsedUrl::port, netParsedUrl::protocol, readReadyWait(), safef(), sameString, sendFtpCommand(), dyString::string, TRUE, and netParsedUrl::user.

Referenced by netUrlOpen().

{
struct netParsedUrl npu;
struct dyString *rs = NULL;
int sd;
long timeOut = 1000000; /* wait in microsec */
char cmd[256];

/* Parse the URL and connect. */
netParseUrl(url, &npu);
if (!sameString(npu.protocol, "ftp"))
    errAbort("Sorry, can only netOpen ftp's currently");
sd = netMustConnect(npu.host, atoi(npu.port));

/* Ask remote ftp server for a file. */

/* don't send a command, just read the welcome msg */
if (readReadyWait(sd, timeOut))
    sendFtpCommand(sd, "", FALSE, FALSE);

safef(cmd,sizeof(cmd),"USER %s\r\n",npu.user);
sendFtpCommand(sd, cmd, FALSE, FALSE);

safef(cmd,sizeof(cmd),"PASS %s\r\n",npu.password);
sendFtpCommand(sd, cmd, FALSE, FALSE);

sendFtpCommand(sd, "TYPE I\r\n", FALSE, FALSE);
/* 200 Type set to I */
/* (send the data as binary, so can support compressed files) */

rs = sendFtpCommand(sd, "PASV\r\n", TRUE, FALSE);
/* 227 Entering Passive Mode (128,231,210,81,222,250) */

safef(cmd,sizeof(cmd),"RETR %s\r\n", npu.file);
/* we can't wait for reply because 
   we need to start the next fetch connect 
   but then if there is an error e.g. missing file,
   then we don't see the err msg because we
   already closed the port and are waiting.
   And our timeout is long - indefinitely so?
*/
sendFtpCommand(sd, cmd, FALSE, TRUE);  

close(sd);

sd = netMustConnect(npu.host, parsePasvPort(rs->string));

/* Clean up and return handle. */
dyStringFree(&rs);
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static int netGetOpenHttp

(

char *

url

)

[static]

Definition at line 573 of file net.c.

References netOpenHttpExt(), and TRUE.

Referenced by netUrlOpen().

{
return netOpenHttpExt(url, "GET", TRUE);
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* netGetString	(	int	sd,
		char	buf[256]
	)

Definition at line 980 of file net.c.

References netReadAll(), UBYTE, and warn().

Referenced by getFileList(), gfPcrGetRanges(), netRecieveString(), pcrServer(), queryServer(), and statusServer().

{
static char sbuf[256];
UBYTE len = 0;
int length;
int sz;
if (buf == NULL) buf = sbuf;
sz = netReadAll(sd, &len, 1);
if (sz == 0)
    return NULL;
if (sz < 0)
    {
    warn("Couldn't read string length");
    return NULL;
    }
length = len;
if (length > 0)
    if (netReadAll(sd, buf, length) < 0)
        {
        warn("Couldn't read string body");
        return NULL;
        }
buf[length] = 0;
return buf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netHttpConnect	(	char *	url,
		char *	method,
		char *	protocol,
		char *	agent
	)

Definition at line 517 of file net.c.

References base64Encode(), dyStringAppend(), dyStringFree, dyStringPrintf(), errAbort(), netParsedUrl::file, freez(), netParsedUrl::host, netMustConnect(), netParseUrl(), newDyString(), netParsedUrl::password, netParsedUrl::port, netParsedUrl::protocol, safef(), sameString, dyString::string, dyString::stringSize, and netParsedUrl::user.

Referenced by netOpenHttpExt().

{
struct netParsedUrl npu;
struct dyString *dy = newDyString(512);
int sd;

/* Parse the URL and connect. */
netParseUrl(url, &npu);
if (!sameString(npu.protocol, "http"))
    errAbort("Sorry, can only netOpen http's currently");
sd = netMustConnect(npu.host, atoi(npu.port));

/* Ask remote server for a file. */
dyStringPrintf(dy, "%s %s %s\r\n", method, npu.file, protocol);
dyStringPrintf(dy, "User-Agent: %s\r\n", agent);
/* do not need the 80 since it is the default */
if (sameString("80",npu.port))
    dyStringPrintf(dy, "Host: %s\r\n", npu.host);
else
    dyStringPrintf(dy, "Host: %s:%s\r\n", npu.host, npu.port);
if (!sameString(npu.user,""))
    {
    char up[256];
    char *b64up = NULL;
    safef(up, sizeof(up), "%s:%s", npu.user, npu.password);
    b64up = base64Encode(up, strlen(up));
    dyStringPrintf(dy, "Authorization: Basic %s\r\n", b64up);
    freez(&b64up);
    }
dyStringAppend(dy, "Accept: */*\r\n");
write(sd, dy->string, dy->stringSize);

/* Clean up and return handle. */
dyStringFree(&dy);
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void netHttpGet	(	struct lineFile *	lf,
		struct netParsedUrl *	npu,
		boolean	keepAlive
	)

Definition at line 1125 of file net.c.

References base64Encode(), dyStringAppend(), dyStringFree, dyStringPrintf(), lineFile::fd, netParsedUrl::file, freez(), netParsedUrl::host, newDyString(), netParsedUrl::password, netParsedUrl::port, safef(), sameString, dyString::string, dyString::stringSize, and netParsedUrl::user.

Referenced by netHttpGetMultiple().

{
struct dyString *dy = newDyString(512);

/* Ask remote server for the file/query. */
dyStringPrintf(dy, "GET %s HTTP/1.1\r\n", npu->file);
dyStringPrintf(dy, "User-Agent: genome.ucsc.edu/net.c\r\n");
dyStringPrintf(dy, "Host: %s:%s\r\n", npu->host, npu->port);
if (!sameString(npu->user,""))
    {
    char up[256];
    char *b64up = NULL;
    safef(up,sizeof(up), "%s:%s", npu->user, npu->password);
    b64up = base64Encode(up, strlen(up));
    dyStringPrintf(dy, "Authorization: Basic %s\r\n", b64up);
    freez(&b64up);
    }
dyStringAppend(dy, "Accept: */*\r\n");
if (keepAlive)
  {
    dyStringAppend(dy, "Connection: Keep-Alive\r\n");
    dyStringAppend(dy, "Connection: Persist\r\n");
  }
else
    dyStringAppend(dy, "Connection: close\r\n");
dyStringAppend(dy, "\r\n");
write(lf->fd, dy->string, dy->stringSize);
/* Clean up. */
dyStringFree(&dy);
} /* netHttpGet */

Here is the call graph for this function:

Here is the caller graph for this function:

int netHttpGetMultiple	(	char *	url,
		struct slName *	queries,
		void *	userData,
		void(*)(void *userData, char *req, char *hdr, struct dyString *body)	responseCB
	)

Definition at line 1158 of file net.c.

References cloneString(), dyStringAppend(), dyStringClear, FALSE, netParsedUrl::file, lineFileParseHttpHeader(), lineFileSlurpHttpBody(), slName::name, netHttpGet(), netHttpLineFileMayOpen(), newDyString(), slName::next, and TRUE.

{
  struct slName *qStart;
  struct slName *qPtr;
  struct lineFile *lf;
  struct netParsedUrl *npu;
  struct dyString *dyQ    = newDyString(512);
  struct dyString *body;
  char *base;
  char *hdr;
  int qCount;
  int qTotal;
  int numParseFailures;
  int contentLength;
  boolean chunked;
  boolean done;
  boolean keepAlive;

  /* Find out how many queries we'll need to do so we know how many times 
   * it's OK to run into end of file in case server ignores Keep-Alive. */
  qTotal = 0;
  for (qPtr = queries;  qPtr != NULL;  qPtr = qPtr->next)
    {
      qTotal++;
    }

  done = FALSE;
  qCount = 0;
  numParseFailures = 0;
  qStart = queries;
  while ((! done) && (qStart != NULL))
    {
      lf = netHttpLineFileMayOpen(url, &npu);
      if (lf == NULL)
        {
          done = TRUE;
          break;
        }
      base = cloneString(npu->file);
      /* Send all remaining requests with keep-alive. */
      for (qPtr = qStart;  qPtr != NULL;  qPtr = qPtr->next)
        {
          dyStringClear(dyQ);
          dyStringAppend(dyQ, base);
          dyStringAppend(dyQ, qPtr->name);
          strcpy(npu->file, dyQ->string);
          keepAlive = (qPtr->next == NULL) ? FALSE : TRUE;
          netHttpGet(lf, npu, keepAlive);
        }
      /* Get as many responses as we can; call responseCB() and 
       * advance qStart for each. */
      for (qPtr = qStart;  qPtr != NULL;  qPtr = qPtr->next)
        {
          if (lineFileParseHttpHeader(lf, &hdr, &chunked, &contentLength))
            {
              body = lineFileSlurpHttpBody(lf, chunked, contentLength);
              dyStringClear(dyQ);
              dyStringAppend(dyQ, base);
              dyStringAppend(dyQ, qPtr->name);
              responseCB(userData, dyQ->string, hdr, body);
              qStart = qStart->next;
              qCount++;
            }
          else
            {
              if (numParseFailures++ > qTotal) {
                done = TRUE;
              }
              break;
            }
        }
    }

  return qCount;
} /* netHttpMultipleQueries */

Here is the call graph for this function:

struct lineFile* netHttpLineFileMayOpen	(	char *	url,
		struct netParsedUrl **	npu
	)			[read]

Definition at line 1104 of file net.c.

References AllocVar, errAbort(), lineFileAttach(), netConnect(), netParseUrl(), sameString, and TRUE.

Referenced by netHttpGetMultiple().

{
int sd;
struct lineFile *lf;

/* Parse the URL and try to connect. */
AllocVar(*npu);
netParseUrl(url, *npu);
if (!sameString((*npu)->protocol, "http"))
    errAbort("Sorry, can only netOpen http's currently");
sd = netConnect((*npu)->host, atoi((*npu)->port));
if (sd < 0)
    return NULL;

/* Return handle. */
lf = lineFileAttach(url, TRUE, sd);
return lf;
} /* netHttpLineFileMayOpen */

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* netLineFileMayOpen

(

char *

url

)

[read]

Definition at line 861 of file net.c.

References endsWith(), lineFileAttach(), lineFileDecompressFd(), netSkipHttpHeaderLines(), netUrlOpen(), startsWith(), TRUE, and warn().

Referenced by netLineFileOpen().

{
int sd = netUrlOpen(url);
if (sd < 0)
    {
    warn("Couldn't open %s", url);
    return NULL;
    }
else
    {
    struct lineFile *lf = NULL;
    if (startsWith("http://",url))
        {
        if (!netSkipHttpHeaderLines(sd, url))
            return NULL;     /* url needed only for err msgs*/
        }
    if (endsWith(url, ".gz") ||
        endsWith(url, ".Z")  ||
        endsWith(url, ".bz2"))
        {
        lf = lineFileDecompressFd(url, TRUE, sd);
           /* url needed only for compress type determination */
        }
    else
        {
        lf = lineFileAttach(url, TRUE, sd);
        }
    return lf;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lineFile* netLineFileMayOpenCatchError

(

char *

url

)

[read]

Definition at line 827 of file net.c.

References endsWith(), lineFileAttach(), lineFileDecompressFd(), netSkipHttpHeaderLinesCatch(), netUrlOpen(), startsWith(), TRUE, and warn().

{
int sd = netUrlOpen(url);
if (sd < 0)
    {
    warn("Couldn't open %s", url);
    return NULL;
    }
else
    {
    struct lineFile *lf = NULL;
    if (startsWith("http://",url))
        {
        if (netSkipHttpHeaderLinesCatch(sd, url) != 0)
            return NULL;     /* url needed only for err msgs*/
        }
    if (endsWith(url, ".gz") ||
        endsWith(url, ".Z")  ||
        endsWith(url, ".bz2"))
        {
        lf = lineFileDecompressFd(url, TRUE, sd);
           /* url needed only for compress type determination */
        }
    else
        {
        lf = lineFileAttach(url, TRUE, sd);
        }
    return lf;
    }
}

Here is the call graph for this function:

struct lineFile* netLineFileOpen

(

char *

url

)

[read]

Definition at line 895 of file net.c.

References netLineFileMayOpen(), and noWarnAbort().

{
struct lineFile *lf = netLineFileMayOpen(url);
if (lf == NULL)
    noWarnAbort();
return lf;
}

Here is the call graph for this function:

int netMustConnect	(	char *	hostName,
		int	port
	)

Definition at line 58 of file net.c.

References netConnect(), and noWarnAbort().

Referenced by netGetOpenFtp(), netHttpConnect(), and netMustConnectTo().

{
int sd = netConnect(hostName, port);
if (sd < 0)
   noWarnAbort();
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netMustConnectTo	(	char *	hostName,
		char *	portName
	)

Definition at line 67 of file net.c.

References errAbort(), and netMustConnect().

Referenced by getFileList(), pcrServer(), queryServer(), statusServer(), and stopServer().

{
if (!isdigit(portName[0]))
    errAbort("netConnectTo: ports must be numerical, not %s", portName);
return netMustConnect(hostName, atoi(portName));
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netMustReadAll	(	int	sd,
		void *	vBuf,
		size_t	size
	)

Definition at line 196 of file net.c.

References errnoAbort(), and netReadAll().

{
int ret = netReadAll(sd, vBuf, size);
if (ret < 0)
    errnoAbort("Couldn't finish netReadAll");
return ret;
}

Here is the call graph for this function:

int netOpenHttpExt	(	char *	url,
		char *	method,
		boolean	end
	)

Definition at line 562 of file net.c.

References netHttpConnect().

Referenced by htmlPageFromForm(), htmlSlurpWithCookies(), netGetOpenHttp(), and netUrlHead().

{
int sd =  netHttpConnect(url, method, "HTTP/1.0", "genome.ucsc.edu/net.c");
if (end)
    write(sd, "\r\n", 2);
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void netParseSubnet	(	char *	in,
		unsigned char	out[4]
	)

Definition at line 213 of file net.c.

References ArraySize, chopString(), freez(), and notGoodSubnet().

{
out[0] = out[1] = out[2] = out[3] = 255;
if (in != NULL)
    {
    char *snsCopy = strdup(in);
    char *words[5];
    int wordCount, i;
    wordCount = chopString(snsCopy, ".", words, ArraySize(words));
    if (wordCount > 3 || wordCount < 1)
        notGoodSubnet(in);
    for (i=0; i<wordCount; ++i)
        {
        char *s = words[i];
        int x;
        if (!isdigit(s[0]))
            notGoodSubnet(in);
        x = atoi(s);
        if (x > 255)
            notGoodSubnet(in);
        out[i] = x;
        }
    freez(&snsCopy);
    }
}

Here is the call graph for this function:

void netParseUrl	(	char *	url,
		struct netParsedUrl *	parsed
	)

Definition at line 241 of file net.c.

References errAbort(), netParsedUrl::file, freeMem(), netParsedUrl::host, netParsedUrl::password, netParsedUrl::port, netParsedUrl::protocol, replaceChars(), sameWord, stringIn, tolowers(), trimSpaces(), and netParsedUrl::user.

Referenced by netGetOpenFtp(), netHttpConnect(), and netHttpLineFileMayOpen().

                                                             :
 *   http://user:password@hostName:port/file
 * This is set up so that the http:// and the port are optional. 
 */
{
char *s, *t, *u, *v, *w;
char buf[1024];

/* Make local copy of URL. */
if (strlen(url) >= sizeof(buf))
    errAbort("Url too long: '%s'", url);
strcpy(buf, url);
url = buf;

/* Find out protocol - default to http. */
s = trimSpaces(url);
s = stringIn("://", url);
if (s == NULL)
    {
    strcpy(parsed->protocol, "http");
    s = url;
    }
else
    {
    *s = 0;
    tolowers(url);
    strncpy(parsed->protocol, url, sizeof(parsed->protocol));
    s += 3;
    }

/* Split off file part. */
u = strchr(s, '/');
if (u == NULL)
   strcpy(parsed->file, "/");
else
   {
   /* need to encode spaces, but not ! other characters */
   char *t=replaceChars(u," ","%20");
   strncpy(parsed->file, t, sizeof(parsed->file));
   freeMem(t);
   *u = 0;
   }


/* Split off user part */
v = strchr(s, '@');
if (v == NULL)
   {
   if (sameWord(parsed->protocol,"http"))
      {
      strcpy(parsed->user, "");
      strcpy(parsed->password, "");
      }
   if (sameWord(parsed->protocol,"ftp"))
      {
      strcpy(parsed->user, "anonymous");
      strcpy(parsed->password, "x@genome.ucsc.edu");
      }
   }
else
   {
   *v = 0;
   /* split off password part */
   w = strchr(s, ':');
   if (w == NULL)
      {
      strncpy(parsed->user, s, sizeof(parsed->user));
      strcpy(parsed->password, "");
      }
   else
      {
      *w = 0;
      strncpy(parsed->user, s, sizeof(parsed->user));
      strncpy(parsed->password, w+1, sizeof(parsed->password));
      }
   s = v+1;
   }


/* Save port if it's there.  If not default to 80. */
t = strchr(s, ':');
if (t == NULL)
   {
   if (sameWord(parsed->protocol,"http"))
      strcpy(parsed->port, "80");
   if (sameWord(parsed->protocol,"ftp"))
      strcpy(parsed->port, "21");
   }
else
   {
   *t++ = 0;
   if (!isdigit(t[0]))
      errAbort("Non-numeric port name %s", t);
   strncpy(parsed->port, t, sizeof(parsed->port));
   }

/* What's left is the host. */
strncpy(parsed->host, s, sizeof(parsed->host));
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netReadAll	(	int	sd,
		void *	vBuf,
		size_t	size
	)

Definition at line 174 of file net.c.

References netBlockBrokenPipes(), and plumberInstalled.

Referenced by netGetHugeString(), netGetLongString(), netGetString(), and netMustReadAll().

{
char *buf = vBuf;
size_t totalRead = 0;
int oneRead;

if (!plumberInstalled)
    netBlockBrokenPipes();
while (totalRead < size)
    {
    oneRead = read(sd, buf + totalRead, size - totalRead);
    if (oneRead < 0)
        return oneRead;
    if (oneRead == 0)
        break;
    totalRead += oneRead;
    }
return totalRead;
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* netRecieveHugeString

(

int

sd

)

Definition at line 1093 of file net.c.

References netGetHugeString(), and noWarnAbort().

{
char *s = netGetHugeString(sd);
if (s == NULL)
     noWarnAbort();   
return s;
}

Here is the call graph for this function:

char* netRecieveLongString

(

int

sd

)

Definition at line 1083 of file net.c.

References netGetLongString(), and noWarnAbort().

Referenced by getHitsFromServer().

{
char *s = netGetLongString(sd);
if (s == NULL)
     noWarnAbort();   
return s;
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* netRecieveString	(	int	sd,
		char	buf[256]
	)

Definition at line 1073 of file net.c.

References netGetString(), and noWarnAbort().

Referenced by getFileList(), gfQuerySeq(), gfQuerySeqTrans(), gfQuerySeqTransTrans(), and queryServer().

{
char *s = netGetString(sd, buf);
if (s == NULL)
     noWarnAbort();   
return s;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean netSendHugeString	(	int	sd,
		char *	s
	)

Definition at line 954 of file net.c.

References FALSE, TRUE, UBYTE, and warn().

                                 : four bytes for length. */
{
unsigned long length = strlen(s);
unsigned long l = length;
UBYTE b[4];
int i;
for (i=3; i>=0; --i)
    {
    b[i] = l & 0xff;
    l >>= 8;
    }
if (write(sd, b, 4) < 0)
    {
    warn("Couldn't send huge string to socket");
    return FALSE;
    }
if (write(sd, s, length) < 0)
    {
    warn("Couldn't send huge string to socket");
    return FALSE;
    }
return TRUE;
}

Here is the call graph for this function:

boolean netSendLongString	(	int	sd,
		char *	s
	)

Definition at line 928 of file net.c.

References FALSE, TRUE, UBYTE, and warn().

Referenced by transQuery(), and transTransQuery().

                                 : two bytes for length. */
{
unsigned length = strlen(s);
UBYTE b[2];

if (length >= 64*1024)
    {
    warn("Trying to send a string longer than 64k bytes (%d bytes)", length);
    return FALSE;
    }
b[0] = (length>>8);
b[1] = (length&0xff);
if (write(sd, b, 2) < 0)
    {
    warn("Couldn't send long string to socket");
    return FALSE;
    }
if (write(sd, s, length)<0)
    {
    warn("Couldn't send long string to socket");
    return FALSE;
    }
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean netSendString	(	int	sd,
		char *	s
	)

Definition at line 906 of file net.c.

References errAbort(), FALSE, TRUE, UBYTE, and warn().

Referenced by dnaQuery(), errorSafeCleanupMess(), pcrQuery(), startServer(), transQuery(), and transTransQuery().

{
int length = strlen(s);
UBYTE len;

if (length > 255)
    errAbort("Trying to send a string longer than 255 bytes (%d bytes)", length);
len = length;
if (write(sd, &len, 1)<0)
    {
    warn("Couldn't send string to socket");
    return FALSE;
    }
if (write(sd, s, length)<0)
    {
    warn("Couldn't send string to socket");
    return FALSE;
    }
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

boolean netSkipHttpHeaderLines	(	int	sd,
		char *	url
	)

Definition at line 731 of file net.c.

References dyStringFree, dyStringPrintf(), FALSE, newDyString(), nextWord(), sameString, sameWord, skipLeadingSpaces(), startsWith(), dyString::string, TRUE, and warn().

Referenced by netLineFileMayOpen().

{
char buf[2000];
char *line = buf;
int maxbuf = sizeof(buf);
int i=0;
char c = ' ';
int nread = 0;
struct dyString *redirectMsg = NULL;
char *sep = NULL;
char *headerName = NULL;
char *headerVal = NULL;
while(TRUE)
    {
    i = 0;
    while (TRUE)
        {
        nread = read(sd, &c, 1);  /* one char at a time, but http headers are small */
        if (nread < 0)
            return FALSE;  /* err reading descriptor */
        if (c == 10)
            break;
        if (c != 13)
            buf[i++] = c;
        if (i >= maxbuf)
            {
            warn("http header line too long > %d chars.",maxbuf);
            return FALSE;
            }
        }
    buf[i] = 0;  /* add string terminator */

    if (sameString(line,""))
        {
        break; /* End of Header found */
        }
    if (startsWith("HTTP/", line))
        {
        char *version, *code;
        version = nextWord(&line);
        code = nextWord(&line);
        if (code == NULL)
            {
            warn("Strange http header on %s\n", url);
            return FALSE;
            }
        if (startsWith("30", code) && isdigit(code[2]) && code[3] == 0)
            {
            redirectMsg = newDyString(256);
            dyStringPrintf(redirectMsg,"Your URL \"%s\" resulted in a redirect message "
                 "(HTTP status code %s %s).  <BR>\n"
                 "Sorry, redirects are not supported.",
                 url, code, line);
            }
        else if (!sameString(code, "200"))
            {
            warn("%s: %s %s\n", url, code, line);
            return FALSE;
            }
        line = buf;  /* restore it */
        }
    headerName = line;
    sep = strchr(line,':');
    if (sep)
        {
        *sep = 0;
        headerVal = skipLeadingSpaces(++sep);
        }
    else
        {
        headerVal = NULL;
        }
    if (sameWord(headerName,"Location"))
        {
        if (redirectMsg)
            {
            dyStringPrintf(redirectMsg, " Redirection location: <A HREF=\"%s\">%s</A>", 
                headerVal, headerVal);
            }
        }
    }
if (redirectMsg)
    {
    warn("%s", redirectMsg->string);
    dyStringFree(&redirectMsg);
    return FALSE;
    }           
return TRUE;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netSkipHttpHeaderLinesCatch	(	int	sd,
		char *	url
	)

Definition at line 655 of file net.c.

References nextWord(), sameString, skipLeadingSpaces(), startsWith(), and TRUE.

Referenced by netLineFileMayOpenCatchError().

{
char buf[2000];
char *line = buf;
int maxbuf = sizeof(buf);
int i=0;
char c = ' ';
int nread = 0;
char *sep = NULL;
char *headerName = NULL;
char *headerVal = NULL;
while(TRUE)
    {
    i = 0;
    while (TRUE)
        {
        nread = read(sd, &c, 1);  /* one char at a time, but http headers are small */
        if (nread < 0)
            return -3;  /* err reading descriptor */
        if (c == 10)
            break;
        if (c != 13)
            buf[i++] = c;
        if (i >= maxbuf)
            {
            return -1;
            }
        }
    buf[i] = 0;  /* add string terminator */

    if (sameString(line,""))
        {
        break; /* End of Header found */
        }
    if (startsWith("HTTP/", line))
        {
        char *version, *code;
        version = nextWord(&line);
        code = nextWord(&line);
        if (code == NULL)
            {
            return -2;
            }
        if (startsWith("30", code) && isdigit(code[2]) && code[3] == 0)
            {
            return (atoi(code));
            }
        else if (!sameString(code, "200"))
            {
            return atoi(code);
            }
        line = buf;  /* restore it */
        }
    headerName = line;
    sep = strchr(line,':');
    if (sep)
        {
        *sep = 0;
        headerVal = skipLeadingSpaces(++sep);
        }
    else
        {
        headerVal = NULL;
        }
    }
return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct dyString* netSlurpFile

(

int

sd

)

[read]

Definition at line 633 of file net.c.

References dyStringAppendN(), and newDyString().

Referenced by htmlPageFromForm(), htmlSlurpWithCookies(), and netSlurpUrl().

{
char buf[4*1024];
int readSize;
struct dyString *dy = newDyString(4*1024);

/* Slurp file into dy and return. */
while ((readSize = read(sd, buf, sizeof(buf))) > 0)
    dyStringAppendN(dy, buf, readSize);
return dy;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct dyString* netSlurpUrl

(

char *

url

)

[read]

Definition at line 646 of file net.c.

References netSlurpFile(), and netUrlOpen().

{
int sd = netUrlOpen(url);
struct dyString *dy = netSlurpFile(sd);
close(sd);
return dy;
}

Here is the call graph for this function:

static int netStreamSocket

(

)

[static]

Definition at line 23 of file net.c.

References warn().

Referenced by netAcceptingSocketFrom(), and netConnect().

{
int sd = socket(AF_INET, SOCK_STREAM, 0);
if (sd < 0)
    warn("Couldn't make AF_INET socket.");
return sd;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int netUrlHead	(	char *	url,
		struct hash *	hash
	)

Definition at line 579 of file net.c.

References cloneString(), errno, hashAdd(), lineFileAttach(), lineFileClose(), lineFileNext(), netOpenHttpExt(), nextWord(), skipLeadingSpaces(), startsWith(), and TRUE.

                                          : */
{
int sd = netOpenHttpExt(url, "HEAD", TRUE);
int status = EIO;
if (sd >= 0)
    {
    char *line, *word;
    struct lineFile *lf = lineFileAttach(url, TRUE, sd);

    if (lineFileNext(lf, &line, NULL))
        {
        if (startsWith("HTTP/", line))
            {
            word = nextWord(&line);
            word = nextWord(&line);
            if (word != NULL && isdigit(word[0]))
                {
                status = atoi(word);
                if (hash != NULL)
                    {
                    while (lineFileNext(lf, &line, NULL))
                        {
                        word = nextWord(&line);
                        if (word == NULL)
                            break;
                        hashAdd(hash, word, cloneString(skipLeadingSpaces(line)));
                        }
                    }
                }
            }
        }
    lineFileClose(&lf);
    }
else
    status = errno;
return status;
}

Here is the call graph for this function:

int netUrlOpen

(

char *

url

)

Definition at line 620 of file net.c.

References errAbort(), netGetOpenFtp(), netGetOpenHttp(), startsWith(), and stringIn.

Referenced by netLineFileMayOpen(), netLineFileMayOpenCatchError(), and netSlurpUrl().

{
if (startsWith("http://",url) || (stringIn("://", url) == NULL))
    return netGetOpenHttp(url);
else if (startsWith("ftp://",url))
    return netGetOpenFtp(url);
else    
    errAbort("Sorry, can only netOpen http and ftp currently");
return -1;    
}

Here is the call graph for this function:

Here is the caller graph for this function:

static void notGoodSubnet

(

char *

sns

)

[static]

Definition at line 206 of file net.c.

References errAbort().

Referenced by netParseSubnet().

{
errAbort("'%s' is not a properly formatted subnet.  Subnets must consist of\n"
         "one to three dot-separated numbers between 0 and 255\n", sns);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int parsePasvPort

(

char *

rs

)

Definition at line 444 of file net.c.

References ArraySize, chopString(), errAbort(), and freez().

Referenced by netGetOpenFtp().

{
char *rsCopy = strdup(rs);
char *words[7];
int wordCount;
char *rsStart = strchr(rs,'(');
char *rsEnd = strchr(rs,')');
int result = 0;
rsStart++;
*rsEnd=0;
wordCount = chopString(rsStart, ",", words, ArraySize(words));
if (wordCount != 6)
    errAbort("PASV reply does not parse correctly");
result = atoi(words[4])*256+atoi(words[5]);    
freez(&rsCopy);
return result;
}    

Here is the call graph for this function:

Here is the caller graph for this function:

static boolean readReadyWait	(	int	sd,
		int	microseconds
	)			[static]

Definition at line 343 of file net.c.

References errno, and warn().

Referenced by getOurAck(), netGetOpenFtp(), rudpReceiveTimeOut(), and sendFtpCommand().

{
struct timeval tv;
fd_set set;
int readyCount;

for (;;)
    {
    if (microseconds > 1000000)
        {
        tv.tv_sec = microseconds/1000000;
        tv.tv_usec = microseconds%1000000;
        }
    else
        {
        tv.tv_sec = 0;
        tv.tv_usec = microseconds;
        }
    FD_ZERO(&set);
    FD_SET(sd, &set);
    readyCount = select(sd+1, &set, NULL, NULL, &tv);
    if (readyCount < 0) 
        {
        if (errno == EINTR)     /* Select interrupted, not timed out. */
            continue;
        else 
            warn("select failure in rudp: %s", strerror(errno));
        }
    else
        {
        return readyCount > 0;  /* Zero readyCount indicates time out */
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct dyString* sendFtpCommand	(	int	sd,
		char *	cmd,
		boolean	seeResult,
		boolean	noTimeoutError
	)			[read]

Definition at line 380 of file net.c.

References dyStringAppendN(), dyStringFree, endsWith(), errAbort(), newDyString(), readReadyWait(), and dyString::string.

Referenced by netGetOpenFtp().

{   
struct dyString *rs = NULL;
int reply = 0;
char buf[4*1024];
int readSize;
char *startLastLine = NULL;
long timeOut = 1000000; /* wait in microsec */

write(sd, cmd, strlen(cmd));

rs = newDyString(4*1024);
while (1)
    {
    while (1)
        {
        if (!readReadyWait(sd, timeOut))
            {
            if (!noTimeoutError)
                errAbort("ftp server response timed out > %ld microsec",timeOut);
            return rs;
            }
        if ((readSize = read(sd, buf, sizeof(buf))) == 0)
            break;

        dyStringAppendN(rs, buf, readSize);
        if (endsWith(rs->string,"\n"))
            break;
        }
        
    /* find the start of the last line in the buffer */
    startLastLine = rs->string+strlen(rs->string)-1;
    if (startLastLine >= rs->string)
        if (*startLastLine == '\n') 
            --startLastLine;
    while ((startLastLine >= rs->string) && (*startLastLine != '\n'))
        --startLastLine;
    ++startLastLine;
        
    if (strlen(startLastLine)>4)
      if (
        isdigit(startLastLine[0]) &&
        isdigit(startLastLine[1]) &&
        isdigit(startLastLine[2]) &&
        startLastLine[3]==' ')
        break;
        
    /* must be some text info we can't use, ignore it till we get status code */

    }

reply = atoi(startLastLine);

if ((reply < 200) || (reply > 399))
    errAbort("ftp server error on cmd=[%s] response=[%s]\n",cmd,rs->string);
    
if (!seeResult) dyStringFree(&rs);

return rs;
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

Variable Documentation

int errno

Referenced by copyFile(), errnoAbort(), errnoWarn(), internetDottedQuadToIp(), internetIpToDottedQuad(), lineFileOpen(), makeDir(), mustOpen(), netAcceptingSocketFrom(), netUrlHead(), nibOpenVerify(), readReadyWait(), rudpOpen(), rudpOpenBound(), rudpReceiveTimeOut(), rudpSend(), startServer(), and wormGeneToOrfNames().

boolean plumberInstalled = FALSE [static]

Definition at line 162 of file net.c.

Referenced by netBlockBrokenPipes(), and netReadAll().

char const rcsid[] = "$Id: net.c,v 1.55 2006/10/06 22:17:38 hiram Exp $" [static]

Definition at line 17 of file net.c.

---