include/readindex.h File Reference

#include <stdio.h>

Include dependency graph for readindex.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	index_scanner
struct	lo_info
struct	listinfo
Functions
index_scanner *	open_index (char *index_prefix)
listinfo *	get_next_list (struct index_scanner *iscn, struct listinfo *info)
listinfo *	get_list_at (struct index_scanner *iscn, struct listinfo *info, long offset)
listinfo *	get_next_biglist (struct index_scanner *iscn, struct listinfo *info)
lo_info *	get_next_lo_info (struct index_scanner *iscn, struct lo_info *lo_info)
void	info_free (struct listinfo *info)
index_scanner *	reset_index (struct index_scanner *scn)
index_scanner *	close_index (struct index_scanner *scn)

---

Detailed Description

Functions designed for sequentially extracting postings lists from lucy index files, primarily designed for use with the spex indices from the deco algorithm.

Yaniv Bernstein 2004

Definition in file readindex.h.

---

Function Documentation

struct index_scanner* close_index

(

struct index_scanner *

scn

)

[read]

Close all the file pointers to the index

Definition at line 166 of file readindex.c.

References index_scanner::idx_file.

Referenced by cluster_clusterSequences().

{
   fclose(scn->idx_file);

   free(scn);

   return scn;
}

Here is the caller graph for this function:

struct listinfo* get_list_at	(	struct index_scanner *	iscn,
		struct listinfo *	info,
		long	offset
	)			[read]

Definition at line 65 of file readindex.c.

References listinfo::doc_count, listinfo::doc_numbers, index_scanner::idx_file, info_free(), listinfo::phrase_frequency, listinfo::phrase_offsets, listinfo::size, vbyte_read(), vec_free(), vec_getvbyte(), vec_init(), and vec::vector.

Referenced by get_next_biglist(), and get_next_list().

{
   struct vec *vector;
   unsigned long num, last_doc = -1;
   int c, d;
   FILE *idx = iscn->idx_file;
   unsigned long veclen,
                 numdocs,
                 occurs,
                 last_offset;

   if (offset >= 0)
      fseek(idx, offset, SEEK_SET);

   /* There is nothing left to read; free structures */
   if (!( vbyte_read(idx, &numdocs)
            && vbyte_read(idx, &occurs)
            && vbyte_read(idx, &veclen) ))
   {
      info_free(info);
      return NULL;
   }

   /* Initialise the info structure if we weren't already given one */
   if (info == NULL)
   {
      info = malloc(sizeof(struct listinfo));
      info->doc_numbers = malloc(numdocs * sizeof(unsigned long));
      info->phrase_frequency = malloc(numdocs * sizeof(unsigned long));
      info->phrase_offsets = malloc(numdocs * sizeof(unsigned long *));
      memset(info->phrase_offsets, 0, numdocs * sizeof(unsigned long *));
      info->size = numdocs;
   }
   
   /* We may need to enlarge various aspects of the structure if it is not
    * big enough to hold the current postings list */
   if (info->size < numdocs)
   {
      info->doc_numbers = realloc(info->doc_numbers, numdocs * sizeof(unsigned long));
      info->phrase_frequency = realloc(info->phrase_frequency, numdocs * sizeof(unsigned long));
      info->phrase_offsets = realloc(info->phrase_offsets, numdocs * sizeof(unsigned long *));
      memset(info->phrase_offsets + info->size, 0, (numdocs - info->size) * sizeof(unsigned long *));
      info->size = numdocs;
   }

   info->doc_count = numdocs;

   vector = vec_init(veclen);
   vector->len = veclen;

   /* Populate the vector's block of memory from the appropriate location
    * in the vector file */
   fread(vector->vector, veclen, 1, idx);

   /* Read in the stats for each document in the postings list */
   for (c = 0; c < numdocs; c++)
   {
      /* Get the document number (stored as a d-gap) */
      vec_getvbyte(vector, &num);
      info->doc_numbers[c] = last_doc + num + 1;
      last_doc += num + 1;

      /* Read the number of times the phrase occurs in this document */
      vec_getvbyte(vector, &info->phrase_frequency[c]);
      if (info->phrase_offsets[c] != NULL)
         free(info->phrase_offsets[c]);

      /* Prepare the memory for the appropriate number of offsets */
      info->phrase_offsets[c] = malloc(info->phrase_frequency[c] * sizeof(unsigned long));
      memset(info->phrase_offsets[c], 0, info->phrase_frequency[c] * sizeof(unsigned long));
      
      /* Read in all the document offsets */
      last_offset = -1;
      for (d = 0; d < info->phrase_frequency[c]; d++)
      {
         vec_getvbyte(vector, info->phrase_offsets[c] + d);
         last_offset += (info->phrase_offsets[c])[d] + 1;
         (info->phrase_offsets[c])[d] = last_offset;
      }
   }

   vec_free(vector);

   return info;
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct listinfo* get_next_biglist	(	struct index_scanner *	iscn,
		struct listinfo *	info
	)			[read]

Definition at line 36 of file readindex.c.

References get_list_at(), get_next_lo_info(), and lo_info::offset.

Referenced by cluster_clusterSequences().

{
   struct lo_info *lo_info;

   lo_info = get_next_lo_info(iscn, NULL);

   if (!lo_info)
      return NULL;

   return get_list_at(iscn, info, lo_info->offset);
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct listinfo* get_next_list	(	struct index_scanner *	iscn,
		struct listinfo *	info
	)			[read]

Return a phrase_info data structure for the next phrase in the index

Definition at line 31 of file readindex.c.

References get_list_at().

Referenced by cluster_clusterSequences().

{
   return get_list_at(iscn, info, -1);
}

Here is the call graph for this function:

Here is the caller graph for this function:

struct lo_info* get_next_lo_info	(	struct index_scanner *	iscn,
		struct lo_info *	lo_info
	)			[read]

Definition at line 48 of file readindex.c.

References index_scanner::offsets, and vbyte_read().

Referenced by get_next_biglist().

{
   FILE *offsetfile = iscn->offsets;

   if (!lo_info)
      lo_info = malloc(sizeof(struct lo_info));

   vbyte_read(offsetfile, &(lo_info->size));
   if (!(vbyte_read(offsetfile, &(lo_info->offset))))
   {
      free(lo_info);
      return NULL;
   }

   return lo_info;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void info_free

(

struct listinfo *

info

)

Deallocates the memory used by the data structure

Definition at line 151 of file readindex.c.

References listinfo::doc_numbers, and listinfo::phrase_frequency.

Referenced by get_list_at().

{
   free(info->doc_numbers);
   free(info->phrase_frequency);

   free(info);
}

Here is the caller graph for this function:

struct index_scanner* open_index

(

char *

index_prefix

)

[read]

Returns a data structure that contains open buffered file pointers to the index file and all the vector files in preparation for sequential reading of postings vectors.

Definition at line 13 of file readindex.c.

References index_scanner::idx_file, and index_scanner::offsets.

Referenced by cluster_clusterSequences().

{
   struct index_scanner *scanner;
   char fnamebuf[FILENAME_MAX + 1];

   fnamebuf[FILENAME_MAX] = '\0';

   scanner = (struct index_scanner *) malloc(sizeof(struct index_scanner));

   snprintf(fnamebuf, FILENAME_MAX, "%s.idx", index_prefix);
   scanner->idx_file = fopen(fnamebuf, "r");

   snprintf(fnamebuf, FILENAME_MAX, "%s.lo", index_prefix);
   scanner->offsets = fopen(fnamebuf, "r");

   return scanner;
}

Here is the caller graph for this function:

struct index_scanner* reset_index

(

struct index_scanner *

scn

)

[read]

Reset all the file pointers back to the beginning of the index

Definition at line 159 of file readindex.c.

References index_scanner::idx_file.

Referenced by cluster_clusterSequences().

{
   fseek(scn->idx_file, 0, SEEK_SET);

   return scn;
}

Here is the caller graph for this function:

---