/*
   File:        freq.c
   Author:      Jeff Schneider
   Created:     Wed Aug 11 18:19:00 EDT 1999
   Description: Utilities for finding frequent sets in datsets

   Copyright 1999, Jeff Schneider
*/

#include "freq.h"

/* Private prototypes */
static void free_matchlist(matchlist *ml);
static matchlist *mk_copy_matchlist(matchlist *ml);
static matchlist *mk_matchlist(int row, int count);
static void free_matchlist_head(matchlist_head *mld);
static matchlist_head *mk_copy_matchlist_head(matchlist_head *mld);
static void *void_mk_copy_matchlist_head(void *dat);
static matchlist_head *mk_empty_matchlist_head(void);
static void add_to_matchlist(matchlist *ml, matchlist *mlnew);
static matchlist *insert_in_matchlist(matchlist *ml, matchlist *mlnew);
static matchlist *add_or_inc_to_matchlist(matchlist *ml, int idx);
static int sum_matchlist(matchlist *ml);
static matchlist *mk_intersect_matchlists(matchlist *mla,matchlist *mlb,int mincount);
static hash_table *mk_match_hash_table(datset *ds, ivec *matchatts);
static ivec_array *mk_match_groups_ivec_array_from_hash_table(hash_table *ht);
static ivec_array *mk_match_groups_ivec_array(datset *ds, ivec *matchatts);
static datset *mk_empty_frequent_datset(datset *ds, ivec *freqatts, int setsize);
static int frequent_symbol_val(datset *fds, datset *ds, ivec *freqatts, int row, int col, ivec *index_map);
static void fill_frequent_datset_from_hash_table(datset *fds, hash_table *ht, ivec *counts);
static hash_table *compute_frequent_hash_table(datset *fds,datset*ds,ivec *matchatts, ivec *freqatts,int setsize, int minsetcount, ivec **counts);

/* Private, unused functions */
#ifdef AUTON_UNUSED_CODE
static void dump_hash_table_matchlists(hash_table *ht);
#endif /* AUTON_UNUSED_CODE */

static void free_matchlist(matchlist *ml)
{
  matchlist *t = ml;
  while(t)
  {
    matchlist *tf = t;
    t = t->next;
    am_free((char *)tf,sizeof(matchlist));
  }
}

int matchlist_num_bytes(matchlist *ml)
{
  int result = 0;
  while ( ml != NULL )
  {
    result += sizeof(ml);
    ml = ml->next;
  }
  return result;
}

int void_matchlist_num_bytes(void *x)
{
  return matchlist_num_bytes((matchlist *)x);
}

static void void_free_matchlist(void *dat)
{
  free_matchlist((matchlist *)dat);
}

static matchlist *mk_copy_matchlist(matchlist *ml)
{
  matchlist *newml = (ml) ? (matchlist *)am_malloc(sizeof(matchlist)) : NULL;
  if (newml) 
  {
    newml->row = ml->row;
    newml->count = ml->count;
    newml->next = mk_copy_matchlist(ml->next);
  }
  return newml;
}

static void *void_mk_copy_matchlist(void *dat)
{
  matchlist *ml = mk_copy_matchlist((matchlist *)dat);
  return (void *)ml;
}

static matchlist *mk_matchlist(int row, int count)
{
  matchlist *ml = (matchlist *)am_malloc(sizeof(matchlist));
  ml->row = row;
  ml->count = count;
  ml->next = NULL;
  return ml;
}

static void free_matchlist_head(matchlist_head *mld)
{
  if (mld->head) free_matchlist(mld->head);
  am_free((char *)mld,sizeof(matchlist_head));
}

static void void_free_matchlist_head(void *dat)
{
  free_matchlist_head((matchlist_head *)dat);
}

static matchlist_head *mk_copy_matchlist_head(matchlist_head *mld)
{
  matchlist_head *newmld=(mld) ? 
                         (matchlist_head *)am_malloc(sizeof(matchlist_head)) :
                         NULL;
  if (newmld) newmld->head = mk_copy_matchlist(mld->head);
  return newmld;
}

static void *void_mk_copy_matchlist_head(void *dat)
{
  matchlist_head *mlh = mk_copy_matchlist_head((matchlist_head *)dat);
  return (void *)mlh;
}

static matchlist_head *mk_empty_matchlist_head(void)
{
  matchlist_head *mlh = (matchlist_head *)am_malloc(sizeof(matchlist_head));
  mlh->head = NULL;
  return mlh;
}

/* When adding a new element to a linked list there are several places you
   could put the new element, but only one that works here.  Possibilities:
   At the head - can't do this because it would change the actual address of
                 this matchlist which may already be stored somewhere else
   At the tail - don't really want to do this because finding the tail means
                 traversing the whole list which leads to quadratic execution
                 times
   In middle   - specifically, we'll put the new element second in the list

   Precondition: ml is non-NULL, mlnew is non-NULL
*/
static void add_to_matchlist(matchlist *ml, matchlist *mlnew)
{
  matchlist *oldnext = ml->next;
  ml->next = mlnew;
  while(mlnew->next) mlnew = mlnew->next;
  mlnew->next = oldnext;
}

/* similar to add_to_matchlist except:
   inserts mlnew into sorted order according to the row field
   ASSUMES mlnew is a single element, not a list (disaster will result if
     if its actually a list
   returns the value of head of the list (because it may change)
   is less efficient because it must traverse the list to find the insertion
     point.
*/
static matchlist *insert_in_matchlist(matchlist *ml, matchlist *mlnew)
{
  matchlist *prev = NULL;
  matchlist *cur = ml;

  while (cur && (mlnew->row > cur->row))
  {
    prev = cur;
    cur = cur->next;
  }
  if (prev == NULL)  /* insert at head */
  {
    mlnew->next = cur;
    return mlnew;
  }
  else  /* all other cases */
  {
    prev->next = mlnew;
    mlnew->next = cur;
    return ml;
  }
}

/* Takes a member id for the matchlist (the row field) and increments the 
   counter if it finds that id in the list, or adds a new element (sorted) 
   with count one if it doesn't find it.  Returns a pointer to the head of 
   the list in case it changed.
*/
static matchlist *add_or_inc_to_matchlist(matchlist *ml, int idx)
{
  matchlist *mlp = ml;

  while(mlp && (mlp->row != idx)) mlp = mlp->next;

  if (mlp) 
  {
    mlp->count++;
    return ml;
  }
  else
  {
    matchlist *nml = mk_matchlist(idx,1);
    return insert_in_matchlist(ml,nml);
  }
}

/* add up all the counts in this matchlist */
static int sum_matchlist(matchlist *ml)
{
  int sum = 0;
  while(ml)
  {
    sum += ml->count;
    ml = ml->next;
  }
  return sum;
}

/* Takes two matchlists, which must be sorted already with no duplicates 
   (according to their row fields) and computes their intersection (according 
   to matching row fields).  The count for the intersection is the min of the 
   two counts.  
   If the sum of the entire new intersected list is less than mincount,
   then the intersection is not done and NULL is returned.  Also, NULL is
   returned even when mincount is 0 if there are no elements in the 
   intersection.

   There are two possibilities here:
   2 pass. count first, then create if necessary.
   1 pass. create and count simultaneously.  then delete if mincount not met.
   We opt for the first under the assumption that many will not meet the
   mincount condition.
*/
static matchlist *mk_intersect_matchlists(matchlist *mla,matchlist *mlb,int mincount)
{
  matchlist *ap = mla;
  matchlist *bp = mlb;
  matchlist *result = NULL;
  matchlist *ml = NULL;
  int sum = 0;

  while (ap && bp)
  {
    if (ap->row == bp->row) sum += int_min(ap->count,bp->count);
    if (ap->row < bp->row) ap = ap->next;
    else                   bp = bp->next;
  }

  if (sum >= mincount)
  {
    ap = mla;
    bp = mlb;
    while (ap && bp)
    {
      if (ap->row == bp->row) 
      {
	ml = mk_matchlist(ap->row,int_min(ap->count,bp->count));
	result = insert_in_matchlist(result,ml);
      }
      if (ap->row < bp->row) ap = ap->next;
      else                   bp = bp->next;
    }
  }
  return result;
}

/* The hash table will store all groups of rows in ds that have matching
   values in the matchatts columns.  Each group is a single entry in the
   hash table, and each entry is of type matchlist.  For efficiency, you
   might want to special case the situations where matchatts is NULL or
   zero length and not call this function then, but it will still work
   if you do.
*/
static hash_table *mk_match_hash_table(datset *ds, ivec *matchatts)
{
  int i,j;
  hash_table *ht = mk_empty_hash_table(void_mk_copy_matchlist,
				       void_free_matchlist,
				       void_matchlist_num_bytes);
  ivec *iv_key = (matchatts && ivec_size(matchatts) > 0) ? 
                  mk_ivec(ivec_size(matchatts)) : mk_ivec(1);
  matchlist *ml;

  set_hash_table_key_type(ht,IVEC_KEY_TYPE);

  for (i=0;i<ds_num_rows(ds);i++)
  {
    /* Generate the key for storing this row */

    if (!matchatts) /* every record is its own group, use row number as key */
    {
      ivec_set(iv_key,0,i);
    }
    else if (ivec_size(iv_key) == 0) /* every record in one group */
    {
      ivec_set(iv_key,0,0);
    }
    else /* the non-trivial case */
    {
      for (j=0;j<ivec_size(iv_key);j++)
	ivec_set(iv_key,j,ds_symbol_ref(ds,i,ivec_ref(matchatts,j)));
    }

    /* Use the key to add this row to the hash table groups */

    ml = (matchlist *)lookup_hash_entry(ht,(char *)iv_key);
    if (ml) add_to_matchlist(ml,mk_matchlist(i,0));
    else 
    {
      ml = mk_matchlist(i,0);
      add_hash_entry(ht,(char *)iv_key,(void *)ml);
      free_matchlist(ml);
    }
  }

  free_ivec(iv_key);

  return ht;
}

#ifdef AUTON_UNUSED_CODE
/* This function is strictly for debugging */
static void dump_hash_table_matchlists(hash_table *ht)
{
  int i,j;
  printf("%d bins  %d elements\n",ht->num_bins, ht->num_elements);
  for (i=0;i<ht->num_bins;i++)
  {
    hlist *hl = ht->array[i];
    while (hl)
    {
      matchlist *ml = (matchlist *)hl->data;
      if (ht->key_type == CHAR_KEY_TYPE) printf("key: %s\nrows: ",hl->key);
      else if (ht->key_type == IVEC_KEY_TYPE)
      {
	ivec *iv = (ivec *)hl->key;
	printf("key: ");
	for (j=0;j<ivec_size(iv);j++) printf("%d ",ivec_ref(iv,j));
	printf("\n");
      }
      while (ml) 
      {
	printf("%5d ",ml->row);
	ml = ml->next;
      }
      hl = hl->next;
      printf("\n");
    }
  }
}
#endif /* AUTON_UNUSED_CODE */

/* Converts the hash_table info into an ivec_array so its easier to handle.
   This code is mostly copied from dump_hash_table_matchlists, since it just
   runs through all the elements in the hash_table.
*/
static ivec_array *mk_match_groups_ivec_array_from_hash_table(hash_table *ht)
{
  int i,idx = 0;
  ivec_array *ia = mk_array_of_empty_ivecs(ht->num_elements);

  for (i=0;i<ht->num_bins;i++)
  {
    hlist *hl = ht->array[i];
    while (hl)
    {
      ivec *iv = mk_ivec(0);
      matchlist *ml = (matchlist *)hl->data;
      while (ml) 
      {
        add_to_ivec(iv,ml->row);
        ml = ml->next;
      }
      ivec_array_set(ia,idx++,iv);
      free_ivec(iv);
      hl = hl->next;
    }
  }
  return ia;
}

static ivec_array *mk_match_groups_ivec_array(datset *ds, ivec *matchatts)
{
  hash_table *ht = mk_match_hash_table(ds,matchatts);
  ivec_array *ia = mk_match_groups_ivec_array_from_hash_table(ht);

  /* strictly debugging */
  /* dump_hash_table_matchlists(ht); */

  free_hash_table(ht);
  return ia;
}

/* Create the new frequent datset.  There will be setsize*3 + 2 attributes.
   The first setsize attributes will contain the value of the freqatts
   being tabulated and be named attvalue0, attvalue1, etc.  If the size of 
   freqatts is 1, then the names of the values for each of the new attribtues 
   will be the same as in the original datset.  If there is more than one 
   freqatt, the names will be the juxtaposition of the attribute name and the 
   attribute value.
   After that the next attributes will be as follows:
   - setsize attributes recording the individual counts for each member of
     the set.
   - setsize attributes recording the ratio of pairs to number of individual
     counts for each member of the set
   - one attribute containing the max of the ratios (used primarily to sort
     the list)
   - one attribute containing the number of occurances of the set
*/
static datset *mk_empty_frequent_datset(datset *ds, ivec *freqatts, int setsize)
{
  int i,j;
  string_array *attvalues = mk_string_array(0);
  datset *fds = mk_empty_datset();
  int fds_attnum;
  char name[64];

  for (i=0;i<setsize;i++)
  {
    sprintf(name,"attvalue%d",i);
    ds_add_symbolic_attribute(fds,name,attvalues);
  }
  for (i=0;i<setsize;i++)
  {
    sprintf(name,"att%dcount",i);
    ds_add_real_attribute(fds,name);
  }
  for (i=0;i<setsize;i++)
  {
    sprintf(name,"att%dratio",i);
    ds_add_real_attribute(fds,name);
  }
  ds_add_real_attribute(fds,"maxratio");
  ds_add_real_attribute(fds,"count");

  for (i=0;i<ivec_size(freqatts);i++)
  {
    int attnum = ivec_ref(freqatts,i);
    for (j=0;j<ds_attnum_to_arity(ds,attnum);j++)
    {
      for (fds_attnum = 0; fds_attnum < setsize; fds_attnum++)
      {
        if (ivec_size(freqatts) == 1)
          ds_add_value_name(fds,fds_attnum,ds_attnum_value_to_name(ds,attnum,j));
        else
        {
          char *valname=mk_concat_strings(ds_attnum_to_name(ds,attnum),
					                                ds_attnum_value_to_name(ds,attnum,j));
          ds_add_value_name(fds,fds_attnum,valname);
          free_string(valname);
        }
      }
    }
  }

  free_string_array(attvalues);

  return fds;
}

/* This function looks up the symbol specified at the given row and column
   and converts it to its index in the resulting frequent datset.
   Note that col must be a member of freqatts and there will a my_error if
   it is not.  The conversion to symbol indices is straightforward.  The
   first attribute in freqatts has all its symbol values keeping their same
   indices.  Then the values for next attribute's values start counting up 
   from there, and so on through all the attributes in freqatts.

   Modified: we can no longer trust the symbol values to stay in order,
             so we create a map converting the above index into the
             actual index in the frequent datset.  We first look in the
             map for the value and if its not there, we compute it from
             the original attribute names and store it in the map.
*/
static int frequent_symbol_val(datset *fds, datset *ds, ivec *freqatts, 
                        int row, int col, ivec *index_map)
{
  int i;
  bool found = FALSE;
  int val = ds_symbol_ref(ds,row,col);
  int symbolval = val;
  int mapval;

  /* compute index of frequent symbol */
  for (i=0;i<ivec_size(freqatts);i++)
  {
    int freqcol = ivec_ref(freqatts,i);
    if (freqcol == col) 
    {
      found = TRUE;
      break;
    }
    val += ds_attnum_to_arity(ds,freqcol);
  }
  if (!found) my_error("frequent_symbol_val: it is illegal to pass a column number not in the freqatts ivec");

  /* if its not stored in the map, compute it and store it there */
  if (ivec_ref(index_map,val) < 0)
  {
    if (ivec_size(freqatts) == 1)
      mapval = ds_attnum_name_to_value(fds,0,ds_attnum_value_to_name(ds,col,symbolval));
    else
    {
      char *valname=mk_concat_strings(ds_attnum_to_name(ds,col),
					                            ds_attnum_value_to_name(ds,col,symbolval));
      mapval = ds_attnum_name_to_value(fds,0,valname);
      free_string(valname);
    }
    if (mapval < 0) 
      my_error("frequent_symbol_val: mistake in name construction");
    ivec_set(index_map,val,mapval);
  }
  return ivec_ref(index_map,val);
}

#ifdef NEVER

/* Having most of the pre-computation in place, this function generates the
   frequent set data and stores it in fds.  The parameters are:
   fds         - the new one being computed, so far only initialized to have
                 the appropriate column names and attribute value names
   ds          - the original datset
   ht          - a hash table storing which rows of records should be grouped
                 together.  see mk_match_hash_table for a description of how
                 it works.
   freqatts    - described in mk_frequent_datset
   setsize     - described in mk_frequent_datset
   minsetcount - described in mk_frequent_datset

   There is one serious philosophical issue to address at this point.  What
   do we do if a particular group has multiple numbers of the same attribute
   value in it.  For example, what if you're computing products commonly
   purchased together and someone buys 3 of them.  Here are some possible 
   things you can do:
   1. Treat it as a purchase of 3 things.  This can be a problem when computing
      the size of sets.  For example, if you proceed blindly ahead and you
      have 3 reds purchased together with 5 blues, you'll find that there are
      15 pairs of reds/blues.  This could be ok, except that it doesn't make
      sense that you've purchases more pairs of reds and blues than you have
      either of them separately.
   2. Compress it down to one purchase of each.  In this case the number of
      pairs from that order as well as the number of individual pieces would
      agree: its 1.  However, this isn't great either because now if you ask
      for how many items of each thing were purchased, you get the wrong
      answer.
   3. Some kind of inbetween hack.  Specifically, we'll use the min in this
      code.  If you have 3 reds and 5 blues we'll say that adds up to 3
      reds/blues pairs.  This may not be "correct", but it doesn't have any
      of the drawbacks listed above.

  This function has been decommissioned  -- JGS  9/14/99
*/
void compute_frequent_datset(datset *fds, datset *ds, ivec *matchatts,
			     ivec *freqatts, int setsize, int minsetcount)
{
  int group,i;
  ivec_array *groups = mk_match_groups_ivec_array(ds,matchatts);
  hash_table *fs_ht = mk_empty_hash_table(vint_copy,vint_free);
  ivec *iv_key = mk_ivec(setsize);
  int num_actual_rows;
  int row1index,row2index,col1index,col2index;
  int row1,row2,col1,col2;
  int col2start;
  int val1,val2;
  int multiple1,multiple2;
  ivec *counts = mk_zero_ivec(ds_attnum_to_arity(fds,0));


  if (setsize > 2)
  {
    printf("compute_frequent_datset: sets greater than size 2 not available\n");
    return;
  }

  printf("%d groups found\n",ivec_array_size(groups));

  /* compute the size one statistics */
  if (setsize > 0)
  {
    for (row1=0;row1<ds_num_rows(ds);row1++)
      for (col1index=0;col1index<ivec_size(freqatts);col1index++)
      {
	col1 = ivec_ref(freqatts,col1index);
	val1 = ds_symbol_ref(ds,row1,col1);
	for (i=0;i<col1index;i++)
	  val1 += ds_attnum_to_arity(ds,ivec_ref(freqatts,i));
	ivec_increment(counts,val1,1);
      }
  }

  if (setsize > 1)
  {
    /* loop over all groups */
    for (group=0;group<ivec_array_size(groups);group++)
    {
      ivec *group_iv = ivec_array_ref(groups,group);
      ivec *multiples = mk_constant_ivec(ivec_size(group_iv)*ivec_size(freqatts),-1);
      
      if (group == ivec_array_size(groups)/100) 
      {
	printf("1%%.."); 
	fflush(stdout);
      }
      if (((group+1) % (ivec_array_size(groups)/10)) == 0)
      {
	printf("%d%%..",(int)(100.0*(double)group/(double)ivec_array_size(groups)+0.5));
	fflush(stdout);
      }

    /* Determine how many of each of attribute value occurs in this group */
      for (row1index=0;row1index<ivec_size(group_iv);row1index++)
      {
	row1 = ivec_ref(group_iv,row1index);
	for (col1index=0;col1index<ivec_size(freqatts);col1index++)
	{
	  col1 = ivec_ref(freqatts,col1index);
	  multiple1 = row1index * ivec_size(freqatts) + col1index;

	  if (ivec_ref(multiples,multiple1) < 0) /* not processed yet */
	  {
	    val1 = ds_symbol_ref(ds,row1,col1);
	    ivec_set(multiples,multiple1,1);
	    for (row2index=row1index+1;
		 row2index<ivec_size(group_iv);
		 row2index++)
	    {
	      row2 = ivec_ref(group_iv,row2index);
	      val2 = ds_symbol_ref(ds,row2,col1);
	      if (val2 == val1)
	      {
		multiple2 = row2index * ivec_size(freqatts) + col1index;
		ivec_increment(multiples,multiple1,1);
		ivec_set(multiples,multiple2,0);
	      }
	    }
	  }
	}
      }
      

    /* These next 4 for loops go over all pairs of attribute values in this
       group and on the freqatts list.  So first we loop over the rows and
       all the freqatts attributes in the row.  Then we repeat the same two
       loops (because we're looking for pairs).  The logic for the last loop
       is a little tricky.  If we happen to be matching a row against itself
       we only want to match to attributes occuring after the one in the
       first member of the pair, otherwise we can match over all of them.
       Note that its not strictly necessary to catch this overlap here since
       the later code will refuse to store anything paired to itself anyway.
    */
      for (row1index=0;row1index<ivec_size(group_iv);row1index++)
      {
	row1 = ivec_ref(group_iv,row1index);
	for (col1index=0;col1index<ivec_size(freqatts);col1index++)
	{
	  col1 = ivec_ref(freqatts,col1index);
	  multiple1 = ivec_ref(multiples,
			       row1index*ivec_size(freqatts)+col1index);

	  /* comment out this line if you want to use type 1 counting as
             described in the header of this function. */
	  if (multiple1 == 0) continue;

	  for (row2index=row1index;row2index<ivec_size(group_iv);row2index++)
	  {
	    row2 = ivec_ref(group_iv,row2index);
	    col2start = (row1 == row2) ? col1index + 1 : 0;
	    for (col2index=col2start;col2index<ivec_size(freqatts);col2index++)
	    {
	      col2 = ivec_ref(freqatts,col2index);
	      multiple2 = ivec_ref(multiples,
				   row2index*ivec_size(freqatts)+col2index);

	      /* comment out this line if you want to use type 1 counting as
		 described in the header of this function. */
	      if (multiple2 == 0) continue;

	    /* now we have row and column indices for our pair.  we need to
               convert them into the appropriate values for storing in the
               new dataset.  Remember the new symbols are numbered sequentially
               just as they occur in the original except if there is more than
               one freqatt.  Then the indices for the non-first freqatt will
               be increment by the number of values occurring before it in
               earlier freqatts.
            */
	      val1 = ds_symbol_ref(ds,row1,col1);
	      for (i=0;i<col1index;i++)
		val1 += ds_attnum_to_arity(ds,ivec_ref(freqatts,i));
	      val2 = ds_symbol_ref(ds,row2,col2);
	      for (i=0;i<col2index;i++)
		val2 += ds_attnum_to_arity(ds,ivec_ref(freqatts,i));

	      /* finally we get to store something -- but don't bother if the 
                 individual counts of either of the two attributes aren't 
                 above threshold because there obviously can't be enough pairs 
                 involving them.
	       */
	      if ((val1 != val2) && 
		  (ivec_ref(counts,val1) >= minsetcount) &&
		  (ivec_ref(counts,val2) >= minsetcount))
	      {
		int *count;
		int initval = 1;
		char *key_string;

		ivec_set(iv_key,0,val1);
		ivec_set(iv_key,1,val2);
		/* hmmm, we may be able to do some 1-way time dependent stuff
		   by taking this sort out, or doing something else here
		   */
		ivec_sort(iv_key,iv_key);
		
		/* convert ivec_key to a char key (it takes far less memory) */
		key_string = mk_string_from_ivec(iv_key);
		
		count = (int *)lookup_hash_entry(fs_ht,key_string);
		if (count) 
		{
		  /* (*count)++;  type 1 or 2 counting (see header comments) */
		  (*count) += int_min(multiple1,multiple2);  /* type 3 */
		}
		else 
		{
		  /* initval = 1; type 1 or 2 counting (see header comments) */
		  initval = int_min(multiple1,multiple2);  /* type 3 */
		  add_hash_entry(fs_ht,key_string,(void *)(&initval));
		  if ((fs_ht->num_elements % 1000000) == 0)
		    printf("fs hash table has %dM elements\n",
			   fs_ht->num_elements/1000000);
		}
		free_string(key_string);
	      }
	    }
	  }
	}
      }
      free_ivec(multiples);
    }
  }

  printf("\nDone computing frequent set w/ %d elements\n",fs_ht->num_elements);

  free_ivec_array(groups);
  free_ivec(iv_key);

  /* info is in the counts ivec, read it from there */
  if (setsize == 1)
  {
    ds_add_rows(fds,ivec_size(counts));
    num_actual_rows = 0;

    for (i=0;i<ivec_size(counts);i++)
      if (ivec_ref(counts,i) >= minsetcount)
      {
	double val = (double)ivec_ref(counts,i);
	ds_symbol_set(fds,num_actual_rows,0,i);
	ds_real_set(fds,num_actual_rows,1,val);  /* individual count */
	ds_real_set(fds,num_actual_rows,2,1.0);  /* ratio */
	ds_real_set(fds,num_actual_rows,3,1.0);  /* max ratio */
	ds_real_set(fds,num_actual_rows,4,val);  /* set count (same as ind) */
	num_actual_rows++;
      }
    ds_delete_last_n_rows(fds,ivec_size(counts) - num_actual_rows);
  }
  else
  {
    /* "dump" the new hash table and fill in the datset with its contents */
    ds_add_rows(fds,fs_ht->num_elements);
    num_actual_rows = 0;
    for (i=0;i<fs_ht->num_bins;i++)
    {
      int attnum;
      hlist *hl = fs_ht->array[i];

      if (((i+1) % (fs_ht->num_bins/10)) == 0)
      {
	printf("%d%%..",(int)(100.0*(double)i/(double)fs_ht->num_bins+0.5));
	fflush(stdout);
      }

      while (hl)
      {
	int *count = (int *)hl->data;
	char *key_string = hl->key;
	ivec *key = mk_ivec_from_string(key_string);
	
	/* if the count is high enough, put it in the datset */
	if (*count >= minsetcount)
	{
	  double max = 0.0;
	  /* first the names */
	  for (attnum=0;attnum<ivec_size(key);attnum++)
	    ds_symbol_set(fds,num_actual_rows,attnum,ivec_ref(key,attnum));
	  /* then their individual counts */
	  for (attnum=0;attnum<ivec_size(key);attnum++)
	    ds_real_set(fds,num_actual_rows,attnum+ivec_size(key),
			(double)ivec_ref(counts,ivec_ref(key,attnum)));
	  /* then their ratios (save the max) */
	  for (attnum=0;attnum<ivec_size(key);attnum++)
	  {
	    double ratio = (double)(*count) /
	                   (double)ivec_ref(counts,ivec_ref(key,attnum));
	    if (ratio > max) max = ratio;
	    ds_real_set(fds,num_actual_rows,attnum+2*ivec_size(key),ratio);
	  }
	  ds_real_set(fds,num_actual_rows,3*ivec_size(key),max);
	  ds_real_set(fds,num_actual_rows,3*ivec_size(key)+1,(double)(*count));
	  num_actual_rows++;
	}
	hl = hl->next;
	free_ivec(key);
      }
    }
    ds_delete_last_n_rows(fds,fs_ht->num_elements - num_actual_rows);
  }

  if (counts) free_ivec(counts);
  if (fs_ht) free_hash_table(fs_ht);
}

#endif

/* dump the hash table and fill in the datset with its contents */
static void fill_frequent_datset_from_hash_table(datset *fds, hash_table *ht,
					  ivec *counts)
{
  int i;
  int num_actual_rows = 0;
  
  ds_add_rows(fds,ht->num_elements);

  for (i=0;i<ht->num_bins;i++)
  {
    int attnum;
    hlist *hl = ht->array[i];

    if (((i+1) % (ht->num_bins/10)) == 0)
    {
      printf("%d%%..",(int)(100.0*(double)i/(double)ht->num_bins+0.5));
      fflush(stdout);
    }

    while (hl)
    {
      matchlist_head *mlh = (matchlist_head *)hl->data;
      double count = sum_matchlist(mlh->head);
      char *key_string = hl->key;
      ivec *key = mk_ivec_from_string(key_string);
	
      double max = 0.0;

      /* first the names */
      for (attnum=0;attnum<ivec_size(key);attnum++)
        ds_symbol_set(fds,num_actual_rows,attnum,ivec_ref(key,attnum));

      /* then their individual counts */
      for (attnum=0;attnum<ivec_size(key);attnum++)
        ds_real_set(fds,num_actual_rows,attnum+ivec_size(key),
                    ivec_ref(counts,ivec_ref(key,attnum)));
      
      /* then their ratios (save the max) */
      for (attnum=0;attnum<ivec_size(key);attnum++)
      {
        double ratio = (count) /
                       ivec_ref(counts,ivec_ref(key,attnum));
        if (ratio > max) max = ratio;
        ds_real_set(fds,num_actual_rows,attnum+2*ivec_size(key),ratio);
      }
      ds_real_set(fds,num_actual_rows,3*ivec_size(key),max);
      ds_real_set(fds,num_actual_rows,3*ivec_size(key)+1,count);
      num_actual_rows++;

      hl = hl->next;
      free_ivec(key);
    }
  }
}

/* How are these frequent sets stored?

   There is a separate hash table for each set size.
   That hash table has elements in it whose keys are strings specifying which
     members (values of the frequent attributes fields) are in this set.
   Stored in the data for each element is something called a matchlist.
   A matchlist is a linked list.  Each element of the list has 2 fields:
     1. row   - actually the index of a particular group (grouped set of rows
                in the original datset) for which that group has at least one
                instance of this particular frequent set (given by the key)
     2. count - tells how many times this frequent set appears in this group

  What a confusing way to represent simple information.  Why this way?

  Once we've constructed small set sizes (say size 1 and size 2), its easy
  to efficiently look for larger ones (say 3) by doing a simple intersection
  operation on the lists from the smaller ones and checking to see that the
  minsetcount is satisfied.

  This function also builds and returns an ivec with all the single element
  counts in it (except that those below minsetcount are 0) if a non-NULL
  pointer is provided.  Unfortunately, some functions want these figures for
  output.

  A note on indexing.  Unfortunately, we can't trust datsets to leave the
  attribute values the way we entered them.  So we have to create a map
  from the straightforward indexing scheme to wherever the datset put them.
  An additional dangerous assumption: that it will order these values the
  same for each attribute since they're all the same for each attribute.
*/
static hash_table *compute_frequent_hash_table(datset *fds,datset*ds,ivec *matchatts,
                                        ivec *freqatts,int setsize,
                                        int minsetcount, ivec **counts)
{
  int i,g,ia,ib,startb,sum;
  ivec_array *groups = mk_match_groups_ivec_array(ds,matchatts);
  hash_table **ht = (hash_table **)am_malloc(sizeof(hash_table *) * setsize);
  hash_table *result = NULL;
  int row1ind, row1, col1ind, col1, val1;
  matchlist *ml = NULL;
  matchlist_head *mlh = NULL;
  ivec *iv_key = NULL;
  char *key_string = NULL;
  string_array *delete_list = NULL;
  ivec *index_map;

  for (i=0;i<setsize;i++) ht[i] = NULL;

  for (i=0,sum=0;i<ivec_size(freqatts);i++)
    sum += ds_attnum_to_arity(ds,ivec_ref(freqatts,i));

  if (counts) *counts = mk_zero_ivec(sum);
  index_map = mk_constant_ivec(sum,-1);

  printf("%d groups.\n",ivec_array_size(groups));

  /* Size one sets first */
  if (setsize > 0)
  {
    ht[0] = mk_empty_hash_table(void_mk_copy_matchlist_head,
				void_free_matchlist_head,
				void_matchlist_num_bytes);

    printf("Begin creating size 1 sets.\n");

    /* blindly fill up the hash table */
    iv_key = mk_ivec(1);
    for (g=0;g<ivec_array_size(groups);g++)
    {
      ivec *group = ivec_array_ref(groups,g);

      if (((g+1) % (ivec_array_size(groups)/10)) == 0)
      {
	printf("%d%%..",(int)(100.0*(double)g/(ivec_array_size(groups)+0.5)));
	fflush(stdout);
      }


      for (row1ind = 0; row1ind < ivec_size(group); row1ind++)
      {
        row1 = ivec_ref(group,row1ind);
        for (col1ind = 0; col1ind < ivec_size(freqatts); col1ind++)
        {
          col1 = ivec_ref(freqatts,col1ind);
          val1 = frequent_symbol_val(fds,ds,freqatts,row1,col1,index_map);
          ivec_set(iv_key,0,val1);
          key_string = mk_string_from_ivec(iv_key);
          mlh = (matchlist_head *)lookup_hash_entry(ht[0],key_string);
          if (mlh)
	    mlh->head = add_or_inc_to_matchlist(mlh->head,g);
	  else
	  {
	    mlh = mk_empty_matchlist_head();
	    mlh->head = add_or_inc_to_matchlist(mlh->head,g);
	    add_hash_entry(ht[0],key_string,(void *)mlh);
	    free_matchlist_head(mlh);
	  }
	  free_string(key_string);
        }
      }
    }

    free_ivec(iv_key);
    free_ivec_array(groups);  /* don't need these anymore */

    printf("\nCreated %d size one sets. Deleting small ones.\n",
	   ht[0]->num_elements);

    /* go back through the hash table and delete entries that are too small.
       Warning: we could be playing with fire in this section.  The code that
       "dumps" the hash table reads from its internal structures at the same
       time the "delete" operation will affect those internal structures.
       So we'll just record the things we want to delete and then come back
       later to do the real deletion.
    */
    delete_list = mk_string_array(0);
    for (i=0;i<ht[0]->num_bins;i++)
    {
      hlist *hl = ht[0]->array[i];
      while(hl)
      {
	bool ok;
	int lsum = sum_matchlist(((matchlist_head *)hl->data)->head);
	if (lsum < minsetcount) add_to_string_array(delete_list,hl->key);
	else if (counts) ivec_set(*counts,int_from_string(hl->key,&ok),lsum);
	hl = hl->next;
      }
    }
    for (i=0;i<string_array_size(delete_list);i++)
      delete_hash_entry(ht[0],string_array_ref(delete_list,i));

    printf("Delete complete.  %d size one sets remaining.\n",
	   ht[0]->num_elements);

    free_string_array(delete_list);
  }

  /* i is the index in the hash table array.  the size of the set is
     actually i+1.  we have the size 1 sets done already.  we'll construct
     each successive set by considering merging whichever two other subsets
     have the smallest number of combinations to consider.  For example,
     if we're constructing sets of size 4, we'll check to see whether the
     product of the number of elements in the size 1 and size 3 sets is
     greater than the square of the elements in the size 2 set, and then
     do the construction on the smaller one.
  */
  for (i=1;i<setsize;i++)
  {
    int seta,setb,set;
    int bestcount = -77;
    int count;
    int besta = -77;
    int bestb = -77;

    /* find the best (smallest) sets to build the ith set from */
    for (set=0;set<(i+1)/2;set++)
    {
      seta = set;
      setb = i - seta - 1;  /* (i+1) - (seta+1) - 1 */
      count = ht[seta]->num_elements * ht[setb]->num_elements;
      if ((set == 0) || (count < bestcount))
      {
	besta = seta;
	bestb = setb;
	bestcount = count;
      }
    }

    printf("Beginning creation of size %d sets using size %d and %d sets.\n",
	   i+1,besta+1,bestb+1);

    /* Dump 2 hash tables and find elements with significant intersections. */
    ht[i] = mk_empty_hash_table(void_mk_copy_matchlist_head,
				void_free_matchlist_head,
				void_matchlist_num_bytes);
    for (ia=0;ia<ht[besta]->num_bins;ia++)
    {
      hlist *hla = ht[besta]->array[ia];
      startb = (besta==bestb) ? ia : 0;

      if (((ia+1) % (ht[besta]->num_bins/10)) == 0)
      {
	printf("%d%%..",(int)(100.0*(double)ia/(double)ht[besta]->num_bins+0.5));
	fflush(stdout);
      }

      while (hla)
      {
	for (ib=startb;ib<ht[bestb]->num_bins;ib++)
	{
	  hlist *hlb = ht[bestb]->array[ib];

	  while (hlb)
	  {
	    matchlist_head *mlha = (matchlist_head *)hla->data;
	    matchlist_head *mlhb = (matchlist_head *)hlb->data;
	    ivec *iva = mk_ivec_from_string(hla->key);
	    ivec *ivb = mk_ivec_from_string(hlb->key);
	    ivec *iv = mk_ivec_union_ordered(iva,ivb);
	    char *new_key = mk_string_from_ivec(iv);

	    /* Check all of the following conditions:
	       1. Do the two component sets contain unique elements.
	       2. Is this set not already in the hash table.
	       3. Is the size of the new set large enough
	     */
	    ml = NULL;
	    if((ivec_size(iv) == (ivec_size(iva) + ivec_size(ivb))) &&
	       !lookup_hash_entry(ht[i],new_key) &&
	       (ml=mk_intersect_matchlists(mlha->head,mlhb->head,minsetcount)))
	    {
	      matchlist_head *l_mlh = mk_empty_matchlist_head();
	      l_mlh->head = ml;
	      add_hash_entry(ht[i],new_key,(void *)l_mlh);
	      free_matchlist_head(l_mlh); /* frees ml too */
	    }

	    if (iva) free_ivec(iva); 
	    if (ivb) free_ivec(ivb); 
	    if (iv) free_ivec(iv);
	    if (new_key) free_string(new_key);

	    hlb = hlb->next;
	  }
	}
	hla = hla->next;
      }
    }
    
    printf("\nCompleted creation of %d element size %d set.\n",
	   ht[i]->num_elements,i+1);
  }

  for (i=0;i<setsize-1;i++) if (ht[i]) free_hash_table(ht[i]);
  result = ht[setsize-1];
  am_free((char *)ht,setsize*sizeof(hash_table *));
  free_ivec(index_map);

  return ht[setsize-1];
}

/* This function creates a new datset from an existing one.  The new datset
   represents the frequent sets appearing in the original one.  The parameters
   are as follows:

   ds        - an existing datset, from which the new one will be built.

   matchatts - rows in ds will be considered to be a group if matchatts
               contain the same value.  for example, if you have a list
               of product orders, you might specify the date field and
               the customer id field here to indicate that you want to 
               group orders together by date and customer.  the goal in 
               that case might be to find out which sets of products 
               commonly get ordered together.  there are two special cases:
               matchatts is 0 length - all records are grouped together.
                                       this is included for completeness,
                                       but probably has no practical use.
               matchatts is NULL     - all records are treated independently.
                                       this usually only makes sense if
                                       the size of freqatts is greater than 1

   freqatts  - these are the attributes you are looking for frequent sets of.
               in the previous example, this would be the product ids.  note
               that it is perfectly valid to have more than one attribute
               listed here.  for example, if you wanted to find common
               customer/product combinations (that customer commonly orders
               that product), then you would leave matchatts NULL, but put
               both product and customer in freqatts.

   setsize   - this indicates the maximum size of the sets.  for example, if
               you're interested in finding common pairs of products, you
               would set this to 2.  If you're interested in common triples, 
               you set it to 3, etc.

   minsetcount - the minimum number of records that must have these 
                 co-occurring freqatts values in order for it to be included
                 in the output.  for example, you may want to only include
                 pairs of products that were ordered together at least 10
                 times in the datset.

   Note that the size of the resulting datset can be very large.  You are
   at a greater risk of creating a large datset if you choose a large setsize,
   because the number of possible tuples gets large.  The easiest way to
   combat this is to increase minsetcount, because the number of them actually
   occurring a large number of times may still be small.

   There is one philosophical issue to address at this point.  What
   do we do if a particular group has multiple numbers of the same attribute
   value in it.  For example, what if you're computing products commonly
   purchased together and someone buys 3 of them.  Here are some possible 
   things you can do:
   1. Treat it as a purchase of 3 things.  This can be a problem when computing
      the size of sets.  For example, if you proceed blindly ahead and you
      have 3 reds purchased together with 5 blues, you'll find that there are
      15 pairs of reds/blues.  This could be ok, except that it doesn't make
      sense that you've purchases more pairs of reds and blues than you have
      either of them separately.
   2. Compress it down to one purchase of each.  In this case the number of
      pairs from that order as well as the number of individual pieces would
      agree: its 1.  However, this isn't great either because now if you ask
      for how many items of each thing were purchased, you get the wrong
      answer.
   3. Some kind of inbetween hack.  Specifically, we'll use the min in this
      code.  If you have 3 reds and 5 blues we'll say that adds up to 3
      reds/blues pairs.  This may not be "correct", but it doesn't have any
      of the drawbacks listed above.
*/
datset *mk_frequent_datset(datset *ds, ivec *matchatts, ivec *freqatts,
                           int setsize, int minsetcount)
{
  bool bailout = FALSE;
  int i;
  datset *fds = NULL;
  hash_table *ht;
  ivec *counts = NULL;

  if (matchatts)
    for (i=0;i<ivec_size(matchatts);i++)
      if (!ds_attnum_to_symbolicp(ds,ivec_ref(matchatts,i)))
      {
	printf("All match attributes must be symbolic in order to make a frequent datset.\n");
	printf("Attribute %s is not symbolic.\n",
	       ds_attnum_to_name(ds,ivec_ref(matchatts,i)));
	bailout = TRUE;
      }

  if (freqatts)
    for (i=0;i<ivec_size(freqatts);i++)
      if (!ds_attnum_to_symbolicp(ds,ivec_ref(freqatts,i)))
      {
	printf("All frequent attributes must be symbolic in order to make a frequent datset.\n");
	printf("Attribute %s is not symbolic.\n",
	       ds_attnum_to_name(ds,ivec_ref(freqatts,i)));
	bailout = TRUE;
      }

  if (!bailout)
  {
    fds = mk_empty_frequent_datset(ds,freqatts,setsize);
    
    /* old way to compute the frequent datset 
    compute_frequent_datset(fds,ds,matchatts,freqatts,setsize,minsetcount);
    */

    ht = compute_frequent_hash_table(fds,ds,matchatts,freqatts,setsize,
				     minsetcount,&counts);

    fill_frequent_datset_from_hash_table(fds,ht,counts);

    free_hash_table(ht);
    free_ivec(counts);
  }

  return fds;
}





