/*
   File:        ds.c
   Author:      Andrew W. Moore
   Created:     Sun Jun  1 14:23:03 EDT 1997
   Description: Simple Datasets

   Copyright 1997, The Auton Lab and SPR
*/

#include "ds.h"
#include "dsut.h"

shape *ds_shape(const datset *ds)
{
  return ds->sh;
}

column *ds_attnum_to_column(const datset *ds,int attnum)
{
#ifndef AMFAST
  if ( attnum < 0 || attnum >= ds_shape(ds)->num_atts ) {
    my_errorf( "ds_attnum_to_column: illegal attnum %d not in range [0,%d]\n",
	       attnum, ds_shape(ds)->num_atts - 1);
  }
#endif
  return(ds->cols_array[attnum]);
}

/* Frees the datset and all its subcomponents. */
void free_datset(datset *x)
{
  int i;
  for ( i = 0 ; i < ds_num_atts(x) ; i++ )
    free_column(x->cols_array[i]);
  AM_FREE_ARRAY(x->cols_array,column *,x->array_size);
  free_shape(x->sh);

  x->cols_array = NULL;
  x->sh = NULL;

  AM_FREE(x,datset);
  /* printf("freed a datset\n");*/
}

bool ds_shape_equal(datset *ds,datset *test_ds)
{
  return shape_equal(ds->sh,test_ds->sh);
}

/************ MK_COPY operations ********/

/* Builds and returns a copy of x with in turn contains copies
   of all of x's subcomponents. */
datset *mk_copy_datset(datset *x)
{
  datset *newx = AM_MALLOC(datset);
  int i;

  if (Verbosity >= 0) printf("Making a copy of a datset\n");

  newx -> num_rows = x -> num_rows;
  newx -> sh = mk_copy_shape(x->sh);
  newx -> array_size = x -> array_size;
  newx -> cols_array = AM_MALLOC_ARRAY(column *,x->array_size);
  for ( i = 0 ; i < ds_num_atts(x) ; i++ )
    newx -> cols_array[i] = mk_copy_column(ds_attnum_to_column(x,i));
  return(newx);
}

/* Sends an ASCII (detailed) printout of all of datset's
   substructures to the specified stream. Use stdout if you
   want to send to the console. */
void fprintf_datset(FILE *s,char *m1,datset *x,char *m2)
{
  char buff[200];
  int i;
  sprintf(buff,"%s -> num_rows",m1);
  fprintf_int(s,buff,x->num_rows,m2);
  sprintf(buff,"%s -> sh",m1);
  fprintf_shape(s,buff,x->sh,m2);
  for ( i = 0 ; i < ds_num_atts(x) ; i++ )
  {
    sprintf(buff,"%s -> cols[%d]",m1,i);
    fprintf_column(s,buff,ds_attnum_to_column(x,i),m2);
  }
}

void pdatset(datset *ds)
{
  fprintf_datset(stdout,"ds",ds,"\n");
}

/* Creates a datset of the given shape in which each value of
   each symbolic attribute is 0 (i.e. the first of the list of
   attribute-value-names for the given columns) and each real-valued
   value is 0.0 */
datset *mk_zero_datset(shape *sh,int num_rows)
{
  datset *ds = AM_MALLOC(datset);
  int i;

  /*  printf("Making a zero size datset\n"); */

  ds -> num_rows = num_rows;
  ds -> sh = mk_copy_shape(sh);
  ds -> array_size = shape_num_atts(sh);
  ds->cols_array = AM_MALLOC_ARRAY(column *,ds->array_size);
  for ( i = 0 ; i < shape_num_atts(sh) ; i++ )
    ds->cols_array[i] = mk_zero_column(shape_colshape_ref(sh,i),num_rows);
  return(ds);
}

/* Makes a datset with no rows and no columns.  You may use the add ...
   functions to give it rows and columns.
 */
datset *mk_empty_datset(void)
{
  string_array *attnames = mk_string_array(0);
  shape *sh = mk_shape(attnames);
  datset *ds = mk_zero_datset(sh,0);
  free_shape(sh);
  free_string_array(attnames);
  return ds;
}

/* Does not change shape of ds in any way, except it
   does ensure that the internal arrays are such that
   ds->array_size is happy about adding a new column */
void ds_require_extra_column(datset *ds)
{
  int old_num_atts = ds_num_atts(ds);
  int new_num_atts = old_num_atts + 1;

  if ( ds->array_size < new_num_atts )
  {
    int new_size = 2 + 2 * ds->array_size;
    int i;
    column **new_cols = AM_MALLOC_ARRAY(column *,new_size);
    for ( i = 0 ; i < old_num_atts ; i++ )
      new_cols[i] = ds->cols_array[i];
    for ( i = old_num_atts ; i < new_size ; i++ )
      new_cols[i] = NULL;
    AM_FREE_ARRAY(ds->cols_array,column *,ds->array_size);
    ds->array_size = new_size;
    ds->cols_array = new_cols;
  }
}

/* Adds a new attribute (column) to the datset, of type REAL_COL.
   The initial values in the column are all 0.0 */
void ds_add_real_attribute(datset *ds,char *attname)
{
  int new_attnum = ds_num_atts(ds);
  ds_require_extra_column(ds);
  shape_add_real_attribute(ds_shape(ds),attname);
  if ( new_attnum >= ds->array_size || new_attnum >= ds->sh->array_size )
    my_error("array_size wrong");
  ds->cols_array[new_attnum] = 
    mk_zero_column(shape_colshape_ref(ds->sh,new_attnum),ds->num_rows);  
}

/* Makes a new symbolic attribute. The arity of the new attribute is
   string_array_size(value_names), and the name of the i'th value is
   string_array_ref(value_names,i).

   The name of the new attribute is attname.

   The initial values in the column are all 0 (corresponding
   to the first element in value_names).

   PRE: value_names must be in INCREASING lexicographic order. */
void ds_add_symbolic_attribute(datset *ds,char *attname,string_array *value_names)
{
  int new_attnum = ds_num_atts(ds);
  ds_require_extra_column(ds);
  shape_add_symbolic_attribute(ds_shape(ds),attname,value_names);
  if ( new_attnum >= ds->array_size || new_attnum >= ds->sh->array_size )
    my_error("array_size wrong");
  ds->cols_array[new_attnum] = 
    mk_zero_column(shape_colshape_ref(ds->sh,new_attnum),ds->num_rows);  
}

void ds_copy_attribute(datset *ds,char *oldname,char *newname)
{
  int newnum = ds_name_to_attnum(ds,newname);
  int oldnum = ds_name_to_attnum(ds,oldname);
  if(oldnum<0){
    char *buff = mk_printf("ds_copy_attribute: %s does not exist!",oldname);
    my_error(buff);
    free_string(buff);
  } else if(newnum>=0){
    char *buff = mk_printf("ds_copy_attribute: %s already exists!",newname);
    my_error(buff);
    free_string(buff);
  } else {
    bool psymb = ds_attnum_to_symbolicp(ds,oldnum);
    bool symbol = ds_symbol_ref_ok(ds,oldnum);
    bool real = ds_real_ref_ok(ds,oldnum);
    int i;

    if(symbol){
      string_array *values = mk_string_array(0);
      for(i=0;i<ds_attnum_to_arity(ds,oldnum);i++)
        add_to_string_array(values,ds_attnum_value_to_name(ds,oldnum,i));
      ds_add_symbolic_attribute(ds,newname,values);
      free_string_array(values);
    } else
      ds_add_real_attribute(ds,newname);

    newnum = ds_name_to_attnum(ds,newname);
    oldnum = ds_name_to_attnum(ds,oldname);   /*This might change when we add a new att!*/
    if(symbol && real){
      if(psymb) ds_set_coltype(ds,newnum,SYMBOL_COL,REAL_COL);
      else ds_set_coltype(ds,newnum,REAL_COL,SYMBOL_COL);
    }

    for(i=0;i<ds_num_rows(ds);i++){
      if(ds_value_is_missing(ds,i,oldnum))
        ds_set_value_missing(ds,i,newnum);
      else
      {
        if(symbol) ds_symbol_set(ds,i,newnum,ds_symbol_ref(ds,i,oldnum));
        if(real) ds_real_set(ds,i,newnum,ds_real_ref(ds,i,oldnum));
      }
    }
  }
}

/* Use this function to mess with a dataset coltype. 

   It will cause a my_error if you attempt to set major_coltype
   to be UNDEFINED_COL or to the same value as minor_coltype.

   The major_type of the attribute will be what is detected by
   ds_attnum_to_symbolicp(...).

   The minor type, if not undefined, will allow both real and symbolic
   refs to the attribute. 

   This function never changes stored values of stored value names,
   except to delete the real component should the real
   component become undefined or to delete the symbolic component
   should the symbolic component become undefined.

   If a symbolic component is created, it has arity 0 and any
   call to ds_symbol_ref will return zero. Its arity can be changed
   by a call to ds_add_value_name or ds_set_value_names.

   If a real component is created, all its values are 0.0.

   THE CASUAL PROGRAMMER SHOULD NOT USE THIS FUNCTION: TO ADD
   SYMBOLIC/REAL COMPONENTS YOU SHOULD USE ONE OF THE COLUMN TRANSFORM
   OPS LISTED BELOW. THIS RESTRICTION IS IN ORDER TO PREVENT
   SYMBOLIC COMPONENTS THAT DISAGREE WITH REAL COMPONENTS.

   To find out what is the major_coltyp of an attribute, use
   ds_attnum_to_symbolicp...if TRUE, the major_coltype is SYMBOL_COL.
   If FALSE the major_coltype is REAL_COL.

   To find out there's a symbol component somewhere use ds_symbol_ref_ok
   To find out there's a real component somewhere use ds_real_ref_ok
*/
void ds_set_coltype(datset *ds,int attnum,int major_coltype,int minor_coltype)
{
  shape_set_coltype(ds_shape(ds),attnum,major_coltype,minor_coltype);
  column_set_coltype(ds_attnum_to_column(ds,attnum),
		     major_coltype,minor_coltype);
}

/* Returns TRUE iff the major or minor mode of the given 
   attribute is SYMBOL_COL. As the name suggests, you may call
   ds_symbol_ref(ds,attnum) if and only if ds_symbol_ref_ok(ds,attnum).
*/
bool ds_symbol_ref_ok(const datset *ds,int attnum)
{
  return shape_symbol_ref_ok(ds_shape(ds),attnum);
}

/* Returns TURE iff all attnums of the given datset pass through
   ds_symbol_ref_ok. Nobody used this, so it doesn't get compiled.
*/

bool ds_all_symbol_ref_ok(datset *ds)
{
  int num_atts = ds_num_atts(ds);
  int i;

  for( i = 0; i < num_atts; i++)
  {
    if(!ds_symbol_ref_ok(ds,i))
      return FALSE;
  }

  return TRUE;
}

/* Returns TRUE iff the major or minor mode of the given 
   attribute is REAL_COL. As the name suggests, you may call
   ds_real_ref(ds,attnum) if and only if ds_real_ref_ok(ds,attnum).
*/
bool ds_real_ref_ok(const datset *ds,int attnum)
{
  return shape_real_ref_ok(ds_shape(ds),attnum);
}

#ifdef AMFAST
#define check_ds_symbol_ref_ok(ds,attnum) /* do nothing */
#define check_ds_real_ref_ok(ds,attnum) /* do nothing */
#define check_value_names_increasing(valnames) /* do nothing */
#else
void check_ds_symbol_ref_ok(const datset *ds,int attnum)
{
  if ( !ds_symbol_ref_ok(ds,attnum) )
    my_error(mk_printf("check_ds_symbol_ref_ok: "
		       "attnum = %d (attname = %s): it wasn't",
		       attnum,ds_attnum_to_name(ds,attnum)));
}

void check_ds_real_ref_ok(datset *ds,int attnum)
{
  if ( !ds_real_ref_ok(ds,attnum) )
    my_error(mk_printf("check_ds_real_ref_ok: "
		       "attnum = %d (attname = %s): it wasn't",
		       attnum,ds_attnum_to_name(ds,attnum)));
}

void check_value_names_increasing(string_array *value_names)
{
  if ( !string_array_is_strictly_sorted(value_names) )
  {
    if (string_array_size(value_names) > 10000)
      my_error(mk_printf("check_value_names_increasing: a size %d "
			 "string_array is not strictly sorted:\n",
			 string_array_size(value_names)));
    else
      my_error(mk_printf("check_value_names_increasing: the following\n"
			 "string array is not strictly_sorted:\n%s\n",
			 mk_string_from_string_array(value_names)));
  }
}
#endif

/* This adds a new value name to the given attribute. After adding, 
   some of the small integer representations of values may change.

   This function ensures that after calling, all value_names for the
   given attribute remain in sorted increasing lexicographic order.

   PRE: ds_symbol_ref_ok(ds,attnum)

   This function is more efficient if the new value has a larger
   lexicographic value_name than any previous value_names, but despite
   that, if possible use ds_set_value_names instead of this if possible and
   if you don't care about losing the previous values.
*/
void ds_add_value_name(datset *ds,int attnum,char *value_name)
{
  int old_arity = ds_attnum_to_arity(ds,attnum);
  int new_arity = old_arity + 1;
  column *col = ds_attnum_to_column(ds,attnum);
  shape *sh = ds_shape(ds);

  /* if someone tries to add a value which is already there,
     the datset is falsely updated to a higher arity and it makes
     values out of bounds --pfm2 */
  if(shape_attnum_name_to_value(sh,attnum,value_name)<0)
  {
    bool can_insert_at_end =
      (old_arity == 0) || 
      strcmp(ds_attnum_value_to_name(ds,attnum,old_arity-1),value_name) < 0;
  
    column_set_arity(col,new_arity);

    if ( can_insert_at_end )
      shape_add_value_name(sh,attnum,value_name);
    else
    {
      int value = shape_insert_value_name(sh,attnum,value_name);
      int row;
      int num_rows = ds_num_rows(ds);

      for ( row = 0 ; row < num_rows ; row++ )
      {
        if ( !column_value_is_missing(col,row) )
        {
      	  int v = column_symbol_ref(col,row);
	        if ( v >= value )
	          column_symbol_set(col,row,v+1);
        }
      }
    }
  }
}

void ds_set_attname(datset *ds,int attnum,char *new_attname)
{
  shape_set_attname(ds_shape(ds),attnum,new_attname);
}
  
/* This deletes the value from the given attribute. After adding, 
   some of the small integer representations of values may change.

   This function ensures that after calling, all value_names for the
   given attribute remain in sorted increasing lexicographic order.

   PRE: ds_symbol_ref_ok(ds,attnum)

   Any datapoint in the dataset that previously had the given value,
   now has a missing value.
*/
void ds_delete_value(datset *ds,int attnum,int value)
{
  column *col = ds_attnum_to_column(ds,attnum);
  shape *sh = ds_shape(ds);

  column_delete_value(col,value);
  shape_delete_value(sh,attnum,value);
}

/* This function obliterates any previous values and redefines the
   value names (and implicitly the arity) of the attribute. After this
   call, all symbolic attribute values are 0.

   PRE: ds_symbol_ref_ok(ds,attnum)
*/
void ds_set_value_names(datset *ds,int attnum,string_array *value_names)
{
  string_array *sorted = mk_sort_string_array(value_names);
  int arity = string_array_size(sorted);
  column *col = ds_attnum_to_column(ds,attnum);
  shape *sh = ds_shape(ds);
  column_set_arity(col,arity);
  shape_set_value_names(sh,attnum,sorted);
  free_string_array(sorted);
}

/* Very efficiently changes all the value names of the 
   given (symbolic) attribute. If the i'th value name was
   previously "xxx" it now becomes "<prefix>xxx" */
void ds_valnames_prepend(datset *ds,int attnum,char *prefix)
{
  shape *sh = ds_shape(ds);
  ds_set_coltype(ds,attnum,SYMBOL_COL,UNDEFINED_COL);
  shape_valnames_prepend(sh,attnum,prefix);
}

/* Returns the number of rows of data in the datset. In the ds.h example
   would return 4. */
int ds_num_rows(const datset *ds)
{
  return(ds->num_rows);
}

/* Returns the number of attributes (i.e. columns) in the dataset.
   In the example would return 6 */
int ds_num_atts(const datset *ds)
{
  return(shape_num_atts(ds_shape((datset *) ds))); /* BAD_CAST */
}
void slow_check_not_missing(const datset *ds,int row,int attnum,char *op)
{
  if ( ds_value_is_missing(ds,row,attnum) )
    my_errorf("\n%s : Attempted to reference missing value\n"
	      "              row = %d\n"
              "              attribute = %s (attnum = %d)\n",
	      op,row,ds_attnum_to_name(ds,attnum),attnum);
}

#ifdef AMFAST
#define check_not_missing(ds,row,attnum,op) /* do nothing */
#else
#define check_not_missing(ds,row,attnum,op) slow_check_not_missing(ds,row,attnum,op)
#endif
void slow_check_real_ref_ok(const datset *ds,int row,int attnum,char *op)
{
  if ( !ds_real_ref_ok(ds,attnum) )
    my_errorf("\n%s : Attempted to reference a real-value on an\n"
	      "       all symbolic-values attribute\n"
	      "              row = %d\n"
              "              attribute = %s (attnum = %d)\n",
	      op,row,ds_attnum_to_name(ds,attnum),attnum);
}

#ifdef AMFAST
#define check_real_ref_ok(ds,row,attnum,op) /* do nothing */
#else
#define check_real_ref_ok(ds,row,attnum,op) slow_check_real_ref_ok(ds,row,attnum,op)
#endif

/* Returns the small-integer-value of the attnum'th attribute on
   the row'th row. In our example, ds_symbol_ref(ds,0,2) == 0,
   because the two symbol values are { "female" "male" }. We know
   they are in that order internally because female is before male
   in the alphabet. 

   PRE: ds_symbol_ref_ok(ds,attnum)
   PRE: ds_value_is_missing(ds,row,attnum) must be FALSE
*/
int ds_symbol_ref(const datset *ds,int row,int attnum)
{
  check_not_missing(ds,row,attnum,"ds_symbol_ref");
  return(column_symbol_ref(ds_attnum_to_column(ds,attnum),row));
}

/* Returns the real value of the attnum'th attribute in the
   row'th row. In our example ds_real_ref(3,1) == 98.0.
   PRE: ds_real_ref_ok(ds,attnum)
   PRE: ds_value_is_missing(ds,row,attnum) must be FALSE
*/
double ds_real_ref(const datset *ds,int row,int attnum)
{
#ifndef AMFAST
  check_not_missing(ds,row,attnum,"ds_real_ref");
  check_real_ref_ok(ds,row,attnum,"ds_real_ref");
#endif
  return(column_real_ref(ds_attnum_to_column(ds,attnum),row));
}

/* Sets the small-integer-value of the attnum'th attribute on
   the row'th row. 

   PRE: ds_symbol_ref_ok(ds,attnum)
   Note that after this, ds_value_is_missing(ds,row,attnum) will be
   FALSE no matter whether or not it was missing before.
*/
void ds_symbol_set(datset *ds,int row,int attnum,int value)
{
#ifdef DOING_ASSERTS
  if ( row < 0 || row >= ds_num_rows(ds) )
    my_errorf("Attempting to set row = %d of datset with %d rows\n",
	      row,ds_num_rows(ds));

  if ( value < 0 || value >= ds_attnum_to_arity(ds,attnum) )
    my_errorf("Attempting to set value = %d of attribute with arity = %d\n",
	      value,ds_attnum_to_arity(ds,attnum));
#endif

  column_symbol_set(ds_attnum_to_column(ds,attnum),row,value);
}

/* Sets the real value of the attnum'th attribute in the
   row'th row.
   PRE: ds_real_ref_ok(ds,attnum)
   Note that after this, ds_value_is_missing(ds,row,attnum) will be
   FALSE no matter whether or not it was missing before.
 */
void ds_real_set(datset *ds,int row,int attnum,double value)
{
  column_real_set(ds_attnum_to_column(ds,attnum),row,value);
}

/* The name of the attnum'th attribute */
char *ds_attnum_to_name(const datset *ds,int attnum)
{
  return(shape_attnum_to_name(ds_shape(ds),attnum));
}

/* Which of the symbolic value of the specified
   (symbolic) attribute has the given name? That's
   what this function returns. If no values have this
   name, the function returns -1 */
int ds_attnum_name_to_value(const datset *ds,int attnum,char *s)
{
  return shape_attnum_name_to_value(ds_shape(ds),attnum,s);
}

/* The arity of the attnum'th attribute. attnum MUST BE symbolic.  */
int ds_attnum_to_arity(const datset *ds,int attnum)
{
  return(shape_attnum_to_arity(ds_shape(ds),attnum)); /* UNCONST_CAST */
}

/* TRUE iff the attnum'th attribute is symbolic */
bool ds_attnum_to_symbolicp(const datset *ds,int attnum)
{
  return(shape_attnum_to_symbolicp(ds_shape(ds),attnum));
}

/* The name of the value'th value (alphabetically) of the
   attnum'th attribute. attnum MUST BE symbolic. */
char *ds_attnum_value_to_name(const datset *ds,int attnum,int value)
{
  return(shape_attnum_value_to_name(ds_shape(ds),attnum,value));
}

void ds_delete_attributes(datset *ds, string_array *sa)
{
  int i;
  for(i = 0; i < string_array_size(sa); i++)
  {
    char *name = string_array_ref(sa, i);
    int attnum = ds_name_to_attnum(ds, name);

    if(attnum < 0)
      my_errorf("failure to delete attribute: %s",name);
    else
      ds_delete_attribute(ds, attnum);
  }
}

void ds_delete_attribute(datset *ds,int attnum)
{
  int i;
  int num_atts = ds_num_atts(ds);

  if (Verbosity >=0) {
    printf("Will delete %s (attnum %d)\n",ds_attnum_to_name(ds,attnum), attnum);
  }


  shape_delete_attribute(ds_shape(ds),attnum);
  free_column(ds->cols_array[attnum] /*ds_attnum_to_column(ds,attnum)*/ );
  for ( i = attnum ; i < num_atts-1 ; i++ )
    ds->cols_array[i] = ds->cols_array[i+1];
}

/* This swaps the order of the attributes with attnums att1 and att2.
   Precondition: att1 != att2, and both are valid attnums for ds. */
void ds_swap_attributes(datset *ds,int att1,int att2)
{
  if (att1 == att2)
        my_error("ds_swap_attributes: att1 must not equal att2");
  else
  {
    column *tem;

    /* Swap the description of the attributes */
    shape_swap_attributes(ds_shape(ds),att1,att2); 
    /* Now swap the data columns */
    tem = ds_attnum_to_column(ds,att1);
    ds->cols_array[att1] = ds_attnum_to_column(ds,att2);
    ds->cols_array[att2] = tem;
  }
}

void ds_rename_attribute(const datset *ds,int attnum,char *newname)
{
  shape_rename_attribute(ds_shape(ds),attnum,newname);
}

void ds_merge_value(datset *ds,int attnum,int oldvalue,int newvalue)
{
  int i;
  if( newvalue < 0 || newvalue >= ds_attnum_to_arity(ds,attnum) )
    my_error("ds_rename_value: illegal new value!");

  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      int this_value = ds_symbol_ref(ds,i,attnum);
      if ( this_value == oldvalue )
        ds_symbol_set(ds,i,attnum,newvalue);
    }
  }
  ds_delete_value(ds,attnum,oldvalue);
}

void ds_rename_value(datset *ds,int attnum,int value,char *newname)
{
  if ( ds_attnum_name_to_value(ds,attnum,newname) >= 0 )
    my_error("ds_rename_value: name already exists!");
  else
  {
    char *oldname = mk_copy_string(ds_attnum_value_to_name(ds,attnum,value));
    int oldvalue;
    int newvalue;
    int i;
    ds_add_value_name(ds,attnum,newname);
       /* Be careful! The above call might change the value <--> name map! */

    newvalue = ds_attnum_name_to_value(ds,attnum,newname);
    oldvalue = ds_attnum_name_to_value(ds,attnum,oldname);
    for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
    {
      if ( !ds_value_is_missing(ds,i,attnum) )
      {
        int this_value = ds_symbol_ref(ds,i,attnum);
        if ( this_value == oldvalue )
          ds_symbol_set(ds,i,attnum,newvalue);
      }
    }
    ds_delete_value(ds,attnum,oldvalue);
    free_string(oldname);
  }
}

void column_set_from_string(loadinst *ldi,colshape *cs,column *col,
			    int row,char *s,char **r_errmess)
{
  bool missing = ds_ignore_token(ldi,s);
  *r_errmess = NULL;

  if ( colshape_both_refs_ok(cs) )
    my_error("column_set_from_string: only applicable to columns\n"
	     "with undefined minor coltype");
  else if ( missing )
    column_set_value_missing(col,row);
  else
  {
    if ( !colshape_is_symbolic(cs) )
    {
      double x = colshape_real_value_from_string(cs,s,r_errmess);

      if ( *r_errmess == NULL )
        column_real_set(col,row,x);
    }
    else
    {
      int x = colshape_symbol_value_from_string(cs,s,r_errmess);
      if ( *r_errmess == NULL )
        column_symbol_set(col,row,x);
    }
  }
}

void fill_datset_row(loadinst *ldi,datset *ds,string_array *tokens,
                     int row,char **r_errmess)
{
  int i;

  /* if number of tokens on a line has less than the number of attributes,
     fill tokens with missing values. */
  if(ds_num_atts(ds) > string_array_size(tokens))
  {
    int diff = ds_num_atts(ds) - string_array_size(tokens);
    
    for(i = 0; i < diff; i++)    
      add_to_string_array(tokens,"?");
  }

  for ( i = 0 ; *r_errmess == NULL && i < string_array_size(tokens) ; i++ )
    column_set_from_string(ldi,
                           shape_colshape_ref(ds_shape(ds),i),
                           ds_attnum_to_column(ds,i),
                           row,
                           string_array_ref(tokens,i),
                           r_errmess);
}

void fill_datset_from_file(const char *filename,loadinst *ldi,datset *ds,
                           char **r_errmess)
{
  FILE *s = fopen_with_errmess(filename,"r",r_errmess);

  if ( *r_errmess == NULL )
  {
    string_array *attnames;
    string_array *tokens;
    int row = 0;
    int line = 0;
    bool finished = FALSE;
    int num_ignored = 0;

    get_attnames_and_first_tokens(s,&attnames,&tokens,ldi->line_format,
                                  &line,r_errmess);

    if ( *r_errmess == NULL && 
         !(ldi->ignore_missing && ds_ignore_tokens(ldi,tokens)) &&
         !(ldi->ignore_short && string_array_size(tokens) < ds_num_atts(ds)) )
    {
      fill_datset_row(ldi,ds,tokens,row,r_errmess);
      row += 1;
    }
    else
      num_ignored += 1;

    while ( !finished && *r_errmess == NULL )
    {
      free_string_array(tokens);
      tokens = mk_next_tokens(s,&line,ldi->line_format);
      if ( tokens == NULL )
        finished = TRUE;
      else if ( ldi->ignore_missing && ds_ignore_tokens(ldi,tokens) )
        num_ignored += 1;
      else if ( ldi->ignore_short && string_array_size(tokens) < ds_num_atts(ds) )
        num_ignored += 1;
      else 
      {
        fill_datset_row(ldi,ds,tokens,row,r_errmess);
        row += 1;
        if ((Verbosity >= 0) && (row > 10 && is_power_of_two(row)))
          printf("Datset load (pass 2) loaded %d lines\n",row);
      }
    }

    if ( attnames != NULL ) free_string_array(attnames);
    if ( tokens != NULL ) free_string_array(tokens);

    if ( *r_errmess != NULL )
    {
      char buff[500];
      sprintf(buff,"Error on line %d of file %s: %s",
              line,filename,*r_errmess);
      free_string(*r_errmess);
      *r_errmess = mk_copy_string(buff);
    }

    if ( *r_errmess == NULL && row != ds_num_rows(ds) )
    {
      char buff[1000];
      sprintf(buff,"I expected to find %d rows in the file. I only found %d",
              ds_num_rows(ds),row);
      *r_errmess = mk_copy_string(buff);
    }

    if ( *r_errmess == NULL && num_ignored > 0 )
    {
      printf("\nWarning! I had to ignore %d line(s) because of missing values\n",
             num_ignored);
    }
  }

  if ( s != NULL ) fclose(s);
}

datset *mk_datset_from_file_and_shape(const char *filename,loadinst *ldi,
                                      shape *sh,int num_rows,
                                      char **r_errmess)
{
  datset *ds = mk_zero_datset(sh,num_rows);
  fill_datset_from_file(filename,ldi,ds,r_errmess);
  if ( *r_errmess != NULL )
  {
    free_datset(ds);
    ds = NULL;
  }
  return(ds);
}

/* Loads a file from the specified file, using instructions from the
   loadinst *ldi structure for deciding things like whether the lines
   are space separated, comma separated (or should be automatically decided)
   and whether records with missing values are to be ignored and if not,
   what the default missing value should be.  If you don't have a loadinst
   handy, just pass in NULL.

   If there are no problems loading, builds and returns the datset.
   and sets *r_errmess == NULL.

   If there are problems loading returns NULL, but makes and returns
   a string in *r_errmess describing the loading problem. The caller
   must eventually free *r_errmess. */
datset *mk_datset_from_file(const char *filename,loadinst *ldi,char **r_errmess)
{
  int num_rows;
  datset *ds = NULL;
  shape *sh = NULL;
  bool ldi_passed_in = TRUE;

  if (!ldi) {
    ldi_passed_in = FALSE;
    ldi = mk_default_loadinst();
  }

  sh = mk_shape_deduced_from_file(ldi,filename,&num_rows,r_errmess);

  /* If the shape failed on the first try, we'll give it one more chance
     because the low-level functions update ldi in an attempt to make
     things work on the second try.
  */
  if (!sh) {
    if (*r_errmess) {
      free_string(*r_errmess);
      *r_errmess = NULL;
    }
    sh = mk_shape_deduced_from_file(ldi,filename,&num_rows,r_errmess);
  }
  
  if ( sh != NULL ) {
    ds = mk_datset_from_file_and_shape(filename,ldi,sh,num_rows,r_errmess);
    free_shape(sh);
  }

  if (!ldi_passed_in) free_loadinst(ldi);
  return(ds);
}

/* return TRUE iff the filename ends in .csv. Commented this out because with
 * the changes to mk_datset_from_file_simplish(), it's now unused. */
bool csv_file(const char *filename)
{
  int len = strlen(filename);

  if ((len > 4) && (caseless_eq_string(filename+len-4,".csv")))
    return TRUE;
  else
    return FALSE;
}

/* Just like mk_datset_from_file_simple except uses the standard
  *r_errmess error message formalism.
*/
datset *mk_datset_from_file_simplish(const char *filename,
				     int argc,char *argv[],char **r_errmess)
{
  datset *ds = NULL;
  loadinst *ldi = mk_loadinst_from_args(argc,argv);
  /* Andrew commented the following out 9/21/2: It seems likely to
     cause more confusion than help. If someone has a space-separated file
     that they ask to load but it has a .csv suffix, I'd like it to still 
     work. */

  /*
  if (csv_file(filename) && (index_of_arg("format",argc,argv) < 0))
    ldi->line_format = COMMA_FORMAT;
  */
  ds = mk_datset_from_file(filename,ldi,r_errmess);
  free_loadinst(ldi);
  return(ds);
}

/* Loads a file from the specified file, assuming the datset file format
   described at the bottom of datset.h. 
   If there are no problems loading, builds and returns the datset.
   If there are problems, prints an error message to stderr and aborts
   the entire program. 

   You can set argc == 0 and argv == NULL for default simple behavior.

   If you use a "real" command-line argc and argv, then the loader
   will search for the following command-line parameters:

     [format <string>] where string is one of

          space : use whitespace as the separator between values
          comma : use comma's as the separator (.csv style)
          auto : automatically choose between the above two (this is default)

     [ignore_missing <string>] where string is one of
          true  : Throw away any rows that contain missing data. (this is
                  the default)
          false : Load rows with missing data. Symbolic attributes
                  get assigned the special value "?". Numeric
                  attributes get given a default value <missing_numeric_value>
                  which by default is -9999.0

    [missing_numeric_value <number>] See above.
*/
datset *mk_datset_from_file_simple(const char *filename,int argc,char *argv[])
{
  char *errmess;
  datset *ds = mk_datset_from_file_simplish(filename,argc,argv,&errmess);
  if ( ds == NULL )
  {
    fprintf(stderr,"\n*** ERROR LOADING DATSET FROM %s ***\n\n%s\n\n",
            filename,errmess);
    my_error("mk_datset_from_file_simple");
  }
  return(ds);
}

/* Takes a symbolic attribute.
   And makes a numeric column
   with the appropriate numbers throughout. 
   real_value = symbol_val_to_real_val[symbol_value]

   PRE: ds_symbol_ref_ok(ds,attnum) and NOT ds_real_ref_ok(ds,attnum)
        to start with.

   POST: Attnum is real-valued major coltype, symbol-valued minor
         coltype. 
*/
void ds_define_real(datset *ds,int attnum,dyv *symbol_val_to_real_val)
{
  int i;
  check_ds_symbol_ref_ok(ds,attnum);
  ds_set_coltype(ds,attnum,REAL_COL,SYMBOL_COL);
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      int symbol_value = ds_symbol_ref(ds,i,attnum);
      double real_value = dyv_ref(symbol_val_to_real_val,symbol_value);
      ds_real_set(ds,i,attnum,real_value);
    }
  }
}

/* Takes a symbolic attribute in which all the values
   have names that are numbers. And makes a numeric column
   with the appropriate numbers throughout. 

   PRE: All the values must be parseable as numbers. attnum must be symbolic
        to start with.
   PRE: ds_symbol_ref_ok(ds,attnum) and NOT ds_real_ref_ok(ds,attnum)
        to start with.

   POST: Attnum is real-valued major coltype, symbol-valued minor
         coltype. 
*/
void ds_realize(datset *ds,int attnum)
{
  int arity = ds_attnum_to_arity(ds,attnum);
  dyv *symbol_val_to_real_val = mk_dyv(arity);
  int i;
  for ( i = 0 ; i < arity ; i++ )
  {
    char *value_name = ds_attnum_value_to_name(ds,attnum,i);
    if ( !is_a_number(value_name) )
      my_error("ds_convert_symbol_to_real: non-number value");
    else
    {
      double value = atof(value_name);
      dyv_set(symbol_val_to_real_val,i,value);
    }
  }
  ds_define_real(ds,attnum,symbol_val_to_real_val);
  free_dyv(symbol_val_to_real_val);
}

bool string_array_is_strictly_sorted(string_array *sa)
{
  int i;
  bool result = TRUE;

  for ( i = 0 ; result && i < string_array_size(sa) - 1 ; i++ )
  {
    char *s0 = string_array_ref(sa,i+0);
    char *s1 = string_array_ref(sa,i+1);
    result = strcmp(s0,s1) < 0;
  }

  return result;
}

/* PRE: ds_real_ref_ok(ds,attnum) && value_names is sorted in lexicographically
        increasing order.

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

	 forall row,
          ds_symbol_ref(ds,row,attnum) == symbol_value_fn(ds,row,attnum,data)

	 and the symbolic value names are value_names
*/  
void ds_define_symbolic(datset *ds,int attnum,
			string_array *value_names,
			int (*symbol_value_fn)(datset *ds,int row,int attnum,
					       void *data),
			void *data)
{
  int i;
  check_ds_real_ref_ok(ds,attnum);
  check_value_names_increasing(value_names);
  ds_set_coltype(ds,attnum,SYMBOL_COL,REAL_COL);
  ds_set_value_names(ds,attnum,value_names);
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      int symbol_value = symbol_value_fn(ds,i,attnum,data);
      ds_symbol_set(ds,i,attnum,symbol_value);
    }
  }
}
int named_breaks_symbol_value(datset *ds,int row,int attnum,void *data)
{
  dyv *breaks = (dyv *) data;
  int sv;
  double rv = ds_real_ref(ds,row,attnum);
 
  for ( sv = 0 ; sv < dyv_size(breaks) && dyv_ref(breaks,sv) < rv ; sv++ )
  { 
    /* skip */
  }
  
  return sv;
}

/* PRE: ds_real_ref_ok(ds,attnum) && value_names is sorted in lexicographically
        increasing order.
        string_array_size(value_names) == dyv_size(breaks) + 1
	breaks is in strictly increasing order

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

forall row,
   ds_symbol_ref(ds,row,attnum) == 0 if ds_real_ref(ds,row,attnum) < breaks[0]
else
   ds_symbol_ref(ds,row,attnum) == 
     1 + min_i such that ds_real_ref(ds,row,attnum) >= breaks[i]
*/  
void ds_symbolize_with_named_breaks(datset *ds,int attnum,
				    string_array *value_names,
				    dyv *breaks)
{
  ds_define_symbolic(ds,attnum,value_names,
		     named_breaks_symbol_value,(void *)breaks);
}

int num_digits_in_integer(int x)
{
  char buff[100];
  sprintf(buff,"%d",x);
  return (int) strlen(buff);
}

string_array *mk_breaks_value_names(dyv *breaks)
{
  string_array *value_names = mk_string_array(0);
  int num_breaks = dyv_size(breaks);
  int arity = num_breaks+1;
  int num_digits = num_digits_in_integer(arity-1);
  char format_lo[100];
  char format_hi[100];
  char buff[100];
  int i;

  sprintf(format_lo,"v%%0%dd:%%g-",num_digits);
  sprintf(format_hi,"v%%0%dd:%%g+",num_digits);

  sprintf(buff,format_lo,0,dyv_ref(breaks,0));
  add_to_string_array(value_names,buff);

  for ( i = 0 ; i < dyv_size(breaks) ; i++ )
  {
    sprintf(buff,format_hi,i+1,dyv_ref(breaks,i));
    add_to_string_array(value_names,buff);
  }
  
  return value_names;
}

/* PRE: ds_real_ref_ok(ds,attnum) 
	breaks is in strictly increasing order

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

forall row,
   ds_symbol_ref(ds,row,attnum) == 0 if ds_real_ref(ds,row,attnum) < breaks[0]
else
   ds_symbol_ref(ds,row,attnum) == 
     1 + min_i such that ds_real_ref(ds,row,attnum) >= breaks[i]

   And the value_name of the i'th symbol is a name denoting the i'th bucket.
*/  
void ds_symbolize_with_breaks(datset *ds,int attnum,dyv *breaks)
{
  string_array *value_names = mk_breaks_value_names(breaks);
  ds_symbolize_with_named_breaks(ds,attnum,value_names,breaks);
  free_string_array(value_names);
}

void save_datset_fast(FILE *s,datset *ds)
{
  int i;
  save_shape(s,ds_shape(ds));
  fprintf(s,"rows = %d\n",ds_num_rows(ds));
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    save_column_fast(s,ds_attnum_to_column(ds,i));
}

/* The version number was added to handled multiple kinds of files.  The
   versions existing so far are:
   version 0 - crashes and/or ignores missing values
   version 1 - handles missing values
   Except for version 0, the version number is given in the first line of
   the file.
*/
datset *mk_datset_from_stream_fast(FILE *s,int *line,char **r_errmess)
{
  int version = 0;
  shape *sh = mk_shape_from_stream(s,line,&version,r_errmess);
  datset *ds = NULL;
  int i;

  if ( *r_errmess == NULL )
  {
    string_array *sa = mk_next_interesting_line(s,line);
    if ( sa == NULL || 
         string_array_size(sa) != 3 ||
         !is_a_number(string_array_ref(sa,2)) )
      *r_errmess = mk_copy_string("Expected rows = <number>");
    else
    {
      int num_rows = atoi(string_array_ref(sa,2));
      ds = mk_zero_datset(sh,num_rows);
    }
    if ( sa != NULL ) free_string_array(sa);

    for ( i = 0 ; i < ds_num_atts(ds) && *r_errmess == NULL ; i++ )
    {
      shape *my_sh = ds_shape(ds);
      colshape *cs = shape_colshape_ref(my_sh,i);
      column *col = 
        mk_column_from_stream_fast(s,cs,line,version,r_errmess);
      if ( col != NULL ) 
      {
      	free_column(ds_attnum_to_column(ds,i));
	      ds->cols_array[i] = col;
      }
    }
  }

  if ( *r_errmess != NULL && ds != NULL )
  {
    free_datset(ds);
    ds = NULL;
  }

  if ( sh != NULL ) free_shape(sh);

  return(ds);
}

datset *mk_datset_from_file_fast(const char *filename,char **r_errmess)
{
  FILE *s = fopen_with_errmess(filename,"r",r_errmess);
  datset *ds = NULL;

  if ( s != NULL )
  {
    int line = 0;
    ds = mk_datset_from_stream_fast(s,&line,r_errmess);
    if ( ds == NULL )
    {
      char buff[2000];
      sprintf(buff,"Error reading dataset around line %d of file %s:\n%s",
              line,filename,*r_errmess);
      free_string(*r_errmess);
      *r_errmess = mk_copy_string(buff);
    }

    fclose(s);
  }
  return(ds);
}

datset *mk_datset_from_file_fast_simple(const char *filename)
{
  char *errmess = NULL;
  datset *ds = mk_datset_from_file_fast(filename,&errmess);
  if ( ds == NULL )
  {
    fprintf(stderr,"ERROR READING DATASET\n\n%s\n",errmess);
    my_error("mk_datset_from_file_fast_simple");
  }
  return(ds);
}

datset *mk_datset_from_xml_stream(FILE* s, int *line)
{
  char *errmess = NULL;
  int version = 1;
  shape *sh = mk_shape_from_stream(s,line,&version,&errmess);
  datset *ds = NULL;
  int i;

  set_base_encoding_to_xml();

  if ( errmess == NULL )
  {
    string_array *sa = mk_next_interesting_line(s,line);
    if ( sa == NULL || 
         string_array_size(sa) != 3 ||
         !is_a_number(string_array_ref(sa,2)) )
      errmess = mk_copy_string("Expected rows = <number>");
    else
    {
      int num_rows = atoi(string_array_ref(sa,2));
      ds = mk_zero_datset(sh,num_rows);
    }
    if ( sa != NULL ) free_string_array(sa);

    for ( i = 0 ; i < ds_num_atts(ds) && errmess == NULL ; i++ )
    {
      shape *my_sh = ds_shape(ds);
      colshape *cs = shape_colshape_ref(my_sh,i);
      column *col = mk_column_from_stream_fast(s,cs,line,version,&errmess);
      if ( col != NULL ) 
      {
      	free_column(ds_attnum_to_column(ds,i));
	ds->cols_array[i] = col;
      }
    }
  }

  if ( errmess != NULL && ds != NULL )
  {
    printf("\n==========\nERROR: %s\n==========\n", errmess);
    free_datset(ds);
    ds = NULL;
  }

  if ( sh != NULL ) free_shape(sh);

  set_base_encoding_to_normal();

  return(ds);
}


/***** Basic Operations Related To Missing Values *****/

/* MISSING VALUES: For every element in the dataset (i.e. every location
                   that is indexed by a row-attribute pair) it is possible
                   that instead of having a value, the element has a "MISSING
                   VALUE". The next few functions allow the programmer to
                   test for and manipulate missing values. The most important
                   thing about missing values is that you cannot call
                   "ds_symbol_ref" or "ds_real_ref" on a location with missing
                   value. If you do, there will be a my_error. We recommend
                   that if you are writing code that doesn't want to consider
                   missing values, you simply put a call like this....

                           if ( ds_has_missing_values(ds) )
                             my_error("No missing values allowed for D-trees");

                   at the start of your top-level function.
*/

/* Returns TRUE if and only if the specified attribute
   on the specified row has a missing value. */
bool ds_value_is_missing(const datset *ds,int row,int attnum)
{
  return column_value_is_missing(ds_attnum_to_column(ds,attnum),row);
}

/* Tells you how many rows in the datset have the specified
   attribute number missing. */
int ds_attnum_to_num_missing(datset *ds,int attnum)
{
  return ds_attnum_to_column(ds,attnum)->num_missing;
}

/* Sets the datset so that the specified row/attribute gets
   given a missing value. After doing this, you may not use
   ds_symbol_ref or ds_real_ref to access this element of the
   datset. If you want to eventually make this element non-missing,
   you must eventually give it a value with ds_symbol_set or
   ds_symbol_ref */
void ds_set_value_missing(datset *ds,int row,int attnum)
{
  column_set_value_missing(ds_attnum_to_column(ds,attnum),row);
}

/* True if and only if the datset has missing values. This is useful
   for you to call just before you run an algorithm that will choke
   on missing values. */
bool ds_has_missing_values(datset *ds)
{
  bool result = FALSE;
  int attnum;
  for ( attnum = 0 ; !result && attnum < ds_num_atts(ds) ; attnum++ )
    result = ds_attnum_to_num_missing(ds,attnum) > 0;
  return result;
}

/* The number of missing values of the given row */
int ds_row_to_num_missing(datset *ds,int row)
{
  int result = 0;
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    if ( ds_value_is_missing(ds,row,attnum) )
      result += 1;

  return result;
}

bool ds_row_has_missing_values(datset *ds,int row)
{
  bool result = FALSE;
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    if ( ds_value_is_missing(ds,row,attnum) )
    {
      result = TRUE;
      break;
    }
  return result;
}
void ds_delete_last_n_rows(datset *ds,int n)
{
  int i;
#ifndef AMFAST
  if ( n < 0 || n > ds_num_rows(ds) )
    my_error("ds_delete_last_n_rows: bad n");
#endif

  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    column_delete_last_n_records(ds_attnum_to_column(ds,i),n);
  ds->num_rows -= n;
}

void ds_keep_subset_rows(datset *ds, ivec *subset)
{
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    column_keep_subset(ds_attnum_to_column(ds,i),subset);
  ds->num_rows = ivec_size(subset);
}

/* Adds the specified number of rows at the end of the dataset. Sets the
   value of all new symbolic entries to value == (int) 0, and for all
   real entries to value == 0.0 */
void ds_add_rows(datset *ds,int num_rows)
{
  int i;

  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    column_add_records(ds_attnum_to_column(ds,i),num_rows);
  ds->num_rows += num_rows;
}

#define MISSING_VALUE_NAME "*MissingValue*"

/* If its not already present, adds a new symbol for MISSING_VALUE.
   Returns the index of this symbol.  If you call this with a non-symbolic
   column, you will suffer miserably.

   Currently does not ensure that *missingvalue* goes in at the right
   lexicographic position. But soon will.
*/
int add_missing_symbol(datset *ds,int attnum)
{
  ds_add_value_name(ds,attnum,MISSING_VALUE_NAME);
  return ds_attnum_name_to_value(ds,attnum,MISSING_VALUE_NAME);
}

/* If you call this with a non-symbolic column, you will suffer miserably. */
int missing_symbol_value(datset *ds,int attnum)
{
  return ds_attnum_name_to_value(ds,attnum,MISSING_VALUE_NAME);
}

/* For the attnum'th symbolic attribute, add a new symbol value 
   MISSING_VALUE_NAME and change all the missing values to have that value.
   WARNING!! any real values WILL BE DELETED!
*/
void convert_missing_to_symbol(datset *ds,int attnum)
{
  int i;
  int missing_value;
  pivec *np;
  column *col;

  if(ds_real_ref_ok(ds,attnum)) ds_set_coltype(ds,attnum,SYMBOL_COL,UNDEFINED_COL);

  missing_value = add_missing_symbol(ds,attnum);
  col = ds_attnum_to_column(ds,attnum);

  /* we've updated the shape, but that doesn't mean the pivec in the data
     has updated its local size as well, so we fix it if necessary */
  if (ds_attnum_to_arity(ds,attnum) > pivec_max_value(col->symbol_values)+1)
  {
    np = mk_pivec(pivec_size(col->symbol_values),
		  pivec_max_value(col->symbol_values)+1);
    for (i=0;i<pivec_size(np);i++) 
      pivec_set(np,i,pivec_ref(col->symbol_values,i));
    free_pivec(col->symbol_values);
    col->symbol_values = np;
  }

  /* now set the missing values to the new value */
  if (col->missing)
    for (i=0;(col->missing)&&(i<pivec_size(col->missing));i++)
      if (pivec_ref(col->missing,i)) ds_symbol_set(ds,i,attnum,missing_value);
}
/* -------------- this is for symbolic attributes ---------- */
void replace_missingvals_with_symbol(datset *ds,int attnum,char *symbol)
{
  int i,v;
  pivec *np;
  column *col;

  if(ds_real_ref_ok(ds,attnum)) 
    ds_set_coltype(ds,attnum,SYMBOL_COL,UNDEFINED_COL);

  v = ds_attnum_name_to_value(ds,attnum,symbol);
  if( v < 0 )
  {
    ds_add_value_name(ds,attnum,symbol);
    v = ds_attnum_name_to_value(ds,attnum,symbol);
  }
  col = ds_attnum_to_column(ds,attnum);

  /* we've updated the shape, but that doesn't mean the pivec in the data
     has updated its local size as well, so we fix it if necessary */
  if (ds_attnum_to_arity(ds,attnum) > pivec_max_value(col->symbol_values)+1)
  {
    np = mk_pivec(pivec_size(col->symbol_values),
		  pivec_max_value(col->symbol_values)+1);
    for (i=0;i<pivec_size(np);i++) 
      pivec_set(np,i,pivec_ref(col->symbol_values,i));
    free_pivec(col->symbol_values);
    col->symbol_values = np;
  }

  /* now set the missing values to the new value */
  if (col->missing)
  {
    for (i=0;(col->missing)&&(i<pivec_size(col->missing));i++)
    {
      if (pivec_ref(col->missing,i)) 
        ds_symbol_set(ds,i,attnum,v);
    }
  }
  free_pivec(col->missing);
  col->missing = NULL;
  col->num_missing = 0;
}
/* -------------- this is for real attributes ---------- */
void replace_missingvals_with_number(datset *ds,int attnum,double val)
{
  int i;
  column *col;

  if(ds_symbol_ref_ok(ds,attnum)) 
    ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);

  col = ds_attnum_to_column(ds,attnum);
  if (col->missing)
  {
    for (i=0;(col->missing)&&(i<pivec_size(col->missing));i++)
    {
      if( pivec_ref(col->missing,i) ) 
        ds_real_set(ds,i,attnum,val);
    }
  }
  free_pivec(col->missing);
  col->missing = NULL;
  col->num_missing = 0;
}

/* -------------- this is for real attributes ---------- */
void replace_number_with_missingvals(datset *ds,int attnum,double val)
{
  int i;

  if(ds_symbol_ref_ok(ds,attnum)) 
    ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);
  
  for (i=0;i<ds_num_rows(ds);i++)
  {
    if( !ds_value_is_missing(ds,i,attnum) && (ds_real_ref(ds,i,attnum)==val) ) 
      ds_set_value_missing(ds,i,attnum);
  }
  return;
}
/* -------------- this is for symbolic attributes ---------- */
void replace_symbol_with_missingvals(datset *ds,int attnum,char *symbol)
{
  int i,val;

  if(ds_real_ref_ok(ds,attnum)) 
    ds_set_coltype(ds,attnum,SYMBOL_COL,UNDEFINED_COL);

  val = ds_attnum_name_to_value(ds,attnum,symbol);
  if( val >= 0 )
  {
    for (i=0;i<ds_num_rows(ds);i++)
    {
      if( !ds_value_is_missing(ds,i,attnum) && (ds_symbol_ref(ds,i,attnum)==val) ) 
        ds_set_value_missing(ds,i,attnum);
    }
  }
  return;
}
