/*
   File:        shape.c
   Author:      Andrew W. Moore
   Created:     Sat Aug 14 12:50:02 EDT 1999
   Description: SPR/Auton Shape of a datset Structure

   Copyright 1999, Schenley Park Research
*/

#include "shape.h"

/* Private prototypes */
static void shape_require_extra_column(shape *sh);
static bool is_parseable_as_a_symbol(char *s);
static void update_colshape_from_token(loadinst *ldi,colshape *cs, bool *r_sbp, char *token,char **r_errmess);
static void update_shape_from_tokens(loadinst *ldi,shape *sh, ivec *attnum_to_sbp,string_array *tokens, int *r_num_rows,char **r_errmess);
static void sort_attribute_value(shape *sh,int attnum);

#ifdef AUTON_UNUSED_CODE
static string_array *mk_parse_symbolic_key(int argc,char *argv[]);
#endif /* AUTON_UNUSED_CODE */

void free_shape(shape *x)
{
  int i;
  for ( i = 0 ; i < x -> num_atts ; i++ )
    free_colshape(shape_colshape_ref(x,i));
  AM_FREE_ARRAY(x->cs_array,colshape *,x->array_size);
  AM_FREE(x,shape);
}

bool shape_equal(shape *s1,shape *s2)
{
  bool result = s1->num_atts == s2 -> num_atts;
  int i;
  for ( i = 0 ; result && i < s1->num_atts ; i++ )
    result = colshape_equal(shape_colshape_ref(s1,i),shape_colshape_ref(s2,i));
  return result;
}



shape *mk_copy_shape(const shape *x)
{
  shape *newx = AM_MALLOC(shape);
  int i;
  newx -> num_atts = x -> num_atts;
  newx -> array_size = x -> array_size;
  newx -> cs_array = AM_MALLOC_ARRAY(colshape *,x->array_size);
  for ( i = 0 ; i < x->num_atts ; i++ )
    newx -> cs_array[i] = mk_copy_colshape(shape_colshape_ref(x,i));
  return(newx);
}



void fprintf_shape(FILE *s,char *m1,shape *x,char *m2)
{
  char buff[200];
  int i;
  sprintf(buff,"%s -> num_atts",m1);
  fprintf_int(s,buff,x->num_atts,m2);
  for ( i = 0 ; i < x->num_atts ; i++ )
  {
    sprintf(buff,"%s -> cs[%d]",m1,i);
    fprintf_colshape(s,buff,shape_colshape_ref(x,i),m2);
  }
}



string_array *mk_attnames_from_tokens(string_array *tokens)
{
  string_array *result;
  int i;
  if ((string_array_size(tokens)>0)&&((string_array_ref(tokens,0))[0] == '#'))
  {
    result = mk_string_array(string_array_size(tokens)-1);
    for (i=0;i<string_array_size(result);i++)
      string_array_set(result,i,mk_copy_string(string_array_ref(tokens,i+1)));
  }
  else result = mk_copy_string_array(tokens);
  return result;
}

void get_attnames_and_first_tokens(FILE *s,
                                   string_array **r_attnames,
                                   string_array **r_tokens,
                                   int line_format,
                                   int *line_number,
                                   char **r_errmess)
{
  char *line_string = mk_string_from_line(s);
  string_array *attnames = NULL;
  string_array *first_data_line = NULL;
  bool this_line_is_data = FALSE;

  *line_number += 1;
  *r_errmess = NULL;
  
  if ( *r_errmess == NULL && line_string == NULL )
    *r_errmess = mk_copy_string("Unexpected EOF");

  while ( line_string != NULL && *r_errmess == NULL && !this_line_is_data ) 
  {
    if ( line_string_is_interesting(line_string) )
      this_line_is_data = TRUE;

    if ( !this_line_is_data )
    {
      free_string(line_string);
      line_string = mk_string_from_line(s);
      *line_number += 1;
    }
  }

  if ( *r_errmess == NULL && line_string == NULL )
    *r_errmess = mk_copy_string("Unexpected end of file");

  if ( *r_errmess == NULL )
  {
    string_array *tokens =
      mk_parse_data_line(line_string,line_format);

    if ( contains_a_number(tokens) )
    {
      attnames = mk_default_attribute_names(string_array_size(tokens));
      first_data_line = tokens;
    }
    else 
    {
      attnames = mk_attnames_from_tokens(tokens);
      free_string_array(tokens);
      first_data_line = mk_next_tokens(s,line_number,line_format);
    }
  }

  if ( *r_errmess == NULL && first_data_line == NULL )
  {
    *r_errmess = mk_copy_string("The file ended before I had a chance"
                                "To load the attribute names and the"
                                "first line of data.");
  }

  if ( *r_errmess == NULL )
  {
    int num_atts = string_array_size(attnames);
    int num_toks = string_array_size(first_data_line);
    if ( num_atts != num_toks )
    {
      char buff[1000];
      sprintf(buff,"First file line (att names) different length (%d) from first data line (%d)",
              num_atts,num_toks);
      *r_errmess = mk_copy_string(buff);
    }
  }

  if ( *r_errmess == NULL )
  {
    *r_attnames = attnames;
    *r_tokens = first_data_line;
  }
  else
  {
    *r_attnames = NULL;
    *r_tokens = NULL;
    if ( attnames != NULL ) free_string_array(attnames);
    if ( first_data_line != NULL ) free_string_array(first_data_line);
  }

  if ( line_string != NULL ) free_string(line_string);
}



shape *mk_shape(string_array *attnames)
{
  shape *sh = AM_MALLOC(shape);
  int i;
  sh -> num_atts = string_array_size(attnames);
  sh -> array_size = sh -> num_atts;
  sh -> cs_array = AM_MALLOC_ARRAY(colshape *,sh->array_size);
  for ( i = 0 ; i < sh->num_atts ; i++ )
  {
    sh->cs_array[i] = mk_colshape(string_array_ref(attnames,i),SYMBOL_COL,UNDEFINED_COL);
  }
  return(sh);
}

shape* mk_shape_real(string_array* attnames)
{
  shape *sh = AM_MALLOC(shape);
  int i;
  sh -> num_atts = string_array_size(attnames);
  sh -> array_size = sh -> num_atts;
  sh -> cs_array = AM_MALLOC_ARRAY(colshape *,sh->array_size);
  for ( i = 0 ; i < sh->num_atts ; i++ )
  {
    sh->cs_array[i] = mk_colshape(string_array_ref(attnames,i),REAL_COL,UNDEFINED_COL);
  }
  return(sh);
}


colshape *slow_shape_colshape_ref(const shape *sh,int attnum)
{
  if ( attnum < 0 || attnum >= shape_num_atts(sh) )
    my_error("slow_shape_colshape_ref");
  return sh->cs_array[attnum];
}

void slow_shape_check_coltype(const shape *sh,int attnum,int coltype)
{
  check_colshape_ref_ok(shape_colshape_ref(sh,attnum),coltype);
}


int shape_num_atts(const shape *sh)
{
  return(sh->num_atts);
}

char *shape_attnum_to_name(const shape *sh,int attnum)
{
  return colshape_name(shape_colshape_ref(sh,attnum));
}

/* Which of the symbolic value of the specified
   (symbolic) attribute has the given name? That's
   what this function returns. If no values have this
   name, the function returns -1 */
int shape_attnum_name_to_value(const shape *sh,int attnum,char *s)
{
  return colshape_name_to_value(shape_colshape_ref(sh,attnum),s);
}

int shape_attnum_to_arity(const shape *sh,int attnum)
{
  return colshape_arity(shape_colshape_ref(sh,attnum));
}

bool shape_attnum_to_symbolicp(const shape *sh,int attnum)
{
  return colshape_is_symbolic(shape_colshape_ref(sh,attnum));
}

char *shape_attnum_value_to_name(const shape *sh,int attnum,int value)
{
  return colshape_value_to_name(shape_colshape_ref(sh,attnum),value);
}

int shape_num_symbolic_atts(const shape *sh)
{
  int result = 0;
  int i;
  for ( i = 0 ; i < shape_num_atts(sh) ; i++ )
  {
    if ( shape_attnum_to_symbolicp(sh,i) )
      result += 1;
  }
  return(result);
}

int shape_num_real_atts(const shape *sh)
{
  return(shape_num_atts(sh) - shape_num_symbolic_atts(sh));
}

void shape_delete_attribute(shape *sh,int attnum)
{
  if ( attnum < 0 || attnum >= shape_num_atts(sh) )
    my_error("shape_delete_attribute");
  else
  {
    int i;
    int num_atts = shape_num_atts(sh);
    free_colshape(shape_colshape_ref(sh,attnum));
    for ( i = attnum ; i < num_atts-1 ; i++ )
      sh->cs_array[i] = shape_colshape_ref(sh,i+1);
  }
  sh->num_atts -= 1;
}

/* This swaps the descriptions of att1 and att2 in the shape.  
   Precondition -- att1 and att2 are in range and not equal
         to each other. */
void shape_swap_attributes(shape *sh,int att1,int att2)
{
        int num_atts = shape_num_atts(sh);

  if ( att1 < 0 || att1 >= num_atts || att2 < 0 || att2 >= num_atts || att1 == att2)
    my_error("shape_swap_attributes");
  else
  {
    colshape *tem = shape_colshape_ref(sh,att1);

    sh->cs_array[att1] = shape_colshape_ref(sh,att2);
    sh->cs_array[att2] = tem;
  }
}

void shape_rename_attribute(shape *sh,int attnum,char *newname)
{
  colshape_rename(shape_colshape_ref(sh,attnum),newname);
}

void shape_rename_value(shape *sh,int attnum,int value,char *newname)
{
  colshape_rename_value(shape_colshape_ref(sh,attnum),value,newname);
}

void shape_delete_value(shape *sh,int attnum,int value)
{
  colshape_delete_value(shape_colshape_ref(sh,attnum),value);
}

/* Does not change shape of sh in any way, except it
   does ensure that the internal arrays are such that
   sh->array_size is happy about adding a new column */
static void shape_require_extra_column(shape *sh)
{
  int old_num_atts = shape_num_atts(sh);
  int new_num_atts = old_num_atts + 1;

  if ( sh->array_size < new_num_atts )
  {
    int new_size = 2 + 2 * sh->array_size;
    int i;
    colshape **new_cs = AM_MALLOC_ARRAY(colshape *,new_size);
    for ( i = 0 ; i < old_num_atts ; i++ )
      new_cs[i] = sh->cs_array[i];
    for ( i = old_num_atts ; i < new_size ; i++ )
      new_cs[i] = NULL;
    AM_FREE_ARRAY(sh->cs_array,colshape *,sh->array_size);
    sh->array_size = new_size;
    sh->cs_array = new_cs;
  }
}

/* Adds a new attribute (column) to the shape, of type REAL_COL */
void shape_add_real_attribute(shape *sh,char *attname)
{
  shape_require_extra_column(sh);
  sh->cs_array[sh->num_atts] = mk_colshape(attname,REAL_COL,UNDEFINED_COL);
  sh -> num_atts += 1;
}

bool string_array_is_sorted(string_array *sa)
{
  bool result = TRUE;
  int i;
  for ( i = 1 ; result && i < string_array_size(sa) ; i++ )
    result = strcmp(string_array_ref(sa,i-1),string_array_ref(sa,i)) < 0;
  return result;
}

/* Makes a new symbolic attribute. The arity of the new attribute is
   string_array_size(value_names), and the name of the i'th value is
   string_array_ref(value_names,i).

   The name of the new attribute is attname.
   */
void shape_add_symbolic_attribute(shape *sh,char *attname,
                                  string_array *value_names)
{
  colshape *cs = mk_colshape(attname,SYMBOL_COL,UNDEFINED_COL);

  shape_require_extra_column(sh);
  colshape_set_value_names(cs,value_names);

  sh->cs_array[sh->num_atts] = cs;
  sh -> num_atts += 1;
}

/* It will cause a my_error if you attempt to set major_coltype
   to be UNDEFINED_COL or to the same value as minor_coltype */
void shape_set_coltype(shape *sh,int attnum,
                       int major_coltype,int minor_coltype)
{
  set_colshape_type(shape_colshape_ref(sh,attnum),major_coltype,minor_coltype);
}


/* Adds the name as a new symbolic value of the attnum'th attribute.
   PRE: shape_symbol_ref_ok(sh,attnum)
 
   NOTE: This does not attempt to ensure the names are in
         lexicographic order.
*/
void shape_add_value_name(shape *sh,int attnum,char *name)
{
  colshape_add_value_name(shape_colshape_ref(sh,attnum),name);
}

/* Very efficiently changes all the value names of the 
   given (symbolic) attribute. If the i'th value name was
   previously "xxx" it now becomes "<prefix>xxx" */
void shape_valnames_prepend(shape *sh,int attnum,char *prefix)
{
  colshape_valnames_prepend(shape_colshape_ref(sh,attnum),prefix);
}

/* For each name in turn, if the name is not already present in the list of
   symbols, it is added.
   This does NOT attempt to ensure that the order of the values 
   corresponds to lexicographic order of their names. */
/* PRE: shape_symbol_ref_ok(cs,attnum) */
void shape_add_value_names(shape *sh,int attnum,
                               string_array *value_names)
{
  colshape_add_value_names(shape_colshape_ref(sh,attnum),value_names);
}

/* Irrespective of previous symbolic values, they are replaced
   with value_names.
   This does NOT attempt to ensure that the order of the values 
   corresponds to lexicographic order of their names. */
/* PRE: shape_symbol_ref_ok(cs,attnum) */
void shape_set_value_names(shape *sh,int attnum,
                               string_array *value_names)
{
  colshape_set_value_names(shape_colshape_ref(sh,attnum),value_names);
}

void save_shape(FILE *s,shape *sh)
{
  int i;
  fprintf(s,"AUTONFastFileFormat Version 1\n");
  for ( i = 0 ; i < shape_num_atts(sh) ; i++ )
    fprintf(s,"%s ",shape_attnum_to_name(sh,i));
  fprintf(s,"\n");
  for ( i = 0 ; i < shape_num_atts(sh) ; i++ )
    save_colshape(s,shape_colshape_ref(sh,i));
}

void shape_set_attname(shape *sh,int attnum,char *new_attname)
{
  colshape_set_attname(shape_colshape_ref(sh,attnum),new_attname);
}

/* Except for version 0, the first line of the file should say
   AUTONFastFileFormat Version <n>
   to specify the version number
 */
shape *mk_shape_from_stream(FILE *s,int *line,int *version,char **r_errmess)
{
  string_array *sa = mk_next_interesting_line(s,line);
  shape *sh = NULL;

  *version = 0;
  *r_errmess = NULL;

  if ( sa == NULL )
    *r_errmess = mk_copy_string("Unexpected end of file reading dataset shape");

  if ( *r_errmess == NULL )
  {
    int i;

    if (eq_string(string_array_ref(sa,0),"AUTONFastFileFormat") && 
        eq_string(string_array_ref(sa,1),"Version"))
    {
      *version = atoi(string_array_ref(sa,2));
      free_string_array(sa);
      sa = mk_next_interesting_line(s,line);
    }

    sh = mk_shape(sa);
    for ( i = 0 ; *r_errmess == NULL && i < shape_num_atts(sh) ; i++ )
    {  
      colshape *cs = mk_colshape_from_stream(s,line,r_errmess);
      if ( cs != NULL )
      {
        free_colshape(shape_colshape_ref(sh,i));
        sh->cs_array[i] = cs;
      }
    }
  }

  if ( *r_errmess != NULL && sh != NULL )
  {
    free_shape(sh);
    sh = NULL;
  }

  if ( sa != NULL ) free_string_array(sa);

  return(sh);
}

FILE *fopen_with_errmess(const char *filename,char *format,char **r_errmess)
{
  FILE *s = fopen(filename,format);
  if ( s != NULL )
    *r_errmess = NULL;
  else
  {
    char buff[1000];
    char *style = ( format[0] == 'r' ) ? " for reading" :
                  ( format[0] == 'w' ) ? " for writing" :
                  ( format[0] == 'a' ) ? " for appending" : "";
    sprintf(buff,"Can't open file %s%s",filename,style);
    *r_errmess = mk_copy_string(buff);
  }
  return(s);
}

static bool is_parseable_as_a_symbol(char *s)
{
  bool result = !is_a_number(s);
  if ( !result )
  {
    double x = atof(s);
    result = x >= 0.0 && x <= 100.0 && 
             fabs(floor(x + 0.5) - x) < 1e-4;
  }
  return(result);
}

#define MISSING_STRING "?"

/* True provided all the symbols in the string array can be parsed
   as numbers and/or are equal to "?" (i.e. MISSING_STRING) */
bool all_entries_are_numeric(string_array *sa)
{
  bool result = TRUE;
  int i;
  for ( i = 0 ; result && i < string_array_size(sa) ; i++ )
    result = eq_string(string_array_ref(sa,i),MISSING_STRING) ||
             is_a_number(string_array_ref(sa,i));
  return(result);
}

bool ds_ignore_token(loadinst *ldi,char *token)
{
  return(token[0] == '?' || token[0] == '#' || (ldi->missing_string != NULL && eq_string(token,ldi->missing_string)));
}

/* Notation:

   sbp is a boolean that refers to a column.

   sbp = true means the column is symbolic and all the value names
                    in the column can be parsed as real numbers.

   sbp = false means either the column is real-valued or its symbol-valued
                     but with at least one value that cannot be parsed as
                     a number.
*/
static void update_colshape_from_token(loadinst *ldi,colshape *cs,
				       bool *r_sbp,char *token,
				       char **r_errmess)
{
  bool is_missing = ds_ignore_token(ldi,token);

  if ( !colshape_is_symbolic(cs) )
  {
    my_assert(!*r_sbp);

    if ( !is_missing && !is_a_number(token) )
    {
      /* If this happens, set ourselves up for a 2nd chance by doing the following:
         Update the loadinst to force it to be symbolic.
         Then if we get called again, this columnn will be forced to be symbolic
         right from the start.
      */
      char *attname = colshape_name(cs);

      if ( !ldi->all_atts_symbolic &&
           caseless_find_index_in_string_array(ldi->these_atts_symbolic,attname) < 0 )
      {
	char *newmess = mk_printf("the value %s is illegal for numeric attribute %s",token,colshape_name(cs));
	printf("%s\nWill try reloading, next time forcing %s to be symbolic\n",
	       newmess,colshape_name(cs));
	add_to_string_array(ldi->these_atts_symbolic,attname);
	add_to_error_message(r_errmess,newmess);
	free_string(newmess);
      }
    }
  }
  else
  {
    if ( !is_missing && !is_parseable_as_a_symbol(token) )
    {
      if ( *r_sbp && 
	   !loadinst_name_to_load_symbolic(ldi, cs->scs->name) )
      {
	if (Verbosity > -1) 
	  printf("%s is loaded as real.\n",cs->scs->name);
        set_colshape_type(cs,REAL_COL,UNDEFINED_COL);
	*r_sbp = FALSE; /* Because it's now real-valued */
      }
      else
      {
        /* This is no longer an error condition.  If we'd prefer that this
           thing be a number (!is_parseable_as_a_symbol()), but its not the
           case that all the elements of this column can be numbers, then
           we'll just leave it as a symbol.  Btw, is_parseable_as_a_symbol()
           is a misleading name.  It doesn't check for strange control 
           characters, etc., but instead looks for numbers that have decimals 
           and/or are large.  JGS 1-12-99
        char buff[1000];
        sprintf(buff,"the numeric value %s is illegal for symbolic attribute %s",
                token,cs->scs->name);
        *r_errmess = mk_copy_string(buff);
        */
        colshape_add_value_name(cs,token);
	/* *r_sbp remains true because it's still the case that all
	   the values are parseable as real numbers */
      }
    }
    else 
    {
      bool is_number = is_a_number(token);

      if ( !is_missing && is_number && ldi -> default_real &&
	   !loadinst_name_to_load_symbolic(ldi,cs->scs->name) && *r_sbp )
      {
	set_colshape_type(cs,REAL_COL,UNDEFINED_COL);
	*r_sbp = FALSE; /* Because its a real column */
      }
      else if ( !is_missing ) 
      {
	if ( !is_number && *r_sbp )
	  *r_sbp = FALSE;
	colshape_add_value_name(cs,token);
      }
    }
  }
}

bool ds_ignore_tokens(loadinst *ldi,string_array *tokens)
{
  bool result = FALSE;
  int i;
  for ( i = 0 ; !result && i < string_array_size(tokens) ; i++ )
    result = ds_ignore_token(ldi,string_array_ref(tokens,i));
  return(result);
}

/* A loadinst is a small data structure specifiying information about what
   to do when loading a datafile. It tells us which line_format to use
   (see later comments on line_formats for more details there).
     ldi->line_format must be one of AUTO_FORMAT WHITESPACE_FORMAT BAR_FORMAT
     or COMMA_FORMAT

   It tells us whether to ignore records that contain a missing value.

   And if we decide not to ignore records with missing values it tells
    us what default value to use for numeric attributes (symbolic
    attributes get the special value "?") */
loadinst *mk_default_loadinst(void)
{
  loadinst *ldi = AM_MALLOC(loadinst);
  ldi -> missing_string = NULL;
  ldi -> line_format = AUTO_FORMAT;
  ldi -> ignore_missing = FALSE;
  ldi -> missing_numeric_value = -9999.0;
  ldi -> ignore_short = FALSE;
  ldi -> default_real = FALSE;
  ldi -> these_atts_symbolic = mk_string_array(0);
  ldi -> all_atts_symbolic = FALSE;

  return ldi;
}

#ifdef AUTON_UNUSED_CODE
static string_array *mk_parse_symbolic_key(int argc,char *argv[])
{
  int i;
  string_array* ret = mk_string_array(0);

  for(i = 0; i < argc; i++)
  {
    char *s = argv[i];
    if ( char_is_in(s,'=') )
    {
      string_array *tokens = mk_broken_string_using_seppers(s,"=");
      if(string_array_size(tokens) == 2 && eq_string("symbolic",string_array_ref(tokens,1)))
      {
        printf("%s is loaded as symbolic.\n",string_array_ref(tokens,0));
        add_to_string_array(ret, string_array_ref(tokens,0));
      }
      free_string_array(tokens);
    }
  }

  return ret;
}
#endif /* AUTON_UNUSED_CODE */

/* The 2nd arg must be one of:
    AUTO_FORMAT
    COMMA_FORMAT
    BAR_FORMAT
    WHITESPACE_FORMAT 
   defined in amdmex.h */
void loadinst_set_line_format(loadinst *ldi,int line_format)
{
  my_assert(line_format == AUTO_FORMAT ||
	    line_format == BAR_FORMAT ||
	    line_format == COMMA_FORMAT ||
	    line_format == WHITESPACE_FORMAT);
  ldi->line_format = line_format;
}

/* Searches the command line for "format XXX" where XXX is one
   of auto comma or space. If doesn't find this, keeps line
   format unchanged */
void loadinst_update_line_format_from_args(loadinst *ldi,int argc,char *argv[])
{
  const char *format_string = string_from_args("format",argc,argv,"auto");
  int line_format = ldi->line_format;

  if ( caseless_eq_string(format_string,"auto") )
    line_format = AUTO_FORMAT;
  else if ( caseless_eq_string(format_string,"comma") )
    line_format = COMMA_FORMAT;
  else if ( caseless_eq_string(format_string,"bar") )
    line_format = BAR_FORMAT;
  else if ( caseless_eq_string(format_string,"space") )
    line_format = WHITESPACE_FORMAT;
  else
    printf("Line format %s unrecognized. Should be auto space bar or comma.\n"
	   "Will use auto.\n",
           format_string);
  
  loadinst_set_line_format(ldi,line_format);
}

/* When the file is loaded, if any attribute has a name that
   appears in attnames then it is loaded as though it were
   symbolic even if all the actual values are numbers. */
void loadinst_set_force_symbolic(loadinst *ldi,string_array *attnames)
{
  free_string_array(ldi->these_atts_symbolic);
  ldi -> these_atts_symbolic = mk_copy_string_array(attnames);
}

/* When the file is loaded, any attribute
   is loaded as though it were
   symbolic even if all the actual values are numbers. */
void loadinst_all_force_symbolic(loadinst *ldi)
{
  ldi -> all_atts_symbolic = TRUE;
}

/* Adds another attname to the set of names that will be loaded
   as symbolic */
void loadinst_add_force_symbolic(loadinst *ldi,char *attname)
{
  add_to_string_array(ldi->these_atts_symbolic,attname);
}

/* Searches the command line for key <tokens>
   where key is defined by the "key" are below and
   <tokens> are one-or-more tokens on the command line
   obeying the command-line set-of-items notation described
   in amdmex.h for mk_string_array_from_args(...).

   Here are examples:

      force_symbolic [ age , gender, id]
      force_symbolic [ age  gender id ]
      force_symbolic [age,gender,id]
      force_symbolic age,gender,id
      force_symbolic "age gender id"
*/
void loadinst_set_force_symbolic_from_args(loadinst *ldi,char *key,
                                           int argc,char *argv[])
{
  string_array *attnames = mk_string_array_from_args(key,argc,argv);
  loadinst_set_force_symbolic(ldi,attnames);
  free_string_array(attnames);
}

/* Sets the default_real flag. */
void loadinst_set_default_real(loadinst *ldi,bool default_real)
{
  ldi->default_real = default_real;
}

/* Sets the default_real flag from args using key "default_real". */
void loadinst_set_default_real_from_args(loadinst *ldi,int argc,char *argv[])
{
  int default_real = bool_from_args("default_real",argc,argv,ldi->default_real);
  loadinst_set_default_real(ldi,default_real);
}

/* Gives an explanation of what this particular loadinst will
   do with the data */
void explain_loadinst(loadinst *ldi)
{
  printf("Here are my dataset loading instructions (loadinst):\n");

  switch ( ldi->line_format )
  {
    case AUTO_FORMAT:
      printf("I will automatically figure out if lines are space separated\n"
	     "comma or bar separated.\n");
      break;
    case COMMA_FORMAT:
      printf("I will insist that lines are comma separated.\n");
      break;
    case BAR_FORMAT:
      printf("I will insist that lines are bar (|) separated.\n");
      break;
    case WHITESPACE_FORMAT:
      printf("I will insist that lines are space separated.\n");
      break;
    default:
      my_error("No such line_format");
      break;
  }

  printf("If loading from .CSV and I find a column containing all numbers\n");

  if ( ldi->default_real )
    printf("then I'll always treat it as a real-valued attribute.\n");
  else
    printf("then I'll treat it as a symbolic attribute if it only has a few\n"
	   "small integer values. Else I'll treat it as real-valued.\n");

  if ( ldi->all_atts_symbolic )
    printf("I'll force all attributes to be symbolic");
  else if ( string_array_size(ldi->these_atts_symbolic) < 1 )
    printf("There are no attributes that I'll force to be symbolic.\n");
  else
  {
    int i;

    printf("No matter what their values look like I'll force the following\n"
	   "attributes to be symbolic:\n");

    for ( i = 0 ; i < string_array_size(ldi->these_atts_symbolic) ; i++ )
      printf("%s ",string_array_ref(ldi->these_atts_symbolic,i));

    printf("\n");
  }
}

/* Makes a loadinst from the command line arguments using the
   syntax described in the individual functions above */
loadinst *mk_loadinst_from_args(int argc,char *argv[])
{
  loadinst *ldi = mk_default_loadinst();
  const char *missing_string = string_from_args("missing",argc,argv,NULL);

  loadinst_update_line_format_from_args(ldi,argc,argv);
  loadinst_set_default_real_from_args(ldi,argc,argv);

  ldi -> ignore_missing = bool_from_args("ignore_missing",argc,argv,
                                         ldi->ignore_missing);
  ldi -> ignore_short = bool_from_args("ignore_short",argc,argv,
                                         ldi->ignore_short);

  ldi -> missing_numeric_value = double_from_args("missing_numeric_value",
                                                argc,argv,
                                                ldi->missing_numeric_value);

  ldi -> missing_string = (missing_string == NULL) ? NULL : mk_copy_string(missing_string);

  loadinst_set_force_symbolic_from_args(ldi,"force_symbolic",argc,argv);

  return ldi;
}

void free_loadinst(loadinst *ldi)
{
  if ( ldi -> missing_string != NULL ) free_string(ldi->missing_string);

  free_string_array(ldi->these_atts_symbolic);
  AM_FREE(ldi,loadinst);
}

/* Returns TRUE if and only if attribute "attname" has been requested
   to be loaded in symbolic form. */
bool loadinst_name_to_load_symbolic(loadinst *ldi, char *attname)
{
  return ldi->all_atts_symbolic || caseless_find_index_in_string_array(ldi->these_atts_symbolic,attname) >= 0;
}

/* Please see "sbp" comment earlier to understand the attnum_to_sbp array */
static void update_shape_from_tokens(loadinst *ldi,shape *sh,
			      ivec *attnum_to_sbp,string_array *tokens,
                              int *r_num_rows,char **r_errmess)
{
  if ( ldi->ignore_missing && ds_ignore_tokens(ldi,tokens) )
  {
    /* skip */
  }
  else if ( ldi->ignore_short && string_array_size(tokens) < shape_num_atts(sh) )
  {
    /* skip */
  }
  else
  {
    int size = string_array_size(tokens);
    int i;

    if ( size > shape_num_atts(sh) )
    {
      if ( *r_errmess == NULL )
      {
	*r_errmess = mk_printf("I expected %d tokens on data "
			       "record %d but I found %d",
			       shape_num_atts(sh),*r_num_rows,size);
	my_error(*r_errmess);
      }      
    } 
    else if (size < shape_num_atts(sh))
    {
      int diff = shape_num_atts(sh) - size;
      for(i = 0; i < diff; i++)
        add_to_string_array(tokens,"?");
      size = shape_num_atts(sh);
    }
    
    /* It may be ok to continue on, looking for all the errors, but if there is
       an error before we even start, then it will crash.  -- Dan 12/20/00

      if(*r_errmess==NULL) 

      Hmmm, not sure what really causes this problem.  Leaving this line in defeats
      the entire point of continuing which is to find all the symbolic columns so
      we can read them correctly as symbolic on the second try.
      Once again, I'll try taking this out so it continues through all errors.
      I fear this will re-create the bug Dan found, but hopefully the next time
      around we can correct it in a way that doesn't cause read failures when we
      get columns of almost all numbers and an annoying symbol midway through.

      JGS 10-23-01
    */

                /* This used to bail out immediately on an error, but now we continue
                   on to be sure we find all the errors.  -- JGS 11/1/00
                 */
    /*    for ( i = 0 ; *r_errmess == NULL && i < size ; i++ )   */

    for ( i = 0 ; i < size ; i++ )
    {
      bool sbp = ivec_ref(attnum_to_sbp,i);
      update_colshape_from_token(ldi,shape_colshape_ref(sh,i),&sbp,
				 string_array_ref(tokens,i),r_errmess);
      ivec_set(attnum_to_sbp,i,sbp);	
    }

    *r_num_rows += 1;
    if ((Verbosity >=0) && (*r_num_rows>10 && is_power_of_two(*r_num_rows)))
      printf("Datset load (pass 1) loaded %d lines\n",*r_num_rows);
  }
}

static void sort_attribute_value(shape *sh,int attnum)
{
  colshape_sort_values(shape_colshape_ref(sh,attnum));
}

bool shape_symbol_ref_ok(const shape *sh,int attnum)
{
  return colshape_symbol_ref_ok(shape_colshape_ref(sh,attnum));
}

bool shape_real_ref_ok(const shape *sh,int attnum)
{
  return colshape_real_ref_ok(shape_colshape_ref(sh,attnum));
}

void sort_all_symbolic_attribute_values(shape *s)
{
  int i;
  for (i=0;i<s->num_atts;i++)
    if ( shape_symbol_ref_ok(s,i) )
      sort_attribute_value(s,i);
}

shape *mk_shape_deduced_from_file(loadinst *ldi,const char *filename,
                                  int *r_num_rows,char **r_errmess)
{
  FILE *s = fopen_with_errmess(filename,"r",r_errmess);
  shape *sh = NULL;

  *r_num_rows = 0;

  if ( *r_errmess == NULL )
  {
    string_array *attnames;
    string_array *tokens=NULL;
    int line = 0;
    bool finished = FALSE;
    ivec *attnum_to_sbp = NULL;

    get_attnames_and_first_tokens(s,&attnames,&tokens,ldi->line_format,
                                  &line,r_errmess);

    if ( *r_errmess == NULL )
    {
      sh = mk_shape(attnames);

      attnum_to_sbp = mk_constant_ivec(string_array_size(attnames),TRUE);

      update_shape_from_tokens(ldi,sh,attnum_to_sbp,
			       tokens,r_num_rows,r_errmess);

    }

                /* Used to bail out at the first sign of trouble, but now we continue
                   in order to identify all the errors.  -- JGS  11/1/00

       If tokens==NULL and we let it continue, string_array crashes occur.
       If I just replace:
         free_string_array(tokens);
         with
         if(tokens) free_string_array(tokens);
       the errors are worse.  So I think if tokens==NULL 
       it is ok to bail out of this loop. --pfm2 12/4/00
                 */
/*    while ( !finished && *r_errmess == NULL ) */

    if ( *r_errmess == NULL )
    {
      while (!finished && tokens != NULL)
      {
	free_string_array(tokens);
	tokens = mk_next_tokens(s,&line,ldi->line_format);
	if ( tokens == NULL ) finished = TRUE;
	else update_shape_from_tokens(ldi,sh,attnum_to_sbp,
				      tokens,r_num_rows,r_errmess);
      }
    }

    if ( attnames != NULL ) free_string_array(attnames);
    if ( tokens != NULL ) free_string_array(tokens);

    if ( attnum_to_sbp != NULL ) free_ivec(attnum_to_sbp);

    /* This next section checks to make sure that something has been
       added to the shape. If not we complain. */
    if ( *r_errmess == NULL && sh != NULL && *r_num_rows == 0 )
      *r_errmess = mk_copy_string("There are no complete lines of data.");
        
    if ( *r_errmess != NULL )
    {
      char *buff;
      buff = mk_printf("Error on line %d of file %s: %s",
              line,filename,*r_errmess);
      free_string(*r_errmess);
      *r_errmess = mk_copy_string(buff);
      if ( sh != NULL ) free_shape(sh);
      sh = NULL;
      free_string(buff);
    }
  }
  if ( s != NULL ) fclose(s);

  if ( sh != NULL )
    sort_all_symbolic_attribute_values(sh);

  return(sh);
}

/* Adds value_name into the list of names of the given attnum,
   inserting at the appropriate position to ensure that the
   values remain in lexicographic order. Returns the small int
   integer value of the new value name. You may assume that if some
   other value_name had small int value v before then afterwards
   it has...

     ...value v if v < result
     ...value v+1 if v >= result

   where result is the result of this function (i.e. the new symbol_value
   of the new value_name)
*/
int shape_insert_value_name(shape *sh,int attnum,char *value_name)
{
  return colshape_insert_value_name(shape_colshape_ref(sh,attnum),value_name);
}

int shape_name_to_attnum(shape* sh,const char *s)
{
  int i;
  int result = -1;
  for ( i = 0 ; result < 0 && i < shape_num_atts(sh) ; i++ )
  {
    if ( caseless_eq_string(s,shape_attnum_to_name(sh,i)) )
      result = i;
  }
  return(result);
}

