/* *
   File:        dsut.c
   Author:      Andrew W. Moore
   Created:     Mon Jun  2 19:58:44 EDT 1997
   Description: Utilities on datsets

   Copyright 1997, Carnegie Mellon University
*/

#include "dsut.h"

void copy_part_of_datset_row_to_dyv( datset *ds, int row, ivec *atts, dyv *dv)
{
  int numatts, i, att;
  double val;

  numatts = ivec_size( atts);

#ifndef AMFAST
  if (dyv_size(dv) != numatts) {
    my_errorf( "copy_part_of_datset_row_to_dyv: Error: dyv size (=%d) "
	       "!= atts size (=%d)",
	       dyv_size( dv), numatts);
  }
#endif

  for (i=0; i<numatts; ++i) {
    att = ivec_ref( atts, i);

    if (ds_real_ref_ok( ds, att)) val = ds_real_ref( ds, row, att);
    else val = ds_symbol_ref( ds, row, att);

    dyv_set( dv, i, val);
  }

  return;
}

void copy_dyv_to_real_column( datset *ds, dyv *dv, int att)
{
  int row;
  double val;

#ifndef AMFAST
  if (!ds_real_ref_ok( ds, att)) {
    my_errorf( "copy_dyv_to_real_column: Error: column %d is not real.", att);
  }
#endif

  for (row = 0; row < ds_num_rows(ds); ++row) {
    val = dyv_ref( dv, row);
    ds_real_set( ds, row, att, val);
  }
  return;
}

dyv *mk_dyv_from_symbolic_col( datset *ds, int attnum )
{
  dyv *result = NULL;
  if( ds_attnum_to_symbolicp(ds,attnum) )
  {
    int n = ds_num_rows(ds);
    int i;
    result = mk_dyv(n);

    for( i=0;i<n;i++)
      dyv_set(result,i,ds_symbol_ref(ds,i,attnum));
  }
  return(result);
}

void copy_ivec_to_symbol_column( datset *ds, ivec *iv, int att)
{
  int row, val;

#ifndef AMFAST
  if (!ds_symbol_ref_ok( ds, att)) {
    my_errorf( "copy_ivec_to_symbolic_column: Error: column %d is "
	       "not symbolic.", att);
  }
#endif

  for (row = 0; row < ds_num_rows(ds); ++row) {
    val = ivec_ref( iv, row);
    ds_symbol_set( ds, row, att, val);
  }
  return;
}


/*---------------------------------*/
/* cf. comments next to mk_ordinal_dyv_from_symbolic_col() */
dyv *mk_dyv_from_symbolic_rows( datset *ds, int attnum, dyv *output, ivec *marked_rows )
{
	dyv *result = NULL;
	
	/* int shift = 0; */
	if( ds_attnum_to_symbolicp(ds,attnum) )
	{
		int arity = ds_attnum_to_arity(ds,attnum);
		dyv *bins = mk_zero_dyv(arity);
		ivec *counts = mk_zero_ivec(arity);
		ivec *sortedcounts;
		double aux;
		int i, binindex;
		int n = ds_num_rows(ds);
		int m;
		int size;
		bool free_rows = FALSE;
		ivec *rows = marked_rows;

		if( rows == NULL )
		{
			rows = mk_identity_ivec(n);
			free_rows = TRUE;
		}
		size = ivec_size(rows);
		result = mk_dyv(size);
		
		for(i=0;i<size;i++) /* first calculate means */
		{
			m = ivec_ref(rows,i);
			binindex = ds_symbol_ref(ds,m,attnum);
			dyv_set(bins,binindex,dyv_ref(bins,binindex)+dyv_ref(output,i));
			ivec_set(counts,binindex,ivec_ref(counts,binindex)+1);
		}
		
		for(i=0;i<arity;i++)
		{
			dyv_set(bins,i,
				( (ivec_ref(counts,i)) ?
				  (dyv_ref(bins,i)/ivec_ref(counts,i)) :
				  0.0 ));
		}
		/* bins contain averages of output values for each symbolic
		   value of the input counts represent the number of instances
		   of subsequent symbolic values and the same - the number of
		   ties the ties should be resolved later, when writing to the
		   result vector */
		
		/* now we've got the ordering */
		sortedcounts = mk_indices_of_sorted_dyv(bins);
	
		m=0;
		for(i=0;i<size;i++)
		{
			m = ivec_ref(rows,i);
			aux = ivec_ref(sortedcounts,ds_symbol_ref(ds,m,attnum));
			dyv_set(result,i,aux);
		}
		free_dyv(bins);
		free_ivec(sortedcounts);
		free_ivec(counts);
		if( free_rows )
		{
			free_ivec(rows);
			rows = NULL;
		}
	}
	
	return( result );
}
/* ----------------- */

/* --------------------------------- */
/* this is meant to be robust to missing values */
dyv *mk_dyv_from_symbolic_rows_robust( datset *ds, int attnum, dyv *output, 
                                       ivec *marked_rows, ivec **missing_index )
{
	dyv *result = NULL;
	
	/* int shift = 0; */
	if( ds_attnum_to_symbolicp(ds,attnum) )
	{
		int arity = ds_attnum_to_arity(ds,attnum);
		dyv *bins = mk_zero_dyv(arity);
		ivec *counts = mk_zero_ivec(arity);
		ivec *sortedcounts;
		double aux;
		int i, binindex;
		int n = ds_num_rows(ds);
		int m;
		int size;
		bool free_rows = FALSE;
		ivec *rows = marked_rows;

    if( *missing_index!=NULL )
    {
      if( ivec_size(*missing_index)>0 )
      {
        free_ivec(*missing_index);
        *missing_index = NULL;
      }
    }
    *missing_index = mk_ivec(0);
		if( rows == NULL )
		{
			rows = mk_identity_ivec(n);
			free_rows = TRUE;
		}
		size = ivec_size(rows);
		result = mk_dyv(size);

		for(i=0;i<size;i++) /* first calculate means */
		{
			m = ivec_ref(rows,i);
      if( ds_value_is_missing(ds,m,attnum) )
      {
	/* missing index entries point to indices of 'rows'
	   and 'result' and not to the dataset rows!!! */
        add_to_ivec( *missing_index, i );
                                        
      }
      else
      {
        binindex = ds_symbol_ref(ds,m,attnum);
        dyv_set(bins,binindex,dyv_ref(bins,binindex)+dyv_ref(output,i));
        ivec_set(counts,binindex,ivec_ref(counts,binindex)+1);
      }
		}
		
		for(i=0;i<arity;i++)
		{
			dyv_set(bins,i,( (ivec_ref(counts,i)) ? (dyv_ref(bins,i)/ivec_ref(counts,i)) : 0.0 ));
		}
		/* bins contain averages of output values for each symbolic
		   value of the input counts represent the number of instances
		   of subsequent symbolic values and the same - the number of
		   ties the ties should be resolved later, when writing to the
		   result vector */
		
		/* now we've got the ordering */
		sortedcounts = mk_indices_of_sorted_dyv(bins);
	
		m=0;
		for(i=0;i<size;i++)
		{
			m = ivec_ref(rows,i);
      if( !ds_value_is_missing(ds,m,attnum) )
      {
        aux = ivec_ref(sortedcounts,ds_symbol_ref(ds,m,attnum));
        dyv_set(result,i,aux);
      }
      else
        dyv_set(result,i,0.0);
		}
		free_dyv(bins);
		free_ivec(sortedcounts);
		free_ivec(counts);
		if( free_rows )
		{
			free_ivec(rows);
			rows = NULL;
		}
    if( ivec_size(*missing_index)==0 )
    {
      free_ivec(*missing_index);
      *missing_index = NULL;
    }
	}
	
	return( result );
}
/* ---------------- */

/* ---------------------------------- */
dyv *mk_dyv_from_rows(const datset *ds, int attnum, ivec *marked_rows )
{
	dyv *result = NULL;

	if( !ds_attnum_to_symbolicp(ds,attnum) )
	{
		int n = ds_num_rows(ds);
		int size;
		int i,m;
		bool free_rows = FALSE;
		ivec *rows = marked_rows;

		if( rows == NULL )
		{
			rows = mk_identity_ivec(n);
			free_rows = TRUE;
		}
		size = ivec_size(rows);
		result = mk_dyv(size);
		
		for( i=0;i<size;i++)
		{
			m = ivec_ref(rows,i);
			dyv_set(result,i,ds_real_ref(ds,m,attnum));
		}

		if( free_rows )
		{
			free_ivec(rows);
			rows = NULL;
		}		
  }	
	return( result );
}


ivec *mk_ivec_from_symbol_rows(const datset *ds, int attnum, ivec *rows){
  ivec *result = NULL;

  if(!ds_symbol_ref_ok(ds,attnum)) {
    my_error("Real Attributed Passed to mk_ivec_from_symbol_rows");
  } else {
    int i, size;
    bool free_rows = FALSE;
    
    if(rows == NULL ) {
      rows = mk_identity_ivec(ds_num_rows(ds));
      free_rows = TRUE;
    }

    size = ivec_size(rows);
    result = mk_ivec(size);
    
    for( i=0;i<size;i++) {
      ivec_set(result,i,ds_symbol_ref(ds,ivec_ref(rows, i),attnum));
    }
    
    if (free_rows) {
      free_ivec(rows);
      rows = NULL;
    }		
  }	
  return( result );
}

/* ---------------------------------- */
dyv *mk_dyv_from_rows_robust( datset *ds, int attnum, ivec *marked_rows, ivec **missing_index )
{
	dyv *result = NULL;

  if( !ds_attnum_to_symbolicp(ds,attnum) )
  {
    int n = ds_num_rows(ds);
    int size;
    int i,m;
    bool free_rows = FALSE;
    ivec *rows = marked_rows;
    
    if( *missing_index!=NULL )
    {
      if( ivec_size(*missing_index)>0 )
      {
        free_ivec(*missing_index);
        *missing_index=NULL;
      }
    }
    *missing_index = mk_ivec(0);
    if( rows == NULL )
    {
      rows = mk_identity_ivec(n);
      free_rows = TRUE;
    }
    size = ivec_size(rows);
    result = mk_dyv(size);

    for( i=0;i<size;i++)
    {
      m = ivec_ref(rows,i);
      if( ds_value_is_missing(ds,m,attnum) )
      {
	/* missing index entries point to indices of 'rows'
	   and 'result' and not to the dataset rows!!! */
        add_to_ivec( *missing_index, i ); 
        dyv_set(result,i,0.0);
      }
      else
        dyv_set(result,i,ds_real_ref(ds,m,attnum));
    }
    
    if( free_rows )
    {
      free_ivec(rows);
      rows = NULL;
    }		
    if( ivec_size(*missing_index)==0 )
    {
      free_ivec(*missing_index);
      *missing_index = NULL;
    }
  }	
  return( result );
}

/* ---------------------------------------- */
double ds_att_mean(const datset *ds,int attnum)
{
  double sum = 0.0;
  int n = 0;
  int i;
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      sum += ds_real_ref(ds,i,attnum);
      n += 1;
    }
  }
  return sum/int_max(1,n);
}

double ds_att_sdev(const datset *ds,int attnum)
{
  double mean = ds_att_mean(ds,attnum);
  double sum = 0.0;
  int n = 0;
  int i;
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      sum += real_square(mean - ds_real_ref(ds,i,attnum));
      n += 1;
    }
  }
  return(sqrt(sum/int_max(1,n-1)));
}

double ds_att_min(const datset *ds,int attnum)
{
  double result = 0.0;
  bool result_found = FALSE;
  int i;

  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      double value = ds_real_ref(ds,i,attnum);
      if ( !result_found || value < result )
      {
        result = value;
        result_found = TRUE;
      }
    }
  }
  return(result);
}

double ds_att_min_rows(const datset *ds, int attnum, ivec *rows) {
  int r;
  double result = 0.0;
  bool result_found = FALSE;
  bool deleteRows = FALSE;

  if (rows == NULL) {
    rows = mk_identity_ivec(ds_num_rows(ds));
    deleteRows = TRUE;
  }
  
  for (r=0; r< ivec_size(rows); r++){
    if (!ds_value_is_missing(ds,ivec_ref(rows, r),attnum)) {
      double value = ds_real_ref(ds, ivec_ref(rows, r),attnum);
      if (value < result || !result_found) {
        result = value;
        result_found = TRUE;
      }
    }
  }
  
  if (deleteRows) free_ivec(rows);
  return(result);
}

double ds_att_max_rows(const datset *ds, int attnum, ivec *rows) {
  int r;
  double result = 0.0;
  bool result_found = FALSE;
  bool deleteRows = FALSE;

  if (rows == NULL) {
    rows = mk_identity_ivec(ds_num_rows(ds));
    deleteRows = TRUE;
  }
  
  for (r=0; r< ivec_size(rows); r++){
    if (!ds_value_is_missing(ds,ivec_ref(rows, r),attnum)) {
      double value = ds_real_ref(ds, ivec_ref(rows, r),attnum);
      if (value > result || !result_found) {
        result = value;
        result_found = TRUE;
      }
    }
  }
  
  if (deleteRows) free_ivec(rows);
  return(result);
}



double ds_att_max(const datset *ds,int attnum)
{
  double result = 0.0;
  bool result_found = FALSE;
  int i;

  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      double value = ds_real_ref(ds,i,attnum);
      if ( !result_found || value > result )
      {
        result = value;
        result_found = TRUE;
      }
    }
  }
  return(result);
}

/* PRE: attnum is symbolic 
   result[i] = number of rows in ds with value i */
ivec *mk_ds_frequencies(const datset *ds,int attnum)
{
  ivec *result = mk_zero_ivec(ds_attnum_to_arity(ds,attnum));
  int i;
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    if ( !ds_value_is_missing(ds,i,attnum) )
    {
      int value = ds_symbol_ref(ds,i,attnum);
      ivec_increment(result,value,1);
    }
  }
  return(result);
}

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_symbolic_att_explain_string_basic(datset *ds,int attnum)
{
  int num_vals = ds_attnum_to_arity(ds,attnum);
  char *result = mk_printf("%d val%s%s",num_vals,(num_vals<3)?"ue":"",
			   (num_vals==1)?"":"s");
  if ( num_vals < 5 )
  {
    char *s = mk_value_names_string_from_attnum(ds,attnum);
    char *new_result = mk_printf("%s %s",result,s);
    free_string(s);
    free_string(result);
    result = new_result;
  }
  return result;
}

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_real_att_explain_string_basic(datset *ds,int attnum,
					  bool do_min_and_max)
{
  char *result;
  if ( do_min_and_max )
  {
    result = mk_printf("low %g, high %g, mean %g", ds_att_min(ds,attnum),
		       ds_att_max(ds,attnum),ds_att_mean(ds,attnum));
  }
  else
    result = mk_copy_string("");

  return result;
}

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_att_explain_string_basic(datset *ds,int attnum,bool do_min_and_max)
{
  char *result;
  if ( ds_attnum_to_symbolicp(ds,attnum) )
  {
    char *sval_string = mk_ds_symbolic_att_explain_string_basic(ds,attnum);
    char *rv1 = (ds_real_ref_ok(ds,attnum)) ?
                mk_ds_real_att_explain_string_basic(ds,attnum,do_min_and_max):
                NULL;
    char *rval_string = (rv1==NULL) ? NULL :
                        mk_printf(" ( also real %s)",rv1);

    result = mk_printf("symb %s%s",sval_string,
		       (rval_string==NULL)?"":rval_string);

    if ( sval_string != NULL ) free_string(sval_string);
    if ( rv1 != NULL ) free_string(rv1);
    if ( rval_string != NULL ) free_string(rval_string);
  }
  else
  {
    char *rval_string = 
      mk_ds_real_att_explain_string_basic(ds,attnum,do_min_and_max);
    char *sv1 = (ds_symbol_ref_ok(ds,attnum)) ?
                mk_ds_symbolic_att_explain_string_basic(ds,attnum) : NULL;
    char *sval_string = (sv1==NULL) ? NULL :
                        mk_printf(" ( also symb %s )",sv1);
    result = mk_printf("real %s%s",rval_string,
		       (sval_string==NULL)?"":sval_string);
    if ( rval_string != NULL ) free_string(rval_string);
    if ( sv1 != NULL ) free_string(sv1);
    if ( sval_string != NULL ) free_string(sval_string);
  }

  return result;
}

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_att_explain_string(datset *ds,int attnum)
{
  return mk_ds_att_explain_string_basic(ds,attnum, /*min_and_max=*/ TRUE);
}

/* Same as above except doesn't try to do min and max of real atts */
char *mk_quick_ds_att_explain_string(datset *ds,int attnum)
{
  return mk_ds_att_explain_string_basic(ds,attnum, /*min_and_max=*/ FALSE);
}

/* Same as above except only does min/max if small dataset */
char *mk_not_slow_ds_att_explain_string(datset *ds,int attnum)
{
  return mk_ds_att_explain_string_basic(ds,attnum,ds_num_rows(ds)<=100000);
}

/* Prints a summary of the attributes in ds to stdout */
void explain_datset(datset *ds)
{
  int i;
  printf("Dataset has %d rows and %d attributes.\n",
         ds_num_rows(ds),ds_num_atts(ds));
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    char *s = mk_not_slow_ds_att_explain_string(ds,i);
    printf("%12s %s\n",ds_attnum_to_name(ds,i),s);
    free_string(s);
  }
}

bool ds_all_symbolic_values_are_numbers(datset *ds,int attnum)
{
  bool result = TRUE;
  int i;
  for ( i = 0 ; result && i < ds_attnum_to_arity(ds,attnum) ; i++ )
    result = is_a_number(ds_attnum_value_to_name(ds,attnum,i));
  return(result);
}

/* Takes all the attributes that are symbolic but have all their
   values as numbers, and turns them into real attributes. */
void ds_make_as_real_as_possible(datset *ds)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    if ( ds_attnum_to_symbolicp(ds,attnum) &&
         ds_all_symbolic_values_are_numbers(ds,attnum) )
      ds_realize(ds,attnum);
  }
}

bool ds_all_real(datset *ds)
{
  bool result = TRUE;
  int i;
  for ( i = 0 ; result && i < ds_num_atts(ds) ; i++ )
    result = result && ds_real_ref_ok(ds,i);
  return result;
}

dyv *mk_sensible_breaks(datset *ds,int attnum,int arity)
{
  double dlo = ds_att_min(ds,attnum);
  double dhi = ds_att_max(ds,attnum);
  double tlo = dlo;
  double thi = dlo + (dhi - dlo) / arity;
  double slo,shi,sdelta;
  dyv *result = mk_dyv(arity-1);
  int i;

  sensible_limits(tlo,thi,&slo,&shi,&sdelta);

  for ( i = 0 ; i < arity-1 ; i++ )
  {
    double x = slo + (i+1) * (shi - slo);
    dyv_set(result,i,x);
  }

  return(result);
}

dyv *mk_user_breaks(datset *ds,int attnum,int num_levels)
{ /* XXX Weird args */
  int i;
  dyv *result = mk_zero_dyv(num_levels-1);
  char prompt[100];
  char *command;
  sprintf(prompt,"Enter %d breakpoints in ascending order: ",num_levels-1);
  for (i=0;i<dyv_size(result);i++){
#ifdef OLD_SPR
    gui_prompt(prompt,CLI_COMMAND_INPUT,FALSE);
    command = gui_getstring(stdin);
#endif
    command = mk_string_from_user(prompt);
    
    sscanf(command,"%lf",&(result->farr[i]));
#ifdef OLD_SPR
    gui_unprompt();
#endif
    free_string(command);
  }
  return result;
}

string_array *mk_value_names_from_attnum(datset *ds,int attnum)
{
  int arity = ds_attnum_to_arity(ds,attnum);
  int v;
  string_array *sa = mk_string_array(arity);
  for ( v = 0 ; v < arity ; v++ )
    string_array_set(sa,v,ds_attnum_value_to_name(ds,attnum,v));
  return sa;
}

string_array *mk_names_from_attnums(const datset *ds,ivec *attnums)
{
  string_array *sa = mk_string_array(0);
  int i;
  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
    add_to_string_array(sa,ds_attnum_to_name(ds,ivec_ref(attnums,i)));
  return sa;
}

/* This is a somewhat special purpose combination of the previous two
   functions.  It is used when doing regressions of both symbolic (using
   1-of-n encodings) and real attributes.  The purpose is to generate
   the "names" of each term in the regression.  Real valued attributes
   get just the attribute name.  Symbolic valued attributes get the direct
   concatenation of the attribute name and the attribute value.
   */
string_array *mk_names_values_from_attnums(datset *ds,ivec *attnums)
{
  string_array *sa = mk_string_array(0);
  int i;
  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    if (ds_real_ref_ok(ds,attnum))
      add_to_string_array(sa,ds_attnum_to_name(ds,attnum));
    else
    {
      int arity = ds_attnum_to_arity(ds,attnum);
      int v;
#ifdef SYM_REGRESS1
      for ( v = 0 ; v < arity ; v++ )
#else
      for ( v = 0 ; v < (arity-1) ; v++ )
#endif
      {
	char *s = mk_concat_strings(ds_attnum_to_name(ds,attnum),
				    ds_attnum_value_to_name(ds,attnum,v));
	add_to_string_array(sa,s);
	free_string(s);
      }
    }
  }
  return sa;
}

char *mk_value_names_string_from_attnum(datset *ds,int attnum)
{
  string_array *sa = mk_value_names_from_attnum(ds,attnum);
  char *s = mk_string_from_string_array(sa);
  free_string_array(sa);
  return s;
}



/* Returns TRUE if the symbolize succeeded. Returns FALSE if it failed
   for some reason (in which case an error message is printed to stdout).

   The details of the kinds of symbolization available are given in
   the documentation of dispatch_symbolize in dscli.c 

   PRE: ds_real_ref_ok(ds,attnum)
   POST: ds_real_ref_ok(ds,attnum), ds_symbol_ref_ok(ds,attnum) and
         ds_attnum_to_symbolicp(ds,attnum) 
*/
bool run_symbolize(datset *ds,int attnum,bool uniform,bool equal,
		   bool manual,bool integer,
		   int num_levels)
{
  bool result = FALSE;

  if ( !ds_real_ref_ok(ds,attnum) ) 
  {
    printf("%s unchanged---no real values from which to symbolize\n",
	   ds_attnum_to_name(ds,attnum));
  }
  else if ( integer )
  {
    if ( ds_att_min(ds,attnum) < 0 )
    {
      printf("Can't integer symbolize %s...it has a -ve minimum value.\n",
	     ds_attnum_to_name(ds,attnum));
    }
    else
    {
      ds_integer_symbolize(ds,attnum);
      result = TRUE;
    }
  }
  else
  {
    dyv *breaks = NULL;
    if (equal) 
      breaks = mk_sensible_breaks(ds,attnum,num_levels);
    else if (uniform) 
      breaks = mk_uniform_breaks(ds,attnum,num_levels);
    else if (manual) 
      breaks=mk_user_breaks(ds,attnum,num_levels);
    else
      my_error("What break style is this?");

    ds_symbolize_with_breaks(ds,attnum,breaks);
    free_dyv(breaks);
    result = TRUE;
  }

  if ( result ) 
  {
    char buff[500];
    int arity = ds_attnum_to_arity(ds,attnum);
    sprintf(buff,"%s now has arity %d\n",ds_attnum_to_name(ds,attnum),arity);
    if ( arity < 5 )
    {
      char *s = mk_value_names_string_from_attnum(ds,attnum);
      sprintf(buff+strlen(buff)-1," and values %s\n",s);
      free_string(s);
    }
    printf(buff);
  }

  return result;
}

/* Returns TRUE if the unsymbolize succeeded. Returns FALSE if it failed
   for some reason (in which case an error message is printed to stdout).

   PRE: ds_symbol_ref_ok(ds,attnum)
   POST: !ds_symbol_ref_ok(ds,attnum) and !ds_attnum_to_symbolicp(ds,attnum) 
*/
bool run_unsymbolize(datset *ds,int attnum)
{
  char *name = ds_attnum_to_name(ds,attnum);
  bool result = FALSE;

  if ( ds_symbol_ref_ok(ds,attnum) )
  {
    if(ds_real_ref_ok(ds,attnum))
    {
      if( Verbosity > 0.0 )
		printf("unsymbolizing %s\n",name);
      ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);
      result = TRUE;
    }
    else
      printf("Cannot unsymbolize %s without undefining the column.\n",
             name);
  }
  else
    printf("%s is not symbolic, so it can not be unsymbolized\n",name);
  return result;
}

bool run_clever_symbolize(datset *ds,int attnum)
{
  int arity = 0;

  if ( ds_att_min(ds,attnum) >= 0 )
  {
    ds_integer_symbolize(ds,attnum);
    arity = ds_attnum_to_arity(ds,attnum);
  }

  if(arity != 2)
    return run_symbolize(ds,attnum,TRUE,FALSE,FALSE,FALSE,3);
  else
  {
    char buff[500];
    sprintf(buff,"%s now has arity %d\n",ds_attnum_to_name(ds,attnum),arity);
    if ( arity < 5 )
    {
      char *s = mk_value_names_string_from_attnum(ds,attnum);
      sprintf(buff+strlen(buff)-1," and values %s\n",s);
      free_string(s);
    }
    printf(buff);
    return TRUE;
  }
}
/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_symbol_ref_ok(ds,attnum)
   and if !ds_real_ref_ok(ds,attnum) and if all symbol values are
   numeric.

   Then it simply sets the ith real value to be the parsed number
   associated with the ith symbol value's name. */

bool run_realize_maybe_silent(datset *ds,int attnum, bool silent)
{
  char *name = ds_attnum_to_name(ds,attnum);
  bool result = FALSE;

  if ( !ds_attnum_to_symbolicp(ds,attnum) ){
    if(!silent)
      printf("%s is unchanged because it is already real-valued\n",name);
  }
  else if ( ds_real_ref_ok(ds,attnum) )
  {
    if(!silent)
      printf("%s returning to its real-valued form (symbolic values cached)\n",
      name);
    ds_set_coltype(ds,attnum,REAL_COL,SYMBOL_COL);
  }
  else if ( !ds_all_symbolic_values_are_numbers(ds,attnum) ){
    if(!silent)
      printf("%s is unchanged because of non-numeric symbolic values\n",name);
  }
  else
  {
    if(!silent)
      printf("%s will be converted.\n",name);
    ds_realize(ds,attnum);
    result = TRUE;
  }
  return result;
}

bool run_realize(datset *ds, int attnum){
  
  return run_realize_maybe_silent(ds,attnum, FALSE);
}

bool run_realize_silent(datset *ds, int attnum)
{
  return run_realize_maybe_silent(ds,attnum, TRUE);
}


/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_real_ref_ok(ds,attnum).

   Then if ds_symbol_ref_ok() it sets the major column type to symbolic
   If !symbol_ref_ok() then it ignores this because then the column would
   have no type
   
*/
bool run_unrealize(datset *ds,int attnum)
{
  char *name = ds_attnum_to_name(ds,attnum);
  bool result = FALSE;

  if ( ds_real_ref_ok(ds,attnum) )
  {
    if(ds_symbol_ref_ok(ds,attnum))
    {
      ds_set_coltype(ds,attnum,SYMBOL_COL,UNDEFINED_COL);
      result = TRUE;
    }
    else
      printf("Cannot unrealize %s without undefining the column.\n",
             name);
  }
  else
    printf("%s is not real, so it can not be unrealized\n",name);
  return result;
}



/*
   PRE: attnum is symbol_ref_ok and NOT real_ref_ok
   POST: attnum is real_ref_ok and NOT symbol_ref_ok, and any
         new value is simply the small-integer representation of
         the old value.
*/
void ds_symb2real(datset *ds,int attnum)
{
  int row;
  ds_set_coltype(ds,attnum,REAL_COL,SYMBOL_COL);
  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    if ( !ds_value_is_missing(ds,row,attnum) )
    {
      int sval = ds_symbol_ref(ds,row,attnum);
      double rval = (double) sval;
      ds_real_set(ds,row,attnum,rval);
    }
  }
  ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);
}

/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_symbol_ref_ok(ds,attnum)
   and if !ds_real_ref_ok(ds,attnum).

   Then it simply sets the ith real value to be a small integer
   representation of the value. For example if you had a dset with
   one attribute called fruit and it originally was

   fruit
   -----
   apple
   orange
   apple
   banana
   banana

   then set of values for the fruit attribute is { apple , banana , orange }
   and the small-integer value of each attribute is simply 0, 1 and 2
   respectively (note how the values are sorted alphabetically).

   The result dataset column is:

   fruit
   -----
   0
   2
   0
   1
   1
*/
bool run_symb2real(datset *ds,int attnum)
{
  char *name = ds_attnum_to_name(ds,attnum);
  bool result = FALSE;

  if ( !ds_attnum_to_symbolicp(ds,attnum) ){
    printf("%s is unchanged because it is already real-valued\n",name);
  }
  else if ( ds_real_ref_ok(ds,attnum) )
    printf("%s is unchanged because it already has real values\n",name);
  else
  {
    int v;
    printf("%s will be converted.\n",name);
    for ( v = 0 ; v < ds_attnum_to_arity(ds,attnum) ; v++ )
    {
      printf("   Replacing value %15s with %5d\n",
	     ds_attnum_value_to_name(ds,attnum,v),v);
    }
    ds_symb2real(ds,attnum);
    result = TRUE;
  }
  return result;
}

/* Makes a dyv x in which dyv_ref(x,i) == ds_real_ref(ds,i,attnum)
   dyv_size(x) == ds_num_rows(ds).
  PRECONDITION: attnum is real-valued. 

  note: missing values are converted as -1.0.  if the column has missing 
        values, the calling function should do something smart with them.
*/
dyv *mk_dyv_from_real_column(datset *ds,int attnum)
{
  int num_rows = ds_num_rows(ds);
  dyv *dv = mk_zero_dyv(num_rows);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    if (ds_value_is_missing(ds,i,attnum)) 
      dyv_set(dv,i,-1.0);
    else                                  
      dyv_set(dv,i,ds_real_ref(ds,i,attnum));
  }
  return dv;
}

ivec *mk_ivec_from_symbol_column(datset *ds,int attnum)
{
  int num_rows = ds_num_rows(ds);
  ivec *iv = mk_zero_ivec(num_rows);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    if (ds_value_is_missing(ds,i,attnum)) ivec_set(iv,i,-1);
    else            ivec_set(iv,i,ds_symbol_ref(ds,i,attnum));
  }
  return iv;
}

/* same as mk_dyv_from_real_column() but don't choke on non-real columns */
dyv *mk_dyv_from_column(const datset *ds,int attnum)
{
  int num_rows = ds_num_rows(ds);
  dyv *dv = mk_zero_dyv(num_rows);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    if (ds_value_is_missing(ds,i,attnum)) 
      dyv_set(dv,i,-1.0);
    else if (ds_real_ref_ok(ds,attnum))
      dyv_set(dv,i,ds_real_ref(ds,i,attnum));
    else
      dyv_set(dv,i,ds_symbol_ref(ds,i,attnum));
  }
  return dv;
}

/* This is a very special purpose function.  It is only useful for computing
   statistics on real attributes, but is not useful if you need to know which
   values belong to which rows.  It discards all the missing values and returns
   a shorter dyv containing only the ones that are present.
   */
dyv *mk_nonmissing_dyv_from_real_column(datset *ds,int attnum)
{
  int num_rows = ds_num_rows(ds);
  int size = num_rows - ds_attnum_to_num_missing(ds,attnum);
  dyv *vals = mk_zero_dyv(size);
  int i,j;

  for (i=0,j=0;i<num_rows;i++)
    if (!ds_value_is_missing(ds,i,attnum))
      dyv_set(vals,j++,ds_real_ref(ds,i,attnum));

  return vals;
}

dyv *mk_uniform_breaks(datset *ds,int attnum,int num_levels)
{
  dyv *vals = mk_nonmissing_dyv_from_real_column(ds,attnum);
  dyv *breaks = mk_dyv(num_levels-1);
  if ( dyv_size(vals) == 0 )
  {
    /* All values are missing and we return integers 1 .. 2 .. num_levels-1
       as the breaks... */
    int i;
    for ( i = 0 ; i < num_levels-1 ; i++ )
      dyv_set(breaks,i,1.0 + (double) i);
  }
  else
  {
    int i;
    int num_present = dyv_size(vals);
    double eps;

    dyv_sort(vals,vals);

    eps = (dyv_ref(vals,num_present-1) - dyv_ref(vals,0)) / (100.0 * num_levels);
    if ( eps == 0.0 ) eps = 1.0;

    for ( i = 0 ; i < num_levels-1 ; i++ )
    {
      double prev_break_value = (i==0) ? -1e80 : dyv_ref(breaks,i-1);
      double ideal_frac = (i+1) / (double)num_levels;
      double delta = 0.2 / (double)num_levels;
      double lo = ideal_frac - delta;
      double hi = ideal_frac + delta;
      int lo_index = (int)floor(lo * num_present);
      int hi_index = int_min(num_present-1,(int)ceil(hi * num_present));
      double lo_val;
      double hi_val;
      double break_value;

      while ( dyv_ref(vals,lo_index) == dyv_ref(vals,hi_index) &&
              (lo_index > 0 || hi_index < num_present-1) )
      {
        if ( lo_index > 0 ) lo_index -= 1;
        if ( hi_index < num_present-1 && 
             dyv_ref(vals,lo_index) == dyv_ref(vals,hi_index) )
          hi_index += 1;
      }

      lo_val = real_max(prev_break_value+eps,dyv_ref(vals,lo_index));
      hi_val = real_max(lo_val+eps,dyv_ref(vals,hi_index));
      break_value = roundest_number_between(lo_val,hi_val);

      dyv_set(breaks,i,break_value);
    }
  }

  free_dyv(vals);
  return breaks;
}

typedef struct ivec_and_string
{
  ivec *iv;
  char *string;
} ivec_and_string;

int integer_symbol_value(datset *ds,int row,int attnum,void *data)
{
  ivec_and_string *ias = (ivec_and_string *) data;
  ivec *integer_values = ias -> iv;
  char *format = ias -> string;
  int ival = ivec_ref(integer_values,row);
  char buff[100];

  sprintf(buff,format,ival);
  return ds_attnum_name_to_value(ds,attnum,buff);
}

/* PRE: ds_real_ref_ok(ds,attnum), ds_att_min(ds,attnum) >= 0.
         
   POST: Any previous symbolic values are destroyed, and instead the
         i'th row's new symbolic value is associated with a small
         string representation of the number. The only string-representation
         numbers that exist as value_names in the new symbolic attribute
         are for real-values that actually exist in the dataset.

         Everything is converted to the nearest integer.

   POST: ds_attnum_to_symbolicp(ds,attnum),
         ds_real_ref_ok(ds,attnum),
	 ds_symbol_ref_ok(ds,attnum)

   Any missing values that existed beforehand will remain missing afterward
*/
void ds_integer_symbolize(datset *ds,int attnum)
{
  double minval = ds_att_min(ds,attnum);
  double maxval = ds_att_max(ds,attnum);
  int imin = (int)floor(0.5+minval);
  int imax = (int)floor(0.5+maxval);
  if ( imin < 0 )
    printf("Can't do an integer symbolize on %s...it's lowest value is -ve\n",
	   ds_attnum_to_name(ds,attnum));
  else
  {
    string_array *symbol_value_names = mk_string_array(0);
    dyv *real_values = mk_dyv_from_real_column(ds,attnum);
    int num_rows = ds_num_rows(ds);
    ivec *integer_values = mk_ivec(num_rows);
    int row;
    int prev_ival = -1;
    char format[1000];
    char temp[1000];
    ivec_and_string ias[1]; /* Used for passing stuff to integer_symbol_value*/

    sprintf(temp,"%d",imax);
    sprintf(format,"%%0%lud",(unsigned long)strlen(temp));
    printf("Using format = \"%s\"\n",format);

    for ( row = 0 ; row < num_rows ; row++ )
      ivec_set(integer_values,row,(int)floor(0.5+dyv_ref(real_values,row)));

    dyv_sort(real_values,real_values);

    for ( row = 0 ; row < num_rows ; row++ )
    {
      double v = dyv_ref(real_values,row);
      int ival = (int)floor(0.5+v);

      if (v < 0.0) continue;  /* this was a missing value */

      if ( ival != prev_ival )
      {
        char buff[100];
        sprintf(buff,format,ival);
	add_to_string_array(symbol_value_names,buff);
	prev_ival = ival;
      }
    }

    ias -> iv = integer_values;
    ias -> string = format;
    ds_define_symbolic(ds,attnum,symbol_value_names,integer_symbol_value,
		       (void *) ias);

    free_string_array(symbol_value_names);
    free_dyv(real_values);
    free_ivec(integer_values);
  }
}

#ifdef NEVER
void symbolify_all_atts(datset *ds,int max_numeric_arity,int new_arity)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    if ( ds_attnum_to_symbolicp(ds,attnum) &&
         ds_all_symbolic_values_are_numbers(ds,attnum) &&
         ds_attnum_to_arity(ds,attnum) > max_numeric_arity )
      ds_convert_symbol_to_real(ds,attnum);
      
    if ( !ds_attnum_to_symbolicp(ds,attnum) )
    {
      dyv *breaks = mk_sensible_breaks(ds,attnum,new_arity);
      ds_convert_real_to_symbol(ds,attnum,breaks);
      free_dyv(breaks);
    }
  }
}
#endif

/* This function doesn't care whether it's given a real,
   symbolic or missing value. In all cases, it constructs 
   a null-terminated string representation of the number,
   symbol or missingness and returns it. */
char *mk_row_attnum_to_name(datset *ds,int row,int attnum)
{
  char *result;

  if ( ds_value_is_missing(ds,row,attnum) )
    result = mk_copy_string("?");
  else if ( ds_attnum_to_symbolicp(ds,attnum) )
    result = mk_copy_string(ds_attnum_value_to_name(ds,attnum,
                                 ds_symbol_ref(ds,row,attnum)));
  else
  {
    char buff[50];
    sprintf(buff,"%g",ds_real_ref(ds,row,attnum));
#ifndef PC_TTY_PLATFORM
    /* %g WILL NOT use a radix character (decimal point) if the fractional
       part of the value is 0.  This can cause a column of real numbers to
       appear as a column of sybolic values.  This is not desireable - PK.

       AWM notes: Sometimes it is...it can halve the size of a .csv file
       with lots of 0s and 1s, and some applications like symbolic
       inputs. If you want a datset column to be real, call
       run_realize(ds,attnum) once the datset is loaded.

       %g is tricky -- instead of trying to replace it, we'll use
       a silly hack to augment its output. */

    /* Okay for the second time we've found clients unable to compile
       our code on non-Linux because of this so it's #ifdef'd out 
       again. Furthermore, aside from the incompatibility and the weirdness
       of the hack, it is not the generally desirable behavior, though
       Paul needed it for one of his applications. We want CSV files to
       be saved as concisely as possible, and we want datsets to load
       in their most condensed form. If a programmer wants to insist that
       a particular input attribute is real they can call run_realize
       on it. So please do not reinstate this trick, even if it is
       reinstated in a non-Linux-specific way. */

#ifdef ALLOW_WELL_INTENTIONED_BUT_UNACCEPTABLY_HORRIFIC_BUT_FINALLY_FIXED_HACKS
    if (strchr(buff,'e')==NULL && strchr(buff,'.')==NULL && strlen(buff)<48)
      strcat( buff, ".0");
#endif
    /* End of silly hack. */
#endif

    result = mk_copy_string(buff);
  }
  return(result);
}

/* This function insists that attnum is symbolic referencable.
   It does NOT malloc a string, just returns a pointer to a
   string buried in ds that specifies the name of the value
   of the given entry in the datset. */
char *ds_row_attnum_to_name(datset *ds,int row,int attnum)
{
  int value = ds_symbol_ref(ds,row,attnum);
  return ds_attnum_value_to_name(ds,attnum,value);
}

void save_datset(FILE *s,datset *ds)
{ /* As far as I can tell, this is just a backend to save_datset_csv.
     Either this or the other function should be retired. */
  int row;
  int attnum;

  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    fprintf(s,"%s%s",ds_attnum_to_name(ds,attnum),
            (attnum==ds_num_atts(ds)-1) ? "\n" : ",");

  fprintf(s,"\n");
  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
    for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    {
      char *str = mk_row_attnum_to_name(ds,row,attnum);
      fprintf(s,"%s%s",str,
            (attnum==ds_num_atts(ds)-1) ? "\n" : ",");
      free_string(str);
    }
}

/* There is already a command to transpose a datset.  It works only on real
   valued attributes.  This function is used to output something more
   importable for any other crippled software (i.e. Microsoft Excel) that
   may have trouble with the number of columns or rows in the untransposed
   datset.  Note that the results of this command will NOT be readable by
   sprat in a way that makes sense.
*/
void save_datset_transpose(FILE *s,datset *ds)
{
  int row;
  int attnum;

  for (attnum=0; attnum<ds_num_atts(ds); attnum++)
  {
    fprintf(s,"%s,",ds_attnum_to_name(ds,attnum));

    for (row=0; row<ds_num_rows(ds); row++)
    {
      char *str = mk_row_attnum_to_name(ds,row,attnum);
      fprintf(s,"%s%s",str,(row==ds_num_rows(ds)-1) ? "\n" : ",");
      free_string(str);
    }
  }
}

void ds_main(int argc, char *argv[])
{
  const char *filename = string_from_args("file",argc,argv,"default.mbl");
  datset *ds = mk_datset_from_file_simple(filename,argc,argv);
  /*
  FILE *s;
  */

/*
  if ( ds_num_rows(ds) * ds_num_atts(ds) < 1000 )
  {
    fprintf_datset(stdout,"ds",ds,"\n");
    wait_for_key();
  }
  explain_datset(ds);
  wait_for_key();
  printf("Will make all atts that I can into symbol atts\n");
  wait_for_key();
  symbolify_all_atts(ds,6,3);
  explain_datset(ds);
  wait_for_key();
  printf("Will save the all-symbol datafile in allsymb.mbl\n");
  s = fopen("allsymb.mbl","w");
  save_datset(s,ds);
  fclose(s);
  printf("...saved\n");
*/

  free_datset(ds);
}


bool filename_is_fast(const char *filename)
{
  int len = (int)strlen(filename);
  bool result = len >= 4 &&
                filename[len-4] == '.' &&
                filename[len-3] == 'f' &&
                filename[len-2] == 'd' &&
                filename[len-1] == 's';
  return(result);
}

char *format_describe(bool fast)
{
  return((fast) ? "fast \"fds\" binary" : "slow ASCII");
}

void apply_loadinst_to_datset(datset *ds,loadinst *ldi,char **r_errmess)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    char *attname = ds_attnum_to_name(ds,attnum);

    if ( loadinst_name_to_load_symbolic(ldi,attname) )
    {
      if ( ds_real_ref_ok(ds,attnum) )
	*r_errmess = mk_printf("Your loadinst (Load Instructions) said to\n"
			       "insist that attribute %s was symbolic. But\n"
			       "in the fast file format it had been stored\n"
			       "as real-valued.",attname);
    }
    else if ( ldi->default_real &&
	      !ds_real_ref_ok(ds,attnum) &&
	      ds_all_symbolic_values_are_numbers(ds,attnum) )
    {
      run_realize_silent(ds,attnum);
      run_unsymbolize(ds,attnum);
    }
  }
}

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format.

   Uses loadinst (described in shape.h) to decide how to interpret the
   file. It even tries to make sure things conform with loadinst even
   if loading from fast .fds file format.

   You can have default loading behavior with

   loadinst *ldi = mk_default_loadinst();
   char *errmess;
   datset *ds = mk_datset_from_loadinst(filename,ldi,&errmess);
   ...
   free_loadinst(ldi);

   If you want fancier loading though, then between making the default
   loadinst and calling mk_datset_from_loadinst you can request one
   of the special loadinst options using the loadinst API defined
   in shape.h.

   Errors are handled in the standard Auton way. 

   If all goes well, this returns a datset and sets *r_errmess to NULL

   If there's a problem, returns NULL and sets *r_errmess to a copy of a
   null-terminated error string which must be eventaually freed with
   free_string(...).
*/
datset *mk_datset_from_loadinst(const char *filename,
				loadinst *ldi,char **r_errmess)
{
  datset *ds;
  bool fast = filename_is_fast(filename);
  char *describe = format_describe(fast);
  if (Verbosity >= 0)
    printf("Loading dataset (%s format) from %s...\n",describe,filename);

  if ( fast )
  {
    ds = mk_datset_from_file_fast(filename, r_errmess);

    if ( *r_errmess == NULL )
      apply_loadinst_to_datset(ds,ldi,r_errmess);
  }
  else
    ds = mk_datset_from_file(filename,ldi,r_errmess);

  if ( ds != NULL ) 
    printf("Successfully Loaded %s (%s format)\n"
	   "R=%d records and M=%d attribute%s.\n",
	   filename,describe,ds_num_rows(ds),ds_num_atts(ds),
	   (ds_num_atts(ds)==1)?"":"s");
  else 
    printf("Aborted load of %s (%s format)\n",filename,describe);
  
  return(ds);
}

datset *mk_datset_from_loadinst_simple(const char *filename,loadinst *ldi)
{
  char *errmess = NULL;
  datset *ds = mk_datset_from_loadinst(filename,ldi,&errmess);
  if ( errmess != NULL )
    my_errorf("Error loading dataset from file %s:\n%s\n",filename,errmess);
  return ds;
}

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format.

   Reads extra options from the command

   You can set argc == 0 and argv == NULL for default simple behavior.

   If you use a "real" command-line argc and argv, then the loader
   will search for the following command-line parameters:

     [format <string>] where string is one of

          space : use whitespace as the separator between values
          comma : use comma's as the separator (.csv style)
          auto : automatically choose between the above two (this is default)

     [ignore_missing <string>] where string is one of
          true  : Throw away any rows that contain missing data. (this is
                  the default)
          false : Load rows with missing data. Symbolic attributes
                  get assigned the special value "?". Numeric
                  attributes get given a default value <missing_numeric_value>
                  which by default is -9999.0

    [missing_numeric_value <number>] See above.
*/
datset *ds_load_with_options(const char *filename,int argc,char *argv[],
			     char **r_errmess)
{
  loadinst *ldi = mk_loadinst_from_args(argc,argv);
  datset *ds = mk_datset_from_loadinst(filename,ldi,r_errmess);
  free_loadinst(ldi);
  return ds;
}

/* Just like ds_load_with_options except my_errors if problem. */
datset *ds_load_with_options_simple(const char *filename, int argc, char * argv[])
{
  char *errmess;
  datset *ds = ds_load_with_options(filename,argc,argv,&errmess);
  if ( errmess != NULL )
    my_error(errmess);
  return ds;
}

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format. */
datset *ds_load(const char *filename)
{
  int argc = 0;
  char **argv = NULL;
  return ds_load_with_options_simple(filename,argc,argv);
}

/* Just like ds_load except it ensures that all inputs are real-valued
   and the output is binary-valued with values "0" and "1". It also
   assumes that the output is the rightmost column. */
datset *ds_load_for_binary_outputs(char *filename)
{
  datset *ds = ds_load(filename);
  int out_attnum = ds_num_atts(ds)-1;
  int i;

  if ( !ds_symbol_ref_ok(ds,out_attnum) )
  {
    run_symbolize(ds,out_attnum,FALSE,FALSE,FALSE,TRUE,-77);
    if ( ds_attnum_to_arity(ds,out_attnum) != 2 ||
	 !eq_string(ds_attnum_value_to_name(ds,out_attnum,0),"0") ||
	 !eq_string(ds_attnum_value_to_name(ds,out_attnum,1),"1") )
      my_errorf("I expected that the output attribute %s would be\n"
		"all values of 0.0 and 1.0. But you have given me\n"
		"a dataset that violates that assumption!\n",
		ds_attnum_to_name(ds,out_attnum));
  }

  for ( i = 0 ; i < ds_num_atts(ds)-1 ; i++ )
  {
    if ( !ds_real_ref_ok(ds,i) )
    {
      if ( ds_all_symbolic_values_are_numbers(ds,i) )
	run_realize(ds,i);
      else
	my_errorf("Input attribute %s cannot be converted into numeric form\n",
		  ds_attnum_to_name(ds,i));
    }
  }
  
  return ds;
}

string_array *directory_pattern_to_filenames(char *dir, char *pattern)
{
  string_array *whole = directory_to_filenames(dir), 
    *result = mk_string_array(0);
  int i;

  for(i = 0; i < string_array_size(whole); i++)
  {
    char *term = string_array_ref(whole,i);
    char *fullpath;
    if(dir != NULL)
      fullpath = mk_concat_strings(dir, term);
    else
      fullpath = mk_copy_string(term);

    if(string_pattern_matches(pattern, term))
      add_to_string_array(result, fullpath);
    /*
      else
      printf("didn't accept %s\n",term);
    */

    free_string(fullpath);
  }

  free_string_array(whole);
  return result;
}

string_array *directory_to_filenames(char *dir)
{

  FILE* fp;
  char buff[1000];
  string_array *list = mk_string_array(0);

  if(dir == NULL)
    sprintf(buff, "dir /a-d /b > C:\\dtf.junk");
  else
    sprintf(buff, "dir /a-d /b %s > C:\\dtf.junk", dir);
  system(buff);
  fp = fopen("C:\\dtf.junk","r");
  while(fgets(buff,1000,fp)!=NULL)
  {
    buff[(int)strlen(buff)-1] = '\0';
    add_to_string_array(list, buff);
  }
  fclose(fp);
  system("del C:\\dtf.junk");

  return list;
}

/* Uses the filename to decide whether to save it in fast or slow format.
   If ends with .fds, saves in fast format. 
   If ends with anything else, saves in slow (ASCII) format. */
void ds_save(const char *filename,datset *ds)
{
  FILE *s = safe_fopen(filename,"w");
  bool fast = filename_is_fast(filename);
  char *buff = mk_printf("Saving dataset (%s format) to %s...",
                            format_describe(fast),filename);
  free_string(buff);

  if ( fast )
    save_datset_fast(s,ds);
  else
    save_datset(s,ds);

  printf("Saved %s (%s format)\n",filename,format_describe(fast));
  fclose(s);
}

/* Saves the dataset to the given stream in CSV (not fast) format */
void save_datset_csv(FILE *s,datset *ds)
{
  save_datset(s,ds);
}

void copy_main(int argc,char * argv[])
{
  const char *filename = string_from_args("in",argc,argv,"in.mbl");
  datset *ds = ds_load_with_options_simple(filename,argc,argv);
  const char *outfile = string_from_args("out",argc,argv,"out.mbl");
  explain_datset(ds);
  ds_save(outfile,ds);

  free_datset(ds);
}

int ds_name_to_attnum(datset *ds,const char *s)
{
  return(shape_name_to_attnum(ds->sh,s));
}

/* Just like ds_name_to_attnum except my_error's if no attribute
   exists with given name. So never returns -1 */
int ds_name_to_attnum_simple(datset *ds,char *attname)
{
  int attnum = ds_name_to_attnum(ds,attname);
  if ( attnum < 0 )
    my_errorf("ds_name_to_attnum_simple : no attribute called %s.\n"
	      "Available attribute names are: %s\n",
	      attname,mk_string_from_string_array(mk_attnames(ds,NULL)));
  return attnum;
}

ivec *mk_unsorted_attnums_from_string_array_basic(datset *ds,
						  string_array *sa,
						  char **r_errmess)
{
  int size = string_array_size(sa);
  ivec *attnums = mk_ivec(0);
  int i;
  *r_errmess = NULL;
  for (i = 0 ; i < size ; i++ )
  {
    int attnum = ds_name_to_attnum(ds,string_array_ref(sa,i));
    if ( attnum >= 0 ) add_to_ivec(attnums,attnum);
    else
    {
      char buff[1000];
      sprintf(buff,"There is no attribute called %s\n",string_array_ref(sa,i));
      add_to_error_message(r_errmess,buff);
    }
  }
  return(attnums);
}

ivec *mk_real_attnums(datset *ds)
{
  ivec *attnums = mk_ivec(0);
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    if ( !ds_attnum_to_symbolicp(ds,i) )
      add_to_ivec(attnums,i);
  return attnums;
}

ivec *mk_symbolic_attnums(datset *ds)
{
  ivec *attnums = mk_ivec(0);
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
    if ( ds_attnum_to_symbolicp(ds,i) )
      add_to_ivec(attnums,i);
  return attnums;
}

ivec *mk_unsorted_attnums_from_string_array(datset *ds,
					    string_array *sa,
					    char **r_errmess)
{
  ivec *result;
  *r_errmess = NULL;
  if ( string_array_size(sa) == 1 && eq_string(string_array_ref(sa,0),"all") )
    result = mk_identity_ivec(ds_num_atts(ds));
  else if ( string_array_size(sa) == 1 && 
	    eq_string(string_array_ref(sa,0),"real") )
    result = mk_real_attnums(ds);
  else if ( string_array_size(sa) == 1 && 
	    eq_string(string_array_ref(sa,0),"symbolic") )
    result = mk_symbolic_attnums(ds);
  else
    result = mk_unsorted_attnums_from_string_array_basic(ds,sa,r_errmess);
  return result;
}

ivec *mk_unsorted_attnums_from_string(datset *ds,const char *s,char **r_errmess)
{
  ivec *result;
  string_array *sa = mk_broken_string(s);
  result = mk_unsorted_attnums_from_string_array(ds,sa,r_errmess);
  free_string_array(sa);
  return result;
}

ivec *mk_unsorted_attnums_from_string_simple(datset *ds,char *s)
{
  char *errmess = NULL;
  ivec *result = mk_unsorted_attnums_from_string(ds,s,&errmess);
  if ( errmess != NULL )
    my_error(errmess);
  return result;
}

ivec *mk_remove_adjacent_duplicates(ivec *in)
{
  ivec *out = mk_ivec(0);
  int i;
  for ( i = 0 ; i < ivec_size(in) ; i++ )
  {
    if ( i == 0 || ivec_ref(out,ivec_size(out)-1) != ivec_ref(in,i) )
      add_to_ivec(out,ivec_ref(in,i));
  }
  return(out);
}

ivec *mk_sorted_attnums_from_string_array(datset *ds,
                                          string_array *sa,char **r_errmess)
{
  ivec *sorted_attnums = NULL;
  ivec *attnums;
  *r_errmess = NULL;
  attnums = mk_unsorted_attnums_from_string_array(ds,sa,r_errmess);
  if ( attnums != NULL )
  {
    sorted_attnums = mk_sivec_from_ivec(attnums);
    free_ivec(attnums);
  }
  return(sorted_attnums);
}

ivec *mk_sorted_attnums_from_string(datset *ds,const char *s,char **r_errmess)
{
  string_array *sa = mk_broken_string(s);
  ivec *attnums = mk_sorted_attnums_from_string_array(ds,sa,r_errmess);
  free_string_array(sa);
  return(attnums);
}

ivec *mk_sorted_attnums_from_string_simple(datset *ds,const char *s)
{
  char *errmess;
  ivec *attnums = mk_sorted_attnums_from_string(ds,s,&errmess);
  if ( attnums == NULL )
  {
    fprintf(stderr,"ERROR PARSING AN ATTRIBUTE NAME: %s\n",errmess);
    my_error("mk_sorted_attnums_from_string_simple");
  }
  return(attnums);
}

ivec *mk_sorted_attnums_from_args(char *key,datset *ds,int argc,char * argv[])
{
  const char *s = string_from_args(key,argc,argv,NULL);
  ivec *result = NULL;

  if ( s == NULL )
    my_errorf("I insist on you providing the name(s) of one or more\n"
	      "attribute(s) on the command line after the keyword %s\n"
	      "If you choose to use two or more attributes, enclose them\n"
	      "in quotes. Thanks.\n",key);
  else
    result = mk_sorted_attnums_from_string_simple(ds,s);

  return(result);
}

void explain_attribute_detailed(FILE *s,datset *ds,int attnum)
{
  bool symbolic = ds_attnum_to_symbolicp(ds,attnum);
  fprintf(s,"%s                 Attribute %3d (%s)\n",
	  ds_attnum_to_name(ds,attnum),attnum,
          (symbolic) ? "Symbolic" : "Real");

  fprintf(s,"    %d missing value%s\n",ds_attnum_to_num_missing(ds,attnum),
          (ds_attnum_to_num_missing(ds,attnum)==1)?"":"s");

  if ( symbolic && ds_real_ref_ok(ds,attnum) )
  {
    char *rval_string = 
      mk_ds_real_att_explain_string_basic(ds,attnum, /*do_min_and_max=*/ TRUE);
    fprintf(s,"    (real values cached: %s)\n",rval_string);
    free_string(rval_string);
  }
 
  if ( !symbolic && ds_symbol_ref_ok(ds,attnum) )
  {
    char *sval_string =
      mk_ds_symbolic_att_explain_string_basic(ds,attnum);
    fprintf(s,"    (symbol values cached: %s)\n",sval_string);
    free_string(sval_string);
  }

  if ( ds_symbol_ref_ok(ds,attnum) )
  {
    int arity = ds_attnum_to_arity(ds,attnum);
    ivec *freqs = mk_ds_frequencies(ds,attnum);
    int i = 0;
    char buff[1000];
    sprintf(buff,"%s","");
    while ( i < arity )
    {
      char newbuff[1000];
      sprintf(newbuff,"%s %s (%d)",buff,
              ds_attnum_value_to_name(ds,attnum,i),
              ivec_ref(freqs,i));
      if ( (int)strlen(buff) > 0 && (int)strlen(newbuff) > 75 )
      {
        fprintf(s,"    %s\n",buff);
        sprintf(buff,"%s","");
      }
      else
      {
        sprintf(buff,"%s",newbuff);
        i += 1;
      }
    }
    if ( (int)strlen(buff) > 0 )
      fprintf(s,"    %s\n",buff);
    free_ivec(freqs);
  }

  if ( ds_real_ref_ok(ds,attnum) )
  {
    int i;
    int num_vals = int_min(8,ds_num_rows(ds));
    fprintf(s,"    mean %g sdev %g min %g max %g\n",
            ds_att_mean(ds,attnum),
            ds_att_sdev(ds,attnum),
            ds_att_min(ds,attnum),
            ds_att_max(ds,attnum));
    fprintf(s,"    First %d vals... ",num_vals);
    for ( i = 0 ; i < num_vals ; i++ )
    {
      char *name = mk_row_attnum_to_name(ds,i,attnum);
      fprintf(s,"%s ",name);
      free_string(name);
    }
    fprintf(s,"\n");
  }
}

void explain_datset_detailed(FILE *s,datset *ds)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    explain_attribute_detailed(s,ds,attnum);
    fprintf(s,"\n");
  }
}

void details_main(int argc,char * argv[])
{
  const char *infname = string_from_args("in",argc,argv,"default.ds");
  const char *outfname = string_from_args("out",argc,argv,NULL);
  datset *ds;
  
  printf("%s %s [in datafilename] [out textfilename]\n",argv[0],argv[1]);
  printf("Loads data from datafilename and prints detailed report\n");
  printf("about the dataset. If specified, results also go to textfilename\n");
  ds = ds_load_with_options_simple(infname,argc,argv);
  explain_datset_detailed(stdout,ds);
  if ( outfname != NULL )
  {
    FILE *s = safe_fopen(outfname,"w");
    explain_datset_detailed(s,ds);
    fclose(s);
  }
  free_datset(ds);
}

/* Result ivec is of size "number of attributes".
   result[i] = number of records which with row-numbers
   included in "rows" in which attribute "attnum" has value i.

   Note rows==NULL means "use all rows". */
ivec *mk_slow_ds_counts(datset *ds,ivec *rows,int attnum)
{
  ivec *result = mk_zero_ivec(ds_attnum_to_arity(ds,attnum));
  int i;
  int num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);

  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows==NULL) ? i : ivec_ref(rows,i);
    int value = ds_symbol_ref(ds,row,attnum);
    ivec_increment(result,value,1);
  }
  return result;
}

#define LOG_2 0.693147180559945286
#define TWO_TO_MINUS_FORTY ( 1.0 / (1024.0 * 1024 * 1024 * 1024) )

/* Returns -z * log_2(z), being sensible and robust 
   when z == 0.0 */
double minus_plog2p(double z)
{
  if ( z == 0.0 || z == 1.0 )
    return(0.0);
  else if ( z > TWO_TO_MINUS_FORTY )
    return( -z * log(z) / LOG_2 );
  else
    return(40.0 * z);
}

/* Assuming that value i occurs counts[i] times, and assuming
   thus that the probability of value i is

      p_i = counts[i] / sum-of-all-counts

   this computes the entropy of this signal as

     sum_over_i = -plog2p p_i */

double entropy_from_counts(const ivec *counts)
{
  int sum = ivec_sum(counts);
  double result = 0.0;

  if ( sum < 0 )
    my_error("entropy_from_counts");
  else if ( sum == 0 )
  { /* skip: result should be 0.0 */ }
  else
  {
    int i;
    for ( i = 0 ; i < ivec_size(counts) ; i++ )
    {
      double p_i = ivec_ref(counts,i) / (double) sum;
      result += minus_plog2p(p_i);
    }
  }

  return result;
}

dyv *mk_entropy_column(const dym *P)
{
  int row;
  dyv *ret = mk_dyv(dym_rows(P));
  const int ncols = dym_cols(P);
  dyv *this_row = mk_dyv(ncols); /* created once outside the loop for speed */

  for(row=0; row<dym_rows(P); row++) {
    double sum;
    double ent = 0.0;
    copy_dym_row_to_dyv(P, this_row, row);
    sum = dyv_sum(this_row);

    if ( sum < 0 )
      my_error("mk_entropy_column");
    else if ( sum == 0 ) {
      /* skip: result should be 0.0 */
    } else {
      int i;
      for ( i = 0 ; i < ncols; i++ ) {
        const double p_i = dyv_ref(this_row,i) / sum;
        ent += minus_plog2p(p_i);
      }
    }
    dyv_set(ret, row, ent);
  }

  free_dyv(this_row);
  return ret;
}

/* This calculates and returns the entropy of the given attribute
   without using an adtree. Note that if rows==NULL, means
   "use all rows".*/
double ds_slow_attribute_entropy(datset *ds,ivec *rows,int attnum)
{
  ivec *counts = mk_slow_ds_counts(ds,rows,attnum);
  double result = entropy_from_counts(counts);
  free_ivec(counts);
  return result;
}

/* Partitions "rows" into NA subrows where 
     NA = ds_attnum_to_arity(ds,attnum)
   and where r is a member of 
     ivec_array_ref(result,v) 
   if and only if 
   
        r is in "rows" 
           AND
        the value of the attnum'th attribute on
        the r'th row is v.
*/
ivec_array *mk_subrows(datset *ds,ivec *rows,int attnum)
{
  int arity = ds_attnum_to_arity(ds,attnum);
  ivec_array *subrows = mk_array_of_zero_length_ivecs(arity);
  int i;
  int num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);
  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows==NULL) ? i : ivec_ref(rows,i);
    int value = ds_symbol_ref(ds,row,attnum);
    add_to_ivec(ivec_array_ref(subrows,value),row);
  }
  return(subrows);
}

/* PRE: The source and dest must have the same shape.
   Copies the entire contents of the source_row'th row of source_ds to
     to the dest_row'th row of dest_ds.

   It is fine for source_ds and dest_ds to be the same datset. It
   does nothing if source_ds = dest_ds and source_row = dest_row. */
void ds_copy_row(datset *source_ds,int source_row,datset *dest_ds,int dest_row)
{
  if ( source_ds == dest_ds /* pointer equality */ &&
       source_row == dest_row )
  {
    /* do nothing */
  }
  else
  {
    int i;
    for ( i = 0 ; i < ds_num_atts(source_ds) ; i++ )
    {
      if ( ds_value_is_missing(source_ds,source_row,i) )
      {
        ds_set_value_missing(dest_ds,dest_row,i);
      }
      else
      {
	if ( ds_symbol_ref_ok(source_ds,i) )
	  ds_symbol_set(dest_ds,dest_row,i,
			ds_symbol_ref(source_ds,source_row,i));

	if ( ds_real_ref_ok(source_ds,i) )
	  ds_real_set(dest_ds,dest_row,i,
		      ds_real_ref(source_ds,source_row,i));
      }
    }
  }
}

/* PRE: The source and dest must have the same shape.
   For each i, copies the entire contents of the source_rows[i]'th row 
   of source_ds to the dest_row[i]'th row of dest_ds.

   It is fine for source_ds and dest_ds to be the same datset. It
   does nothing whenever
   source_ds = dest_ds and source_row[i] = dest_row[i]. */
void ds_copy_rows(datset *source_ds,ivec *source_rows,
		  datset *dest_ds,ivec *dest_rows)
{
  if ( ivec_size(source_rows) != ivec_size(dest_rows) )
    my_error("ds_copy_rows: mismatched rows sizes");
  else
  {
    int i;
    for ( i = 0 ; i < ivec_size(source_rows) ; i++ )
    {
      int source_row = ivec_ref(source_rows,i);
      int dest_row = ivec_ref(dest_rows,i);
      ds_copy_row(source_ds,source_row,dest_ds,dest_row);
    }
  }
}

void ds_swap_values(datset *ds,int row_i,int row_j,int attnum)
{
  int i = row_i;
  int j = row_j;
  bool missing_i = ds_value_is_missing(ds,i,attnum);
  bool missing_j = ds_value_is_missing(ds,j,attnum);
  bool symbolic = ds_attnum_to_symbolicp(ds,attnum);

  if ( missing_i && missing_j )
  {
    /* skip */
  }
  else if ( missing_i )
  {
    if ( symbolic )
    {
      int valj = ds_symbol_ref(ds,j,attnum);
      ds_symbol_set(ds,i,attnum,valj);
    }
    else
    {
      double valj = ds_real_ref(ds,j,attnum);
      ds_real_set(ds,i,attnum,valj);
    }
    ds_set_value_missing(ds,j,attnum);
  }
  else if ( missing_j )
  {
    if ( symbolic )
    {
      int vali = ds_symbol_ref(ds,i,attnum);
      ds_symbol_set(ds,j,attnum,vali);
    }
    else
    {
      double vali = ds_real_ref(ds,i,attnum);
      ds_real_set(ds,j,attnum,vali);
    }
    ds_set_value_missing(ds,i,attnum);
  }
  else if ( symbolic )
  {
    int vali = ds_symbol_ref(ds,i,attnum);
    int valj = ds_symbol_ref(ds,j,attnum);
    ds_symbol_set(ds,i,attnum,valj);
    ds_symbol_set(ds,j,attnum,vali);
  }
  else
  {
    double vali = ds_real_ref(ds,i,attnum);
    double valj = ds_real_ref(ds,j,attnum);
    ds_real_set(ds,i,attnum,valj);
    ds_real_set(ds,j,attnum,vali);
  }
}

void ds_swap_rows(datset *ds,int i,int j)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    ds_swap_values(ds,i,j,attnum);
}

/* A cheap alternative to ds_shuffle for people who want to randomize
   their results a little but don't want to pay the huge computational
   cost of a full ds_shuffle
 */
void ds_partial_shuffle(datset *ds, int shuffle_size)
{
  int num_rows = ds_num_rows(ds);
  int i;
  for ( i = 0 ; i < shuffle_size ; i++ )
  {
    int j = i + int_random(num_rows-i);
    if ( j > i )
      ds_swap_rows(ds,i,j);
  }
}

void ds_shuffle(datset *ds)
{
  int num_rows = ds_num_rows(ds);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    int j = i + int_random(num_rows-i);
    if ( j > i )
      ds_swap_rows(ds,i,j);
  }
}

void ds_add_attribute_of_like_shape(const datset *source_ds,int source_attnum,
                                    datset *dest_ds)
{
  char *attname = ds_attnum_to_name(source_ds,source_attnum);
  int dest_attnum = ds_num_atts(dest_ds);
  if ( ds_symbol_ref_ok(source_ds,source_attnum) )
  {
    string_array *valnames = mk_valnames(source_ds,source_attnum);
    ds_add_symbolic_attribute(dest_ds,attname,valnames);
    if ( ds_real_ref_ok(source_ds,source_attnum) )
      ds_set_coltype(dest_ds,dest_attnum,SYMBOL_COL,REAL_COL);
    free_string_array(valnames);
  }
  else
    ds_add_real_attribute(dest_ds,attname);
}

/* Adds a new symbolic attribute to ds with given attname.

   The valnames of the new attribute are the sorted duplicateless
   version of row_to_valname

   The symbolic value stored in the i'th row of the new attribute
   has the same valname as the i'th element of row_to_valname

   Thus you can think of this operation as copying in the array of
   strings into the new column. 

   PRE: ds_num_rows(ds) == string_array_size(row_to_valname)
*/
void ds_add_column_given_valnames(datset *ds,char *attname,
				  string_array *row_to_valname)
{
  string_array *valnames = 
    mk_sort_string_array_remove_duplicates(row_to_valname);
  int num_rows = string_array_size(row_to_valname);
  int i;
  int attnum = ds_num_atts(ds);
  int rows_needed = num_rows - ds_num_rows(ds);

  my_assert(rows_needed >= 0);

  if ( rows_needed > 0 )
    ds_add_rows(ds,rows_needed);

  my_assert(num_rows == ds_num_rows(ds));

  ds_add_symbolic_attribute(ds,attname,valnames);

  my_assert(ds_num_rows(ds) <= num_rows);

  for ( i = 0 ; i < num_rows ; i++ )
  {
    char *valname = string_array_ref(row_to_valname,i);
    int value = ds_attnum_name_to_value(ds,attnum,valname);
    ds_symbol_set(ds,i,attnum,value);
  }

  for ( i = 0 ; i < num_rows ; i++ )
  {
    my_assert(eq_string(string_array_ref(row_to_valname,i),
			ds_attnum_value_to_name(ds,attnum,
						ds_symbol_ref(ds,i,attnum))));
  }

  free_string_array(valnames);
}

/* Updates the (row2,attnum2) cell of ds2 to be the same as
   the (row1,attnum1) cell of ds1.

   The copying includes the ability to copy missing-value-ness

   PRE: The shape of attnum1 in ds1 must be exactly the same as the
        shape of attnum2 in ds2.
*/
void ds_copy_value(const datset *ds1,int row1,int attnum1,
                   datset *ds2,int row2,int attnum2)
{
  if ( ds_value_is_missing(ds1,row1,attnum1) )
    ds_set_value_missing(ds2,row2,attnum2);
  else 
  {
    if ( ds_symbol_ref_ok(ds1,attnum1) )
    {
      if ( !ds_symbol_ref_ok(ds2,attnum2) )
        my_error("ds_copy_value: can't do symbol_ok to not symbol_ok");
      my_assert(ds_attnum_to_arity(ds1,attnum1) == 
		ds_attnum_to_arity(ds2,attnum2));

      ds_symbol_set(ds2,row2,attnum2,ds_symbol_ref(ds1,row1,attnum1));
    }
    else if ( ds_symbol_ref_ok(ds1,attnum1) )
      my_error("ds_copy_value: can't do not symbol_ok to symbol_ok");

    if ( ds_real_ref_ok(ds1,attnum1) )
    {
      if ( !ds_real_ref_ok(ds2,attnum2) )
        my_error("ds_copy_value: can't do real_ok to not real_ok");
      ds_real_set(ds2,row2,attnum2,ds_real_ref(ds1,row1,attnum1));
    }
    else if ( ds_real_ref_ok(ds1,attnum1) )
      my_error("ds_copy_value: can't do not real_ok to real_ok");
  }
}

/* Creates a new datset called dsnew in which the i'th row
   of dsnew is the rows[i]'th row of ds. Thus dsnew constists
   of a subset of the rows of ds, and contains ivec_size(rows)
   such rows.

   It also contains only the specified subset of attributes.

   rows may be NULL denoting "use all rows"

   attnums may be NULL denoting "use all attributes".

 The new datset has exactly the same shape as the original. For example,
 if the original had a categorical attribute called "color" with values
 red green and blue then the (provided "color" is included in attnums)
 the new datset will have a symbolic attribute called color, with the
 same three values in the shape, even if only two colors actually
 appear in the rows selected by "rows"
*/
datset *mk_ds_subset(const datset *ds,ivec *rows,ivec *attnums)
{
  int dest_num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);
  int dest_num_atts = (attnums==NULL) ? ds_num_atts(ds) : ivec_size(attnums);
  datset *dest_ds = mk_empty_datset();
  int dest_attnum;

  ds_add_rows(dest_ds,dest_num_rows);

  for ( dest_attnum = 0 ; dest_attnum < dest_num_atts ; dest_attnum++ )
  {
    int source_attnum = (attnums == NULL) ? dest_attnum : 
                        ivec_ref(attnums,dest_attnum);
    int dest_row;
    ds_add_attribute_of_like_shape(ds,source_attnum,dest_ds);
    for ( dest_row = 0 ; dest_row < dest_num_rows ; dest_row++ )
    {
      int source_row = (rows==NULL) ? dest_row : ivec_ref(rows,dest_row);
      ds_copy_value(ds,source_row,source_attnum,
		    dest_ds,dest_row,dest_attnum);
    }
  }

  return dest_ds;
}

/* Returns a random subset of { 0 , 1 , ... n-1 } of size
   subsize.

   WARNING: CURRENT IMPLEMENTATION VERY INEFFICIENT IF
   subsize is very small compared with n */
ivec *mk_random_subset_of_ints(int n,int subsize)
{
  ivec *iv = mk_identity_ivec(n);
  ivec *result = mk_ivec(subsize);
  int i;

  shuffle_ivec(iv);

  for ( i = 0 ; i < subsize ; i++ )
    ivec_set(result,i,ivec_ref(iv,i));
  free_ivec(iv);
  return result;
}

/* Makes a datset containing a randomly chosen subset
   of the rows of the original datset, in random order. */
datset *mk_ds_random_subset(datset *ds,int num_rows)
{
  ivec *rows = mk_random_subset_of_ints(ds_num_rows(ds),num_rows);
  datset *newds = mk_ds_subset(ds,rows,NULL);
  free_ivec(rows);
  return newds;
}

/* ivec_ref(result,i) = the attribute with the i'th
   fewest missing values (i == 0 means least missing) */
ivec *mk_attnums_sorted_by_num_missing(datset *ds)
{
  int num_atts = ds_num_atts(ds);
  dyv *attnum_to_num_missing = mk_dyv(num_atts);
  ivec *result;
  int attnum;
  for ( attnum = 0 ; attnum < num_atts ; attnum++ )
    dyv_set(attnum_to_num_missing,attnum,
            ds_attnum_to_num_missing(ds,attnum));
  result = mk_indices_of_sorted_dyv(attnum_to_num_missing);
  free_dyv(attnum_to_num_missing);
  return result;
}


/* Prints a sorted list of the attributes that have missing
   values */
void fprint_attributes_sorted_by_num_missing(FILE *s,datset *ds)
{
  ivec *iv = mk_attnums_sorted_by_num_missing(ds);
  int i;
  int num_atts = ds_num_atts(ds);
  bool finished = FALSE;

  for ( i = 0 ; i < num_atts && !finished ; i++ )
  {
    int attnum = ivec_ref(iv,num_atts-i-1);
    int nm = ds_attnum_to_num_missing(ds,attnum);
    if ( nm == 0 )
    {
      printf("The remaining attributes have no missing values\n");
      finished = TRUE;
    }
    else
    {
      fprintf(s,"%20s %5d missing\n",
            ds_attnum_to_name(ds,attnum),nm);
    }
  }
  free_ivec(iv);
}

string_array *mk_tokens_from_full_file(FILE *s,int max_tokens)
{
  string_array *sa = mk_string_array(0);
  bool finished = FALSE;
  int line = 0;
  while ( !finished && string_array_size(sa) < max_tokens )
  {
    string_array *next = mk_next_interesting_line(s,&line);
    if ( next == NULL )
      finished = TRUE;
    else
    {
      int i;
      for ( i = 0 ; i < string_array_size(next) && string_array_size(sa) < max_tokens ; i++ )
        add_to_string_array(sa,string_array_ref(next,i));
      free_string_array(next);
    }
  }
  return sa;
}

/* Looks at the set of tokens (space and/or line separated
   strings in the file, with blank lines or lines beginning
   with a # symbol ignored) in the file. Renames the nth attribute
   to be the nth token in the file. If two few tokens in the file
   only adjusts the FIRST k attributes. If too many, ignores 
   all but the first N. */
void rename_from_file(datset *ds,char *filename)
{
  FILE *s = fopen(filename,"r");
  if ( s == NULL )
    printf("Can't open %s for reading\n",filename);
  else
  {
    int num_atts = ds_num_atts(ds);
    string_array *sa = mk_tokens_from_full_file(s,num_atts+1);
    int sa_size = string_array_size(sa);
    int i;
    if ( sa_size < num_atts )
      printf("Dataset has %d attributes, but file only has %d names.\n"
             "Will only change first %d attribute names.\n",
             num_atts,sa_size,sa_size);
    else if ( sa_size > num_atts )
      printf("Dataset has %d attributes, but file has more than that\n"
             "many names. Will only use the first %d names in the file.\n",
             num_atts,num_atts);
    for ( i = 0 ; i < int_min(num_atts,sa_size) ; i++ )
    {
      char *newname = string_array_ref(sa,i);
      printf("Renaming %s to %s\n",ds_attnum_to_name(ds,i),newname);
      ds_rename_attribute(ds,i,newname);
    }
    free_string_array(sa);
    fclose(s);
  }
}

/* result is the smallest L such that 2^L > n */
int int_log_two(int n)
{
  unsigned int ell = (unsigned int) 0;
  unsigned int one = (unsigned int) 1;
  unsigned int bits_in_int = (unsigned int) (8 * sizeof(int));
  unsigned int un_n = (unsigned int) n;

  while ( ell < bits_in_int && ((unsigned int)(one << ell)) <= un_n )
    ell += one;
  return (int) ell;
}

/* Creates logN new binary attributes called name_bit_L, name_bit_L-1, ..
   name_bit_1, .. name_bit_0 giving the bits of attribute attnum. L equals 
   or is larger than logN where the attnum'th attribute has maxiimum
   value N. */
/* PRE: attnum is real-valued */
void ds_binary_explosion(datset *ds,int attnum)
{
  int max_abs = (int)ceil(real_max(ds_att_min(ds,attnum),ds_att_max(ds,attnum)));
  int num_bits = int_log_two(max_abs);
    /* num_bits is the smallest L such that 2^L > max_abs */
  int base_new_attnum = ds_num_atts(ds);
  int i;
  string_array *values = mk_broken_string("0 1");
  for ( i = 0 ; i < num_bits ; i++ )
  {
    char attname[100];
    sprintf(attname,"%s_bit_%d",ds_attnum_to_name(ds,attnum),i);
    ds_add_symbolic_attribute(ds,attname,values);
  }
  for ( i = 0 ; i < num_bits ; i++ )
  {
    int row;
    int new_attnum = base_new_attnum + i;
    printf("Creating attribute %s\n",ds_attnum_to_name(ds,new_attnum));
    for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
    {
      if ( ds_value_is_missing(ds,row,attnum) )
        ds_set_value_missing(ds,row,new_attnum);
      else
      {
        int v = (int)(0.00001 + ds_real_ref(ds,row,attnum));
        int bit = (v>>i) & 0x1;
        ds_symbol_set(ds,row,new_attnum,bit);
      }
    }
  }
  free_string_array(values);
}

/* Ignores symbolic attributes that are mentioned in attnums */
void ds_binary_explosion_of_attnums(datset *ds,ivec *attnums)
{
  int i;
  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    if ( ds_attnum_to_symbolicp(ds,attnum) )
      printf("%s will NOT be binary_exploded: it's symbolic.\n",ds_attnum_to_name(ds,attnum));
    else
      ds_binary_explosion(ds,attnum);
  }
}

/* Creates N new binary attributes called name_is_v1, name_is_vN, ..
   name_is_vi is 1 iff the value of the attnum attribue is equal to vi
   N is the number of values the symbolic attribute attnum takes.
   If the attribute has missing values, then N is incremented and
   the new attribute is called name_is_missing.
*/
/* PRE: attnum is symbolic */
void ds_symbolic_binary_explosion(datset *ds,int attnum)
{
  int arity = ds_attnum_to_arity(ds, attnum);
  int i;
  int base_new_attnum = ds_num_atts(ds);
  string_array *values = mk_broken_string("0 1");
  bool has_missing = FALSE;
  if(ds_attnum_to_num_missing(ds, attnum) > 0) {
    has_missing = TRUE;
    arity++;
  }

  for ( i = 0 ; i < arity ; i++ )
  {
    char attname[100];
    sprintf(attname, "%s_is_%s", ds_attnum_to_name(ds, attnum),
            i==arity-1 && has_missing ?
            "missing" : ds_attnum_value_to_name(ds, attnum, i));
    ds_add_symbolic_attribute(ds, attname, values);
  }
  for ( i = 0 ; i < arity ; i++ )
  {
    int row;
    int new_attnum = base_new_attnum + i;
    printf("Creating attribute %s\n", ds_attnum_to_name(ds, new_attnum));
    for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
    {
      if (ds_value_is_missing(ds, row, attnum))
      {
        bool b = i == arity-1;
        ds_symbol_set(ds, row, new_attnum, b);
      }
      else
      {
        bool b = ds_symbol_ref(ds, row, attnum) == i;
        ds_symbol_set(ds, row, new_attnum, b);
      }
    }
  }
  free_string_array(values);
}

/* Replaces given attribute name with their downcased versions. */
void downcase_attname(datset *ds,int attnum)
{
  char *new_name = mk_downcase_string(ds_attnum_to_name(ds,attnum));
  ds_rename_attribute(ds,attnum,new_name);
  free_string(new_name);
}

/* Replaces all attribute names with their downcased versions. */
void downcase_all_attnames(datset *ds)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
    downcase_attname(ds,attnum);
}

/* Returns -1 if all attributes are symbolic. Else returns attnum
   such that 0 <= attnum < ds_num_atts(ds) and such that
   ds_attnum_to_symbolicp(ds,attnum) is FALSE and such that forall
   i < attnum, ds_attnum_to_symbolicp(ds,i) is TRUE */
int find_leftmost_real_attnum(datset *ds)
{
  int result = -1;
  int i;
  for ( i = 0 ; result < 0 && i < ds_num_atts(ds) ; i++ )
    if ( !ds_attnum_to_symbolicp(ds,i) )
      result = i;
  return result;
}

/* Returns the fraction variance explained when using attnum1 to predict
   attnum2 using linear regression.

   PRECONDITION: attnum1 and attnum2 must be real-valued (this function will
   call my_error() [defined in ambs.h] is that's violated. */
double fraction_variance_explained(datset *ds,int attnum1,int attnum2)
{
	double Xi, Yi;
	double sumXi = 0.0; 
	double sumYi = 0.0;
	double sumXiYi = 0.0;
	double sumXiXi = 0.0;
	double sumYiYi = 0.0;
	double R = ds_num_rows(ds);
	double m, b;  /*fitting to y = mx + b*/
	double E = 0.0;
	double Ymean, YVar, numerator, denominator;
	int r;

	for(r = 0; r<R; r++){
		Xi = ds_real_ref(ds, r, attnum1);
		Yi = ds_real_ref(ds, r, attnum2);
		sumXi += Xi;
		sumYi += Yi;
		sumXiYi += Xi * Yi;
		sumXiXi += Xi * Xi;
		sumYiYi += Yi * Yi;
	}
	numerator = (((double)R)*sumXiYi - sumXi * sumYi);
	denominator = (((double)R)*sumXiXi - sumXi * sumXi);
	if(denominator == 0.0) return 0.0; /*no x variance.  
										cannot make good prediction.  
										say 0 correlation.*/

	m = numerator / denominator;
	b = (sumYi - m*sumXi)/((double)R);

	/* E = sum{( yi - m*xi - b )^2}
		 = sum{( yi^2 - 2m xi yi - 2yi b + 2m xi b +m^2 xi^2 +b^2 } */
	E = sumYiYi - 2.0*m*sumXiYi - 2.0*b*sumYi + 
		2.0*m*b*sumXi + m*m*sumXiXi + b*b*((double) R);
	E /= ((double) R);

	Ymean = sumYi/((double) R);
	YVar = sumYiYi/((double) R) - Ymean*Ymean;
	if( YVar == 0.0 ) return 0.0; /*no y variance. 
								   cannot make good prediction. */
	
	/*printf("m is %g and b is %g\n", m, b);
	printf("YVariance is %g and E is %g\n", YVar, E);*/

	return( 1.0 - E / YVar );	
}

bool check_and_warn_ds_has_missing_values(datset *ds)
{
  bool result = FALSE;
  int a;
  for ( a = 0 ; a < ds_num_atts(ds) ; a++ )
  {
    int nm = ds_attnum_to_num_missing(ds,a);
    if ( nm > 0 )
    {
      printf("Attribute %s has %d missing value(s)\n",
             ds_attnum_to_name(ds,a),nm);
      result = TRUE;
    }
  }
  return result;
}

int ds_value_from_string_no_ambiguity(datset *ds,int attnum,char *s){
  int ret = ds_attnum_name_to_value(ds,attnum,s);
  if(ret==-1 && is_a_number(s)) ret = atoi(s)-1;
  return ret;
}

int ds_value_from_string(datset *ds,int attnum,char *s){
  int value;
  int value_from_name = ds_attnum_name_to_value(ds,attnum,s);

  if ( !is_a_number(s) )
    value = value_from_name;
  else
  {
    int value_from_number = atoi(s);
    if ( value_from_name < 0 )
      value = value_from_number;
    else if ( value_from_number >= ds_attnum_to_arity(ds,attnum) )
      value = value_from_name;
    else if ( value_from_name == value_from_number )
      value = value_from_name;
    else
    {
      printf("I'm very sorry, but it is ambiguous whether you\n"
             "mean the value with index %d (which would be \"%s\")\n"
             "or the actual value \"%s\". I am going to assume the\n"
             "former.\n",value_from_number,
             ds_attnum_value_to_name(ds,attnum,value_from_number),
             ds_attnum_value_to_name(ds,attnum,value_from_name));
      value = value_from_number;
    }
  }             
  if ( value < 0 || value >= ds_attnum_to_arity(ds,attnum) )
    value = -1;
  return value;
}

/* Returns a ds with the same shape, but in which all real values
   are randomly chosen uniformly between min and max of their
   attribute, and all symbol values are uniformly randomly chosen.
   There are no missing values in the result. */
datset *mk_random_ds_of_like_shape(datset *ds)
{
  datset *newds = mk_copy_datset(ds);
  int row,attnum;
  int num_atts = ds_num_atts(newds); 
  int num_rows = ds_num_rows(newds);
  for ( attnum = 0 ; attnum < num_atts ; attnum++ )
  {
    if ( ds_attnum_to_symbolicp(newds,attnum) )
    {
      int arity = ds_attnum_to_arity(newds,attnum);
      ds_set_coltype(newds,attnum,SYMBOL_COL,UNDEFINED_COL);
      for ( row = 0 ; row < num_rows ; row++ )
	ds_symbol_set(newds,row,attnum,int_random(arity));
    }
    else
    {
      double minval = ds_att_min(newds,attnum);
      double maxval = ds_att_max(newds,attnum);

      ds_set_coltype(newds,attnum,REAL_COL,UNDEFINED_COL);
      for ( row = 0 ; row < num_rows ; row++ )
	ds_real_set(newds,row,attnum,range_random(minval,maxval));
    }
  }
  return newds;
}
      
/* rows may be NULL, meaning use all rows.
   attnums may be NULL meaning use all attnums.
*/
dym *mk_dym_from_part_of_datset(datset *ds,ivec *rows,ivec *attnums)
{
  int num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);
  int num_cols = (attnums==NULL) ? ds_num_atts(ds) : ivec_size(attnums);
  dym *x = mk_dym(num_rows,num_cols);
  int i,j;

  ds_make_as_real_as_possible(ds);

  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows==NULL) ? i : ivec_ref(rows,i);
    for ( j = 0 ; j < num_cols ; j++ )
    {
      int attnum = (attnums==NULL) ? j : ivec_ref(attnums,j);
      double val = ds_real_ref_ok(ds,attnum) ? ds_real_ref(ds,row,attnum) :
	           ds_symbol_ref(ds,row,attnum);
      dym_set(x,i,j,val);
    }
  }

  return x;
}

/* Returns a matrix whose (i,j)'th element has the same value as the
   j'th attribute in the i'th row of the dataset */
dym *mk_dym_from_datset(datset *ds)
{
  return mk_dym_from_part_of_datset(ds,NULL,NULL);
}

/* Loads a dym from a file named in filename. Uses the fast auton
   formats if the filename so indicates (e.g. if it ends with .fds). But
   uses slow simple .csv (comma separated) format otherwise.

   Signals an error and exits the whole program if can't find the file
   or there's a syntax error while loading. But in that case prints a 
   message to the console explain what was wrong. */
dym *mk_dym_from_ds_file(const char *filename,int argc,char * argv[])
{
  datset *ds = ds_load_with_options_simple(filename,argc,argv);
  int num_rows = int_from_args("num_rows",argc,argv,999999999);
  dym *x;

  num_rows = int_min(ds_num_rows(ds),num_rows);
  if ( num_rows < ds_num_rows(ds) )
  {
    datset *subds;
    printf("Making a random subset of %d rows out of %d\n",num_rows,
           ds_num_rows(ds));
    subds = mk_ds_random_subset(ds,num_rows);
    free_datset(ds);
    ds = subds;
    printf("..finished making subset\n");
  }

  x = mk_dym_from_datset(ds);
  free_datset(ds);
  return x;
}

/* Searches for key on the command line, and uses subsequent token
   as the filename. MAKES and returns the column names in *r_attnames.

   Can handle .fds files and .csv files (plus space separated or bar
   separated)

   Automatically realizes any attributes that are symbolic but which
   have symbolic values that can be parsed as real.

   If it's a csv file with no attribute names at the top, invents
   attribute names x0 , x1 , x2 .. */
dym *mk_dym_and_attnames_from_args(char *key,int argc,char *argv[],
				   string_array **r_attnames)
{
  char *filename = string_from_args("in",argc,argv,"default.csv");
  datset *ds = mk_real_datset_from_file(filename);
  dym *x = mk_dym_from_datset(ds);

  *r_attnames = mk_attnames(ds,NULL);

  free_datset(ds);

  return x;
}

int ds_num_products(datset *ds,ivec *attnums)
{
  int i,out=1;
  for(i=0;i<ivec_size(attnums);i++) out *= ds_attnum_to_arity(ds,ivec_ref(attnums,i));
  return out;
}

/* Makes a dym from the file "filename". If filename is in .fds
   format, loads it fds-style. (.fds is the Auton-style fast-load-and-save
   format of file).

   my_error()'s and exits if there's a missing file or syntax error
*/
dym *mk_dym_from_any_file_format(char *filename)
{
  dym *x;

  if ( filename_is_fast(filename) )
  {
    datset *ds = ds_load(filename);
    x = mk_dym_from_datset(ds);
    free_datset(ds);
  }
  else
    x = mk_dym_from_filename_simple(filename);
  
  return x;
}

/* Makes a dym from the file "filename". If filename is in .fds
   format, loads it fds-style. (.fds is the Auton-style fast-load-and-save
   format of file).

   my_error()'s and exits if there's a missing file or syntax error

   If num_points < 0 or num_points >= number of points in the datafile,
   returns the full file. But if 0 <= num_points < full file size, returns
   a random sample of the rows in the file.
*/ 
dym *mk_dym_from_any_file_format_maybe_sample(char *filename,int num_points)
{
  dym *x = mk_dym_from_any_file_format(filename);
  if ( num_points >= 0 && dym_rows(x) > num_points )
  {
    dym *newx = mk_dym(num_points,dym_cols(x));
    ivec *iv = mk_identity_ivec(dym_rows(x));
    int i;
    shuffle_ivec(iv);
    for ( i = 0 ; i < num_points ; i++ )
    {
      int row = ivec_ref(iv,i);
      dyv *dv = mk_dyv_from_dym_row(x,row);
      copy_dyv_to_dym_row(dv,newx,i);
      free_dyv(dv);
    }
    free_ivec(iv);
    free_dym(x);
    x = newx;
  }
  return x;
}

shape *mk_all_symbolic_shape_simple(int num_atts)
{
  string_array *attnames = mk_default_attribute_names(num_atts);
  shape *sh = mk_shape(attnames);
  free_string_array(attnames);
  return sh;
}

shape *mk_all_real_shape_simple(int num_atts)
{
  shape *sh = mk_all_symbolic_shape_simple(num_atts);
  int i;
  for ( i = 0 ; i < num_atts ; i++ )
    shape_set_coltype(sh,i,REAL_COL,UNDEFINED_COL);
  return sh;
}

datset *mk_datset_from_dym(dym *x)
{
  int num_rows = dym_rows(x);
  int num_cols = dym_cols(x);
  shape *sh = mk_all_real_shape_simple(num_cols);
  datset *ds = mk_zero_datset(sh,num_rows);
  int i,j;
  for ( i = 0 ; i < num_rows ; i++ )
    for ( j = 0 ; j < num_cols ; j++ )
      ds_real_set(ds,i,j,dym_ref(x,i,j));
  free_shape(sh);
  return ds;
}

/* Saves dym to the specified filename. If a special auton name (e.g.
   .fds format) saves it in fast efficient easy-to-load format. Else saves
   it in a simple .csv format */
void save_dym_to_any_file_format(char *filename,dym *x)
{
  if ( filename_is_fast(filename) )
  {
    datset *ds = mk_datset_from_dym(x);
    ds_save(filename,ds);
    free_datset(ds);
  }
  else
    save_dym_to_filename(filename,x);
}

void datset_change_arity(datset *ds,int attnum,int arity)
{
  if ( !ds_real_ref_ok(ds,attnum) )
    printf("Not changing arity of %s because it is not real valued\n",
	   ds_attnum_to_name(ds,attnum));
  else
  {
    ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);
    run_symbolize(ds,attnum,TRUE,FALSE,FALSE,FALSE,arity);
  }
}	   

void datset_integerize(datset *ds,int attnum)
{
  if ( !ds_real_ref_ok(ds,attnum) )
    printf("Not integerizing %s because it is not real valued\n",
	   ds_attnum_to_name(ds,attnum));
  else
  {
    ds_set_coltype(ds,attnum,REAL_COL,UNDEFINED_COL);
    run_symbolize(ds,attnum,FALSE,FALSE,FALSE,TRUE,-1);
  }
}	   

/* This is a less disruptive version of datset_symbolize_all.  It only
   operates on attributes that don't already have symbol values.  It is
   assumed that if they have symbol values, its because the user wanted
   them that way.
*/
void datset_symbolize_all_not_symbolized(datset *ds)
{
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if (ds_symbol_ref_ok(ds,i)) continue;

    if ( !ds_real_ref_ok(ds,i) &&
         ds_all_symbolic_values_are_numbers(ds,i) &&
	 ds_attnum_to_arity(ds,i) > 2 )
      run_realize(ds,i);

    if ( ds_real_ref_ok(ds,i) && 
         !(ds_symbol_ref_ok(ds,i) && ds_attnum_to_arity(ds,i) == 2 ) )
      datset_change_arity(ds,i,2);
  }
}

/* This function makes sure that every attribute has symbolic
   values. It also makes sure that if an attribute is symbolic,
   but all its values can be parsed as numbers and its arity
   is greater than two, that the symbolic values are then
   obtained by quantization to two levels */
void datset_symbolize_all(datset *ds)
{
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( !ds_real_ref_ok(ds,i) &&
         ds_all_symbolic_values_are_numbers(ds,i) &&
	 ds_attnum_to_arity(ds,i) > 2 )
      run_realize(ds,i);

    if ( ds_real_ref_ok(ds,i) && 
         !(ds_symbol_ref_ok(ds,i) && ds_attnum_to_arity(ds,i) == 2 ) )
      datset_change_arity(ds,i,2);
  }
}

/* When a datset is loaded there can be some attributes
   which only have symbolic attributes, but for which those
   symbolic values are all strings that can be parsed as numbers.
   This function "cures" all such attributes, and makes them
   primarily real-valued, but retaining the original symbolic
   values. */
void datset_realize_all_possible(datset *ds)
{
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( !ds_real_ref_ok(ds,i) &&
         ds_all_symbolic_values_are_numbers(ds,i) )
      run_realize(ds,i);
  }
}

string_array *mk_string_array_from_attnums(datset *ds,ivec *attnums)
{
  int size = ivec_size(attnums);
  string_array *sa = mk_string_array(size);
  int i;
  for ( i = 0 ; i < size ; i++ )
    string_array_set(sa,i,ds_attnum_to_name(ds,ivec_ref(attnums,i)));
  return sa;
}

char *mk_string_from_attnums(datset *ds,ivec *attnums)
{
  char *result;

  if ( ivec_size(attnums) == 0 )
    result = mk_printf("<empty set of attributes>");
  else
  {
    string_array *sa = mk_string_array_from_attnums(ds,attnums);
    result = mk_string_from_string_array(sa);
    free_string_array(sa);
  }
  return result;
}

char *filename_from_args(int argc,char * argv[])
{
  char *filename = NULL;
  char *errmess = NULL;

  if ( argc < 2 )
    errmess = mk_printf("Hey, you need to give a data filename");
  else
  {
    if ( filename == NULL && file_exists(argv[1]) )
    {
      filename = argv[1];
      printf("Using filename %s (first argument on command line)\n",filename);
    }
    
    if ( filename == NULL )
    {
      filename = string_from_args("in",argc,argv,NULL);
      if ( filename != NULL )
      {
	if ( !file_exists(filename) )
	{
	  errmess = mk_printf("You indicated you wanted datafile %s by\n"
			      "putting \"in %s\" on the command line.\n"
			      "But that file doesn't exist.\n",
			      filename,filename);
	  filename = NULL;
	}
      }
    }

    if ( filename == NULL && file_exists(argv[2]) )
    {
      filename = argv[2];
      printf("Using filename %s (second argument on command line)\n",filename);
    }

    if ( filename == NULL )
      errmess = mk_printf("No data filename supplied.\n");
  }

  if ( filename == NULL ) 
    my_errorf("DATA FILE NAME MISSING:\n"
	      "I expect to either see a filename as the first or second\n"
	      "argument on the user command line, or else to appear\n"
	      "somewhere on the command line proceeded by the token \"in\".\n"
	      "But I either didn't find a filename on your command line,\n"
	      "or the corresponding file did not exist.\n");

  return filename;
}
    

datset *mk_datset_from_argc_argv(int argc,char * argv[])
{
  const char *filename = filename_from_args(argc,argv);
  datset *ds = ds_load_with_options_simple(filename,argc,argv);
  return ds;
}

/* In all of the following functions, rows may be NULL, denoting "use all
   rows". And attnums may be NULL denoting "USE ALL ATTRIBUTES".

   Implicit precondition that all values referenced in the dataset are
   real-valued and not-missing.

   For an automatic conversion to 1-of-N coding method that handles
   symbolic attributes, see Jeff's mk_dym_from_datset_portion in regress.h 
*/
dym *mk_dym_from_part_of_datset_column(datset *ds,ivec *rows,int attnum)
{
  ivec *attnums = mk_ivec_1(attnum);
  dym *x = mk_dym_from_part_of_datset(ds,rows,attnums);
  free_ivec(attnums);
  return x;
}

dyv *mk_dyv_from_part_of_datset_column(datset *ds,ivec *rows,int attnum)
{
  int num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);
  dyv *x = mk_dyv(num_rows);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows==NULL) ? i : ivec_ref(rows,i);
    dyv_set(x,i,ds_real_ref(ds,row,attnum));
  }
  return x;
}

ivec *mk_ivec_from_part_of_datset_column(datset *ds,ivec *rows,int attnum)
{
  int num_rows = (rows==NULL) ? ds_num_rows(ds) : ivec_size(rows);
  ivec *x = mk_ivec(num_rows);
  int i;
  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows==NULL) ? i : ivec_ref(rows,i);
    ivec_set(x,i,ds_symbol_ref(ds,row,attnum));
  }
  return x;
}
  
/* attnums may be null denoting use all attnums */
dyv *mk_dyv_from_part_of_datset_row(datset *ds,int row,ivec *attnums)
{
  int num_atts = (attnums==NULL) ? ds_num_atts(ds) : ivec_size(attnums);
  dyv *x = mk_dyv(num_atts);
  int i;
  for ( i = 0 ; i < num_atts ; i++ )
  {
    int attnum = (attnums==NULL) ? i : ivec_ref(attnums,i);
    dyv_set(x,i,ds_real_ref(ds,row,attnum));
  }
  return x;
}

/* dyv_array_ref(result,i) = dyv obtained from the row[i]'th
   row of the dataset */
/* rows may be NULL denoting use all rows */
/* attnums may be null denoting use all attnums */
dyv_array *mk_dyv_array_from_part_of_datset(datset *ds,ivec *rows,
					    ivec *attnums)
{
  int num_rows, i, row;
  ivec *my_rows;
  dyv *xi;
  dyv_array *x;
  
  my_rows = (rows == NULL) ? mk_identity_ivec( ds_num_rows( ds)) : rows;
  num_rows = ivec_size( my_rows);
  x = mk_dyv_array( num_rows);

  for (i=0; i<num_rows; i++)
  {
    row = ivec_ref( my_rows, i);
    xi = mk_dyv_from_part_of_datset_row( ds, row, attnums);
    /* We know that there is no sense in copying this dyv. */
    dyv_array_set_no_copy( x, i, xi);
  }

  if (rows == NULL) free_ivec( my_rows);

  return x;
}

/* Loads a datset in which all attributes are forced to be
   real (and not symbolic), and in which there are no missing values.

   If it discovers this is impossible (for example if one attribute
   has values that cannot be parsed as numbers) my_error()'s with a detailed
   error message.

   If it loads from fds format and finds some attributes are symbolic
   but with all parseable-as-numbers values, does the conversion 
   automatically without complaint. */
datset *mk_real_datset_from_file(char *filename)
{
  datset *ds = ds_load(filename);
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( !ds_real_ref_ok(ds,i) )
    {
      if ( ds_all_symbolic_values_are_numbers(ds,i) )
      {
	run_realize(ds,i);
	run_unsymbolize(ds,i);
      }
      else
      {
	int badval = -1;
	int v;
	for ( v = 0 ; badval < 0 && v < ds_attnum_to_arity(ds,i) ; v++ )
	{
	  char *name = ds_attnum_value_to_name(ds,i,v);
	  if ( !is_a_number(name) )
	    badval = v;
	}
	my_assert_always(badval >= 0);

	my_errorf("Error loading an ALL-REAL dataset from file %s.\n"
		  "You should specify a dataset in which all the attributes\n"
		  "have real values. Attribute %s has at least one value\n"
		  "that cannot be parsed as a number: %s\n",filename,
		  ds_attnum_to_name(ds,i),
		  ds_attnum_value_to_name(ds,i,badval));
      }
    }
    if ( ds_attnum_to_num_missing(ds,i) > 0 )
      my_errorf("Error loading an ALL-REAL dataset from file %s.\n"
		"You should specify a dataset in which there are\n"
		"no missing values. Attribute %s has %d missing value(s)\n",
		  ds_attnum_to_name(ds,i),
		  ds_attnum_to_num_missing(ds,i));
  }
  return ds;
}

dyv_array *mk_dyv_array_from_file(char *filename)
{
  datset *ds = mk_real_datset_from_file(filename);
  dyv_array *da = mk_dyv_array_from_part_of_datset(ds,NULL,NULL);
  free_datset(ds);
  return da;
}

dyv_array *mk_dyv_array_from_args(char *key,int argc,char *argv[])
{
  char *filename = string_from_args(key,argc,argv,NULL);
  if ( filename == NULL )
    my_errorf("You need to specify a dataset name after the keyword \"%s\"\n"
	      "on the command line.\n",key);
  return mk_dyv_array_from_file(filename);
}

/* Returns all rows such that row_to_marked[row] = do_marked */
ivec *mk_rows_from_marks(ivec *row_to_marked,bool do_marked)
{
  ivec *rows = mk_ivec(0);
  int row;
  for ( row = 0 ; row < ivec_size(row_to_marked) ; row++ )
  {
    bool row_is_marked = ivec_ref(row_to_marked,row) != 0;
    if ( row_is_marked == do_marked )
      add_to_ivec(rows,row);
  }
  return rows;
}

/* Returns all rows such that row_to_marked[row] = TRUE */
ivec *mk_rows_from_row_to_marked(ivec *row_to_marked)
{
  return mk_rows_from_marks(row_to_marked,TRUE);
}

dym *mk_simple_dym_from_ds_file(const char *filename, int argc, char *argv[])
{
  datset *ds = ds_load_with_options_simple(filename,argc,argv);
  int num_rows = int_from_args("num_rows",argc,argv,999999999);
  dym *x;

  num_rows = int_min(ds_num_rows(ds),num_rows);
  if ( num_rows < ds_num_rows(ds) )
  {
    datset *subds;
    printf("Making a random subset of %d rows out of %d\n",num_rows,
           ds_num_rows(ds));
    subds = mk_ds_random_subset(ds,num_rows);
    free_datset(ds);
    ds = subds;
    printf("..finished making subset\n");
  }

  ds_realize_all(ds);

  x = mk_simple_dym_from_datset(ds);

  free_datset(ds);
  return x;
}

void ds_realize_all(datset *ds)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    if ( ds_attnum_to_symbolicp(ds,attnum) )
    {
      if ( ds_all_symbolic_values_are_numbers(ds,attnum) )
        ds_realize(ds,attnum);
      else
      {
        my_error(mk_printf("I cannot convert attribute %s to numeric\n",
                           ds_attnum_to_name(ds,attnum)));
      }
    }
  }
}

dym *mk_simple_dym_from_datset(const datset *ds)
{
  int rows = ds_num_rows(ds);
  int cols = ds_num_atts(ds);
  dym *x = mk_dym(rows,cols);
  int i,j;

  for ( i = 0 ; i < rows ; i++ )
    for ( j = 0 ; j < cols ; j++ )
      dym_set(x,i,j,ds_real_ref(ds,i,j));

  return x;
}

string_array *mk_valnames(const datset *ds,int attnum)
{
  int i;
  int arity = ds_attnum_to_arity(ds,attnum);
  string_array *valnames = mk_string_array(arity);

  for ( i = 0 ; i < arity ; i++ )
    string_array_set(valnames,i,ds_attnum_value_to_name(ds,attnum,i));
  return valnames;
}

/* result[i] = valname of values[i] in attnum'th attribute of ds */
string_array *mk_valnames_from_values(datset *ds,int attnum,ivec *values)
{
  int size = ivec_size(values);
  string_array *valnames = mk_string_array(size);
  int i;

  for ( i = 0 ; i < size ; i++ )
  {
    int value = ivec_ref(values,i);
    char *valname = ds_attnum_value_to_name(ds,attnum,value);
    string_array_set(valnames,i,valname);
  }

  return valnames;
}

/* If every element in valnames can be parsed as a legal value of
   the given attribute then returns a result ivec in which

     result[i] = value of attnum corresponding to name valnames[i] 

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_values_from_valnames(datset *ds,int attnum,string_array *valnames,
			      char **r_errmess)
{
  int size = string_array_size(valnames);
  int i;
  ivec *values = mk_ivec(size);

  my_assert(*r_errmess == NULL);

  for ( i = 0 ; *r_errmess == NULL && i < size ; i++ )
  {
    char *valname = string_array_ref(valnames,i);
    int value = ds_attnum_name_to_value(ds,attnum,valname);
    if ( value < 0 )
      *r_errmess = mk_printf("There are no records in which attribute %s has\n"
			     "value %s\n",ds_attnum_to_name(ds,attnum),
			     valname);
    else
      ivec_set(values,i,value);
  }

  if ( *r_errmess != NULL )
  {
    free_ivec(values);
    values = NULL;
  }

  return values;
}

/*
Makes an ivec of all the rows in ds in which attnum has a value
 included in the ivec "values".

It is NOT required that values is a sivec.
 */
ivec *mk_rows_from_attnum_and_values(datset *ds,int attnum,ivec *values)
{
  ivec *siv = mk_sivec_from_ivec(values);
  int row;
  ivec *rows = mk_ivec(0);

  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    int value = ds_symbol_ref(ds,row,attnum);
    if ( is_in_sivec(siv,value) )
      add_to_ivec(rows,row);
  }

  free_ivec(siv);

  return rows;
}

/*
Makes an ivec of all the rows in ds in which attnum has a value
whose name is include in "valnames"

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_rows_from_attnum_and_valnames(datset *ds,int attnum,
				       string_array *valnames,char **r_errmess)
{
  ivec *values = mk_values_from_valnames(ds,attnum,valnames,r_errmess);
  ivec *rows;

  if ( values == NULL )
    rows = NULL;
  else
    rows = mk_rows_from_attnum_and_values(ds,attnum,values);

  if ( values != NULL )
    free_ivec(values);

  return rows;
}

/* If every element in valnames can be parsed as a legal value of
   the given attribute then returns a result sivec (set of integers)
   in which 

     v is in result if and only if v's name is in valnames

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_values_sivec_from_valnames(datset *ds,int attnum,
				    string_array *valnames,char **r_errmess)
{
  ivec *values = mk_values_from_valnames(ds,attnum,valnames,r_errmess);
  ivec *result = NULL;
  if ( values != NULL )
    result = mk_sivec_from_ivec(values);

  if ( values != NULL ) free_ivec(values);

  return result;
}


/* NULL if no problem */
char *mk_attnum_choice_errmess(datset *ds,char *options,int attnum)
{
  bool none_ok = char_is_in(options,'n');
  bool symbol_ok = char_is_in(options,'s');
  bool real_ok = char_is_in(options,'r');
  bool missing_ok = char_is_in(options,'m');
  char *result;
  char *string = (attnum < 0) ? "none" : ds_attnum_to_name(ds,attnum);
  if ( attnum >= 0 && ds_symbol_ref_ok(ds,attnum) && symbol_ok )
  {
    if ( missing_ok || ds_attnum_to_num_missing(ds,attnum)==0 )
      result = NULL;
    else
      result = mk_printf("%s is illegal because it has missing values\n",string);
  }
  else if ( attnum >= 0 && ds_real_ref_ok(ds,attnum) && real_ok )
  {
    if ( missing_ok || ds_attnum_to_num_missing(ds,attnum)==0 )
      result = NULL;
    else
      result = mk_printf("%s is illegal because it has missing values\n",string);
  }
  else if ( none_ok && attnum < 0 )
    result = NULL;
  else
  {
    if ( attnum < 0 )
      result = mk_printf("You must choose an attribute. \"none\" is unacceptable.\n");
    else if ( attnum < 0 )
      result = mk_printf("There is no such attribute as %s\n",string);
    else if ( !real_ok )
      result = mk_printf("%s is real_valued. I insist on symbol_valued\n",string);
    else
      result = mk_printf("%s is symbol_valued. I insist on real_valued\n",string);
  }

  return result;
}

bool attnum_choice_ok(datset *ds,char *options,int attnum)
{
  char *errmess = mk_attnum_choice_errmess(ds,options,attnum);
  bool result = errmess == NULL;
  if ( errmess != NULL ) free_string(errmess);
  return result;
}

int safe_default_attnum(datset *ds,char *options,int default_attnum,bool *okay)
{
  int result = 7777;

  if ( attnum_choice_ok(ds,options,default_attnum) )
  {
    *okay = TRUE;
    result = default_attnum;
  }
  else
  {
    int i = default_attnum+1;
    *okay = FALSE;
    if ( i >= ds_num_atts(ds) )
      i = -1;

    while ( !(*okay) && i != default_attnum )
    {
      if ( attnum_choice_ok(ds,options,i) )
      {
        *okay = TRUE;
        result = i;
      }
      else
      {
        i += 1;
        if ( i >= ds_num_atts(ds) )
          i = -1;
      }
    }
  }

  return result;
}



/* Returns an attnum or else -1 denoting "no attribute".

   options is a string:

      if contains 'n' allows the user to state "none"
      if contains 'r' allows the user to select real attribute
      if contains 's' allows the user to select symbol attribute
      if contains 'm" allows the user to select attribute with missing values
*/
int attnum_from_string(datset *ds,const char *name,char *options,
                       char **r_errmess)
{
  bool okay = TRUE;
  int attnum = safe_default_attnum(ds,options,0,&okay);

  *r_errmess = NULL;

  if ( !okay )
    *r_errmess = 
      mk_printf("I cannot run this command because there is no\n"
		"possible legal choice of an attribute satisfying \"%s\".\n"
		"This happened for attribute %s",
		options,name);

  if ( *r_errmess == NULL )
  {
    attnum = ds_name_to_attnum(ds,name);
    if ( attnum < 0 )
      *r_errmess = mk_printf("There is no attribute called %s",name);
  }

  if ( *r_errmess == NULL )
    *r_errmess = mk_attnum_choice_errmess(ds,options,attnum);

  return attnum;
}

/* Returns an attnum matching the constraints using the command-line.

   my_error's if user enters something illegal or if it can't find an
   attribute to satisfy the command line.

   options is a string:

      if contains 'n' allows the user to state "none"
      if contains 'r' allows the user to select real attribute
      if contains 's' allows the user to select symbol attribute
      if contains 'm" allows the user to select attribute with missing values
*/
int attnum_from_args(datset *ds,const char *key,char *options,
		     int argc, char *argv[])
{
  const char *name = string_from_args(key,argc,argv,NULL);
  char *errmess;
  int attnum = -77;
  if ( name == NULL )
    my_errorf("You must supply %s <attribute> on the command line",key);
  attnum = attnum_from_string(ds,name,options,&errmess);
  if ( errmess != NULL )
    my_errorf("Problem with specifying %s %s on the command line: %s",
	      key,name,errmess);
  return attnum;
}



/* Returns a value for the given attribute. my_error's if there's a problem.
*/
int value_from_args(datset *ds,int attnum,
		    const char *key,int argc, char *argv[])
{
  const char *name = string_from_args(key,argc,argv,NULL);
  int value;

  if ( name == NULL )
    my_errorf("You must supply a value for attribute %s with option -%s "
              "on the command line",ds_attnum_to_name(ds,attnum),key);

  value = ds_attnum_name_to_value(ds,attnum,(char *) name);

  if ( value < 0 )
  {
    int i;
    printf("Attribute %s has no value %s. Here are its values\n",
	   ds_attnum_to_name(ds,attnum),name);
    for ( i = 0 ; i < ds_attnum_to_arity(ds,attnum) ; i++ )
      printf("%s ",ds_attnum_value_to_name(ds,attnum,i));
    printf("\n");
    my_error("So try again.\n");
  }
  return value;
}

/* NULL attnums means "use all atributes" */
string_array *mk_attnames(datset *ds,ivec *attnums)
{
  int size = (attnums == NULL) ? ds_num_atts(ds) : ivec_size(attnums);
  string_array *sa = mk_string_array(size);
  int i;
  for ( i = 0 ; i < size ; i++ )
  {
    int attnum = (attnums == NULL) ? i : ivec_ref(attnums,i);
    string_array_set(sa,i,ds_attnum_to_name(ds,attnum));
  }
  return sa;
}

/* Loads a datset in which the caller forces, via the "assume" string,
   exactly what the attributes are called and what coltypes (REAL or SYMBOLIC)
   each attribute has.

   For example, 

    datset *ds = ds_load_assuming("plop.csv","height/r gender/s age/r");

   Loads a datset from the file plop.csv, but insists that there are
   exactly three attributes. The first attribute must be interpretable
   as real-valued (and will be loaded as
   real valued), and must be called height. The second will be loaded
   as symbolic and must be called gender. The third will be called age
   and must be interpretable as real-valued (and will be loaded as
   real valued).
*/
datset *ds_load_assuming(char *filename,char *assume)
{
  string_array *sa = mk_broken_string(assume);
  loadinst *ldi = mk_default_loadinst();
  int i;
  string_array *attnames = mk_string_array(0);
  datset *ds;
  ivec *real_atts = mk_ivec(0);

  loadinst_set_default_real(ldi,TRUE);

  for ( i = 0 ; i < string_array_size(sa) ; i++ )
  {
    char *ass = string_array_ref(sa,i);
    bool symbolic = -77;
    char *attname = NULL;
    string_array *sub = mk_broken_string_using_seppers(ass,"/");

    if ( string_array_size(sub) == 1 )
    {
      symbolic = FALSE;
      attname = string_array_ref(sub,0);
    }
    else if ( string_array_size(sub) != 2 ||
         ( !eq_string(string_array_ref(sub,1),"s") && 
	   !eq_string(string_array_ref(sub,1),"r") ) )
    {
      my_errorf("I expect assume strings to be of the form <attname>/X\n"
		"where X is r or s. But you gave me %s\n",ass);
    }
    else
    {
      attname = string_array_ref(sub,0);
      symbolic = eq_string(string_array_ref(sub,1),"s");
    }

    if ( symbolic )
      loadinst_add_force_symbolic(ldi,attname);
    else
      add_to_ivec(real_atts,i);
    add_to_string_array(attnames,attname);

    free_string_array(sub);
  }

  ds = mk_datset_from_loadinst_simple(filename,ldi);

  if ( ds_num_atts(ds) != string_array_size(attnames) )
    my_errorf("Error loading dataset from %s: I expected\n"
	      "%d attributes (with the following names):\n%s\n"
	      "but you gave me a dataset with %d attributes:\n%s",
	      filename,string_array_size(attnames),
	      mk_string_from_string_array(attnames),
	      ds_num_atts(ds),mk_attnames(ds,NULL));

  for ( i = 0 ; i < string_array_size(attnames) ; i++ )
  {
    char *requested_attname = string_array_ref(attnames,i);
    char *actual_attname = ds_attnum_to_name(ds,i);
    if ( !caseless_eq_string(requested_attname,actual_attname) )
      my_errorf("Attribute (i.e. column) number %d (counting the "
                "leftmost column as number 1) should be named %s.\n"
		"But in file %s it is named %s.\n",
		i+1,requested_attname,filename,actual_attname);
  }

  for ( i = 0 ; i < ivec_size(real_atts) ; i++ )
  {
    int attnum = ivec_ref(real_atts,i);
    if ( !ds_real_ref_ok(ds,attnum) )
      my_errorf("I expected to load attribute %s from file %s as\n"
		"being numeric-valued. But I could not do so.\n",
		ds_attnum_to_name(ds,attnum),filename);
  }

  free_ivec(real_atts);
  free_string_array(sa);
  free_loadinst(ldi);
  free_string_array(attnames);

  return ds;
}

/* Loads a dym in which the caller forces, via the "assume" string,
   exactly what the attributes must be called within the CSV file.

   For example, 

    dym *d = mk_dym_from_file_assuming("plop.csv","height age");

   Loads a dym from the file plop.csv, but insists that there are
   exactly two attributes called height and age respectively. Note that
   each attribute corresponds to one column of the dym.
*/
dym *mk_dym_from_file_assuming(char *filename,char *assume)
{
  datset *ds = ds_load_assuming(filename,assume);
  int i;
  dym *x;

  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( !ds_real_ref_ok(ds,i) )
      my_errorf("When trying to load a matrix (DYM) from file %s\n"
		"I discovered that attribute %s is not interpretable\n"
		"as a real-valued attribute\n",filename,
		ds_attnum_to_name(ds,i));
  }

  x = mk_dym_from_datset(ds);

  free_datset(ds);

  return x;
}

/* Returns TRUE if and only if the string representation of the
   value of ds_symbol_ref(ds,row,attnum) is equal to name.

   Returns FALSE if value is missing.

   PRE: attnum must be symbol_ref_ok */
bool ds_row_attnum_has_name(datset *ds,int row,int attnum,char *name)
{
  bool result;

  if ( ds_value_is_missing(ds,row,attnum) )
    result = FALSE;
  else
  {
    int value = ds_symbol_ref(ds,row,attnum);
    char *valname = ds_attnum_value_to_name(ds,attnum,value);
    result = eq_string(valname,name);
  }

  return result;
}

/* Prints a human readable description of the given row of data */
void explain_datapoint(FILE *s,datset *ds,int row,char *id)
{
  int num_atts = ds_num_atts(ds);
  int attnum;
  int buflen = 2;
  fprintf(s,"\n*** %s ***\n  ",id);

  for ( attnum = 0 ; attnum < num_atts ; attnum++ )
  {
    char *attname = ds_attnum_to_name(ds,attnum);
    char *valname = mk_row_attnum_to_name(ds,row,attnum);
    char buff[1000];
    sprintf(buff,"%s=%s%s",attname,valname,(attnum==num_atts-1)?"":" ,");
    if ( buflen + (int)strlen(buff) > 78 )
    {
      buflen = 2;
      fprintf(s,"\n  ");
    }
    buflen += (int)strlen(buff) + 1;
    fprintf(s,"%s ",buff);
    free_string(valname);
  }
  fprintf(s,"\n");
}

void ds_remove_minor_coltypes(datset *ds)
{
  int attnum;
  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    if ( ds_real_ref_ok(ds,attnum) && ds_symbol_ref_ok(ds,attnum) )
      run_unsymbolize(ds,attnum);
  }
}

datset *mk_datset_from_args(char *key,int argc,char *argv[])
{
  char *fname = string_from_args(key,argc,argv,NULL);
  if ( fname == NULL )
    my_errorf("The keyword/value pair %s <filename> must appear on\n"
	      "the command line to specify a dataset to load.\n"
	      "But it seems to be missing.\n",key);
  return ds_load(fname);
}

datset *mk_datset_from_double_arr(double *data, int cols, int rows)
{
  datset* returner;
  shape* myshape;
  string_array* colnames;
  int c_itr;
  int i,j;
  
  colnames = mk_string_array(cols);

  for(c_itr=0;c_itr<cols;c_itr++)
    {
    char my_cname[30];

    sprintf(my_cname, "AUTO%d", c_itr);
    string_array_set(colnames, c_itr, my_cname);
    }
  fprintf_string_array(stdout, "", colnames, "\n");
  myshape = mk_shape_real(colnames);
  fprintf_shape(stdout, "", myshape, "\n");
  returner = mk_zero_datset(myshape, rows); 

  for (i = 0; i < cols; i++)
    {                                                  
    for (j = 0 ; j < rows; j++ )
      {
      ds_real_set(returner, j, i, *(data + (j*cols) + i) );
      }
    }
  free_string_array(colnames);
  free_shape(myshape);
  return(returner);
}

/* Forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
void datset_ensure_symbolic_nomissing(datset *ds,int arity)
{
  int i;

  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( !ds_symbol_ref_ok(ds,i) )
    {
      run_symbolize(ds,i,TRUE,FALSE,FALSE,FALSE,arity);
      run_unrealize(ds,i);
    }
    else if ( ds_all_symbolic_values_are_numbers(ds,i) &&
	      ds_attnum_to_arity(ds,i) >= 4 )
    {
      run_realize(ds,i);
      run_symbolize(ds,i,TRUE,FALSE,FALSE,FALSE,arity);
      run_unrealize(ds,i);
    }

    if ( ds_attnum_to_num_missing(ds,i) > 0 )
      convert_missing_to_symbol(ds,i);
  }
}

/* Loads a datset and forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
datset *mk_symbolic_datset_from_args(int argc,char *argv[])
{
  datset *ds = mk_datset_from_argc_argv(argc,argv);
  int arity = int_from_args("arity",argc,argv,2);
  datset_ensure_symbolic_nomissing(ds,arity);
  return ds;
}


/* ignore_attnames may be NULL, meaning the same as if it has size
   zero (i.e. ignore nothing) */
/* Loads a datset and forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
datset *mk_symbolic_datset_from_filename(char *filename,int arity,
					 string_array *ignore_attnames)
{
  datset *ds = ds_load(filename);

  if ( ignore_attnames != NULL )
  {
    int i;

    for ( i = 0 ; i < string_array_size(ignore_attnames) ; i++ )
    {
      char *attname = string_array_ref(ignore_attnames,i);
      int attnum = ds_name_to_attnum(ds,attname);
      if ( attnum < 0 )
	printf("You asked me to ignore attribute %s, but it's not in\n"
	       "the dataset anyway\n",attname);
      else
      {
	printf("You asked me to ignore attribute %s, so I hereby delete it.\n",
	       attname);
	ds_delete_attribute(ds,attnum);
      }
    }
  }

  datset_ensure_symbolic_nomissing(ds,arity);

  return ds;
}

ivec *mk_sorted_attnums_from_names_friendly(datset *ds,string_array *sa)
{
  string_array *safe_sa = mk_string_array(0);
  ivec *attnums;
  int i;
  char *errmess;

  for ( i = 0 ; i < string_array_size(sa) ; i++ )
  {
    char *name = string_array_ref(sa,i);
    int attnum = ds_name_to_attnum(ds,name);
    if ( attnum >= 0 )
      add_to_string_array(safe_sa,name);
  }
  attnums = mk_sorted_attnums_from_string_array(ds,safe_sa,&errmess);
  if ( errmess != NULL ) my_error(errmess);
  free_string_array(safe_sa);
  return attnums;
}

int ds_name_to_attnum_friendly(datset *ds,char *name)
{
  int attnum = ds_name_to_attnum(ds,name);
  if ( attnum < 0 )
    attnum = ds_num_atts(ds)-1;
  return attnum;
}

/* result[i] = string representation of contents of cell at location
    row = rows[i] and col = attnum
   
   rows == NULL means "use all rows"
*/
string_array *mk_row_to_value(datset *ds,ivec *rows,int attnum)
{
  int num_rows = (rows == NULL) ? ds_num_rows(ds) : ivec_size(rows);
  string_array *sa = mk_string_array(num_rows);
  int i;

  for ( i = 0 ; i < num_rows ; i++ )
  {
    int row = (rows == NULL) ? i : ivec_ref(rows,i);
    char *s = mk_row_attnum_to_name(ds,row,attnum);
    string_array_set(sa,i,s);
    free_string(s);
  }
  
  return sa;
}


