/* 
   File:        dscli.c
   Author:      Andrew W. Moore
   Created:     Mon Jun  2 19:58:44 EDT 1997
   Description: Utilities on datsets

   Copyright 1997, Carnegie Mellon University
*/

#include "dscli.h"
#include "dex.h"
#include "compat.h"
#include "join.h"
#include "freq.h"
#include "genarray.h"
#include "utils.h"

#ifdef PC_MVIS_PLATFORM
#include <direct.h>
#endif

/* Private prototypes */
static void select_attnums(datset *ds,ivec *attnums);
static void select_from_string_array(datset *ds,string_array *attnames);
static void reinitialize_row_to_marked(datset *ds,ivec **row_to_marked,int *num_marked);
static ivec *mk_unsorted_attnums_from_command_with_errmess(datset *ds,command *c, int start_arg,int end_arg, char **r_errmess);
static ivec *mk_unsorted_attnums_from_command(datset *ds,command *c, int start_arg,int end_arg);
static void dispatch_brief(void *env,command *c,bool act);
static void dispatch_info(void *env,command *c,bool act);
static void dispatch_correlate(void *env,command *c,bool act);
static void dispatch_symbolize(void *env,command *c,bool act);
static void dispatch_symb2real(void *env,command *c,bool act);
static void dispatch_realize(void *env,command *c,bool act);
static void dispatch_explode(void *env,command *c,bool act);
static void run_normalize(datset *ds,int attnum);
static void dispatch_normalize(void *env,command *c,bool act);
static void dispatch_symbolize_missing(void *env,command *c,bool act);
static void dispatch_unsymbolize(void *env,command *c,bool act);
static void dispatch_rename(void *env,command *c,bool act);
static void dispatch_set_missing(void *env,command *c,bool act);
static void dispatch_delete_value(void *env,command *c,bool act);
static void dispatch_rename_value(void *env,command *c,bool act);
static void dispatch_delete(void *env,command *c,bool act);
static void dispatch_copy(void *env,command *c,bool act);
static void dispatch_copyrows(void *env,command *c,bool act);
static void dispatch_new(void *env,command *c,bool act);
static void dispatch_new_value(void *env,command *c,bool act);
static void dispatch_define(void *env,command *c,bool act);
static void dispatch_select(void *env,command *c,bool act);
static void dispatch_swap(void *env,command *c,bool act);
void dispatch_save(void *env,command *c,bool act);
static void dispatch_load(void *env,command *c,bool act);
static void dispatch_loadmany(void *env,command *c,bool act);
static void dispatch_ls(void *env, command *c, bool act);
static void dispatch_unrealize(void *env,command *c,bool act);
static void dispatch_sortmissing(void *env,command *c,bool act);
static void dispatch_join(void *env,command *c,bool act);
static void dispatch_sortrows(void *env,command *c,bool act);
static void dispatch_sortatts(void *env,command *c,bool act);
static void dispatch_removerows(void *env,command *c,bool act);
static void dispatch_concatenate(void *env,command *c,bool act);
static void dispatch_batchrename(void *env,command *c,bool act);
static void dispatch_togglemarks(void *env,command *c,bool act);
static void dispatch_showmarked(void *env,command *c,bool act);
static void process_marked_rows(datset *ds,dex *dx,bool mark,bool submark, ivec *row_to_marked,int *num_marked);
static void dispatch_mark_or_unmark(void *env,command *c);
static void dispatch_mark(void *env,command *c,bool act);
static void dispatch_unmark(void *env,command *c,bool act);
static void dispatch_submark(void *env,command *c,bool act);
static void dispatch_shuffle(void *env,command *c,bool act);
static void dispatch_compat(void *env,command *c,bool act);
#ifdef OBSCURE
static void dispatch_binary_explosion(void *env,command *c,bool act);
#endif /* OBSCURE */
static void dispatch_frequent(void *env,command *c,bool act);
static void dispatch_transpose(void *env,command *c,bool act);
static void dispatch_shear(void *env,command *c,bool act);
static void dispatch_cartesian(void *env,command *c,bool act);
static void dispatch_nomissing(void *env,command *c,bool act);
static void editenv_command(void *env,command *c);
void dsenv_command(void *env,command *c);

/* Private headers for unused functions */
#ifdef AUTON_UNUSED_CODE
static bool attnums_from_command_ok(datset *ds,command *c, int start_arg,int end_arg);
static void run_explode(datset *ds,int attnum);
static void dispatch_downcase(void *env,command *c,bool act);
static int bwrite_dset(datset *d, char *file);
static void dispatch_edit(void *env,command *c,bool act);

#endif /* AUTON_UNUSED_CODE */

/************************ INPUTS AND OUTPUTS *************************/
void print_ins_and_out(datset *ds,ivec *in_attnums,int out_attnum,int max_atts_printed)
{
  int i;
  string_array *sa = mk_string_array(0);
  char *s;
  add_to_string_array(sa,"Inputs:");
  for ( i = 0 ; i < ivec_size(in_attnums) ; i++ ){
    s = mk_printf("%s",ds_attnum_to_name(ds,ivec_ref(in_attnums,i)));
    add_to_string_array(sa,s);
    free_string(s);
  }
  s = mk_printf("\nOutput: %s\n",(out_attnum<0)?"Undefined":ds_attnum_to_name(ds,out_attnum));
  add_to_string_array(sa,s);
  free_string(s);
  s = mk_string_from_string_array(sa);
  my_info(s);
  free_string(s);
  free_string_array(sa);  
}

static void select_attnums(datset *ds,ivec *attnums)
{
  int attnum;
  for ( attnum = ds_num_atts(ds)-1 ; attnum >= 0 ; attnum-- )
  {
    if ( !is_in_ivec(attnums,attnum) )
    {
      printf("Will delete attribute %s\n",ds_attnum_to_name(ds,attnum));
      ds_delete_attribute(ds,attnum);
    }
  }
  printf("Dataset now has %d attributes.\n",ds_num_atts(ds));
}

static void select_from_string_array(datset *ds,string_array *attnames)
{
  char *errmess = NULL;
  ivec *attnums = mk_unsorted_attnums_from_string_array(ds,attnames,&errmess);
  if ( attnums == NULL )
    printf("I won't do anything. Syntax Error:\n%s\n",errmess);
  else
    select_attnums(ds,attnums);
  if ( errmess != NULL ) free_string(errmess);
  if ( attnums != NULL ) free_ivec(attnums);
}

/* This function reads the string array to see if it can process it as
   one of the following commands:

inputs <attnames>
inputs all
ignore <attnames>
add <attnames>
output <attname>

  and it updates *in_attnums and/or out_attnum accordingly (maybe reallocating
  in_attnums.

  Note that if something is declared an output it is automatically removed from inputs.

  Note that "inputs all" sets the inputs to be all attributes EXCEPT the current output

  Note that if there's an illegal attname, this prints a message and does nothing.

  On exit returns TRUE if it changed something.

  Makes sure that in_attnums is sorted in ascending order.
  */   
bool command_line_changes_inputs_or_output(datset *ds,string_array *sa,ivec **in_attnums,
                                           int *out_attnum,bool realvalued_out_ok)
{
  bool result = FALSE;
  if ( string_array_size(sa) > 0 )
  {
    char *s = string_array_ref(sa,0);
    string_array *scop = mk_copy_string_array(sa);
    string_array_remove(scop,0);

    if (eq_string(s,"inputs") )
    {
      char *errmess;
      ivec *new_ins = mk_sorted_attnums_from_string_array(ds,scop,&errmess);
      if ( errmess != NULL )
      {
        printf(errmess);
        free_string(errmess);
      }
      else
      {
        free_ivec(*in_attnums);
        *in_attnums = new_ins;
        ivec_remove_value(*in_attnums,*out_attnum);

        result = TRUE;
      }
    }
    else if ( eq_string(s,"ignore") )
    {
      char *errmess;
      ivec *ignore = mk_sorted_attnums_from_string_array(ds,scop,&errmess);
      if (errmess)
      {
	printf("IGNORE COMMAND WARNING!!\n");
        printf(errmess);
        free_string(errmess);
	errmess = NULL;
      }

      if (ignore)
      {
        ivec *new_ins;
        ivec_sort(ignore,ignore);
        ivec_sort(*in_attnums,*in_attnums);
        new_ins = mk_ivec_diff_ordered(*in_attnums,ignore);
        free_ivec(*in_attnums);
        free_ivec(ignore);
        *in_attnums = new_ins;
        result = TRUE;
      }
    }
    else if ( eq_string(s,"add") )
    {
      char *errmess;
      ivec *add = mk_sorted_attnums_from_string_array(ds,scop,&errmess);
      if ( errmess != NULL )
      {
        printf(errmess);
        free_string(errmess);
      }
      else
      {
        ivec *new_ins = mk_ivec_union(*in_attnums,add);
        free_ivec(*in_attnums);
        free_ivec(add);
        *in_attnums = new_ins;
        ivec_remove_value(*in_attnums,*out_attnum);
        result = TRUE;
      }
    }
    else if ( eq_string(s,"output") )
    {
      if ( string_array_size(scop) != 1 )
        printf("You must simply type \"output <attribute>\"\n");
      else
      {
        int new_attnum = ds_name_to_attnum(ds,string_array_ref(scop,0));
        if ( new_attnum < 0 )
          printf("%s is not a legal attribute name\n",string_array_ref(scop,0));
        else
        {
          *out_attnum = new_attnum;
          ivec_remove_value(*in_attnums,*out_attnum);
          result = TRUE;
        }
      }
    }

    free_string_array(scop);
  }
  ivec_sort(*in_attnums,*in_attnums);
  return result;
}

#ifdef NEVER
int value_from_arg(datset *ds,int attnum,char *s)
{
  int value;
  int value_from_name = ds_attnum_name_to_value(ds,attnum,s);

  if ( !is_a_number(s) )
    value = value_from_name;
  else
  {
    int value_from_number = atoi(s);
    if ( value_from_name < 0 )
      value = value_from_number;
    else if ( value_from_number >= ds_attnum_to_arity(ds,attnum) )
      value = value_from_name;
    else if ( value_from_name == value_from_number )
      value = value_from_name;
    else
    {
      printf("I'm very sorry, but it is ambiguous whether you\n"
             "mean the value with index %d (which would be \"%s\")\n"
             "or the actual value \"%s\". I am going to assume the\n"
             "former.\n",value_from_number,
             ds_attnum_value_to_name(ds,attnum,value_from_number),
             ds_attnum_value_to_name(ds,attnum,value_from_name));
      value = value_from_number;
    }
  }             
  if ( value < 0 || value >= ds_attnum_to_arity(ds,attnum) )
    value = -1;
  return value;
}
#endif

bool check_attnum(datset *ds,char *s)
{
  bool result;
  if ( s == NULL )
  { 
    printf("You must supply an attribute.\n");
    result = FALSE;
  }
  else
  { 
    int attnum = ds_name_to_attnum(ds,s);
    result = attnum >= 0 && attnum < ds_num_atts(ds);
    if ( !result )
      printf("%s is not a legal attribute.\n",s);
  }
  return(result);
}

static void reinitialize_row_to_marked(datset *ds,ivec **row_to_marked,int *num_marked)
{
  if ( *row_to_marked != NULL )
    free_ivec(*row_to_marked);
  
  *row_to_marked = mk_zero_ivec(ds_num_rows(ds));

  *num_marked = 0;
}

void unmark_all(dsenv *de)
{
  reinitialize_row_to_marked(de->ds,&(de->row_to_marked),&(de->num_marked));
}

void mark_record(dsenv *de, int row)
{
  datset *ds = de->ds;

  if ((row > -1) && (row < ds_num_rows(ds)))
  {
    if (!de->row_to_marked) unmark_all(de);
    if (ivec_ref(de->row_to_marked, row) == 0) de->num_marked++;
    ivec_set(de->row_to_marked,row,1);
  }
  else
    my_error("mark_record: invalid row");
}

ivec *mk_ivec_marked_rows(dsenv *de) {
  int row;
  ivec *result = mk_ivec(0);

  for (row=0; row < ivec_size(de->row_to_marked); row++) {
    if(ivec_ref(de->row_to_marked, row) != 0) 
      add_to_ivec(result, row);
  }
  /*  fprintf_ivec(stdout, "marked rows: ", result, ""); */
  return result;
}

bool attnum_ok(datset *ds,command *c,int arg)
{
  bool result = TRUE;
  char *name = command_arg_ref(c,arg);
  if ( name == NULL )
  {
    result = FALSE;
    printf("I need an attribute as item number %d on the line.\n",arg);
  }
  else
  {
    int attnum = ds_name_to_attnum(ds,name);
    if ( attnum < 0 )
    {
      printf("Item number %d on the command should have been an attribute\n"
             "name, but I can't parse \"%s\" as either.\n",
             arg,name);
      result = FALSE;
    }
  }
  return result;
}

bool attnums_ok_from(datset *ds,command *c,int arg)
{
  bool ok = TRUE;
  string_array *attnames = mk_copy_string_array(c->args);
  int i;

  for ( i = 1 ; i < arg && string_array_size(attnames) > 0 ; i++ )
    string_array_remove(attnames,0);

  /*Added by Dan*/
  i = find_index_in_string_array(attnames,"done");
  if(i>=0) while(i<string_array_size(attnames)){
    string_array_remove_last_element(attnames); 
  } 

  if ( string_array_size(attnames) == 0 )
  {
    ok = FALSE;
    printf("From item number %d on the line onwards, you must supply\n"
           "one or more attribute names or numbers, or\ninstead you can\n"
	   "specify one of: all symbolic real\n",arg);
  }
  else
  {
    char *errmess;
    c -> attnums = mk_sorted_attnums_from_string_array(ds,attnames,&errmess);
    if (!c->attnums) ok = FALSE;
    if (errmess)
    {
      printf("ERROR: %s\n",errmess);
      free_string(errmess);
    }
  }
  free_string_array(attnames);
  return ok;
}

bool attnums_ok(datset *ds,command *c)
{
  return attnums_ok_from(ds,c,1);
}

/* end_arg == -1 denotes "use all args from argnum == start_arg on".
   Remember that arg==1 corresponds to the first token the user typed
   AFTER the actual command name */
static ivec *mk_unsorted_attnums_from_command_with_errmess(datset *ds,command *c,
						    int start_arg,int end_arg,
						    char **r_errmess)
{
  int real_end = (end_arg < 0) ? command_num_tokens(c) : end_arg;
  ivec *attnums = NULL;
  *r_errmess = NULL;
  if ( real_end < end_arg )
    *r_errmess = mk_printf("I was expecting attribute names as items %d\n"
			   "through %d on the command line. But to my\n"
			   "horror, I discover you have only supplied items\n"
			   "1 though %d.\n",
			   start_arg,end_arg,command_num_tokens(c)-1);        
  else
  {
    char *errmess;
    string_array *attnames = 
      mk_string_array_segment(c->line,start_arg,end_arg);
    attnums = mk_unsorted_attnums_from_string_array(ds,attnames,&errmess);
    if ( attnums == NULL )
      my_error(errmess);
    free_string_array(attnames);
  }
  return attnums;
}
  
#ifdef AUTON_UNUSED_CODE
/* end_arg == -1 denotes "use all args from argnum == start_arg on".
   Remember that arg==1 corresponds to the first token the user typed
   AFTER the actual command name */
static bool attnums_from_command_ok(datset *ds,command *c,
			     int start_arg,int end_arg)
{
  char *errmess;
  ivec *attnums = 
    mk_unsorted_attnums_from_command_with_errmess(ds,c,start_arg,end_arg,
						  &errmess);
  bool ok = attnums != NULL;
  if ( !ok )
    printf("*** %s\n",errmess);
  if ( errmess != NULL ) free_string(errmess);
  if ( attnums != NULL ) free_ivec(attnums);
  return ok;
}
#endif /* AUTON_UNUSED_CODE */

/* end_arg == -1 denotes "use all args from argnum == start_arg on".
   Remember that arg==1 corresponds to the first token the user typed
   AFTER the actual command name */
static ivec *mk_unsorted_attnums_from_command(datset *ds,command *c,
				       int start_arg,int end_arg)
{
  char *errmess;
  ivec *attnums = 
    mk_unsorted_attnums_from_command_with_errmess(ds,c,start_arg,end_arg,
						  &errmess);
  if ( errmess != NULL )
    my_error(errmess);

  return attnums;
}
  
dsenv *mk_dsenv_incfree_datset(datset *ds)
{
  dsenv *de = AM_MALLOC(dsenv);
  de -> ds = ds;
  de -> row_to_marked = mk_zero_ivec(ds_num_rows(ds));
  de -> num_marked = 0;
  de -> ds_changed = FALSE;
  de -> acted = FALSE;
  return de;
}

datset *mk_datset_free_dsenv(dsenv *de)
{
  datset *ds = de->ds;
  free_ivec(de->row_to_marked);
  AM_FREE(de,dsenv);
  return ds;
}

string_array *mk_datset_attnames(datset *ds,bool all,bool symbols){
  int i;
  bool ok = TRUE;
  string_array *ret = mk_string_array(0);
  for(i=0;i<ds_num_atts(ds);i++) {
    if(!all){
      if(symbols) ok = ds_symbol_ref_ok(ds,i);
      else ok = ds_real_ref_ok(ds,i);
    }
    if(ok) add_to_string_array(ret,ds_attnum_to_name(ds,i));
  }
  return ret;
}

string_array *mk_datset_attvals(datset *ds,int attnum){
  int i;
  string_array *ret = mk_string_array(0);
  for(i=0;i<ds_attnum_to_arity(ds,attnum);i++)
    add_to_string_array(ret,ds_attnum_value_to_name(ds,attnum,i));
  return ret;
}

static void dispatch_brief(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("brief: Give a brief listing of details of each attribute.\n"
"  You are simply told the names and whether the attributes are symbolic\n"
"  or real. Each attribute is described on one line. For more details on\n"
"  attributes, use the info command.\n");

  if ( act )
    explain_datset(de->ds);
}

static void dispatch_info(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("info <attributes> : Give detailed attribute information.\n"
                "  <attributes> may be \"all\" or a list of attribute names or numbers\n"
"  For the symbolic attributes, you are told what the symbolic values\n"
"  are, and how many times each value occurs in the dataset. For the real-valued\n"
"  attributes, you are told their means, variances, mins and maxes. In \n"
"  each case you are also shown the first eight values. You are also told\n"
"  how many values are missing.\n");

  if ( act )
  {
    int i;
    string_array *params = mk_datset_attnames(de->ds,1,1);
    void *arg = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(arg && attnums_ok(de->ds,c)) for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
    {
      int attnum = ivec_ref(c->attnums,i);
      explain_attribute_detailed(stdout,de->ds,attnum);
    }
    free_arg(arg,ARG_STRING_SET);
  }
}

static void dispatch_correlate(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("correlate <att1> <att2> : Find the correlation between two real-valued attributes");

  if ( act && attnum_ok(de->ds,c,1) && attnum_ok(de->ds,c,2) )
  {
	datset *ds = de->ds;
	int attnum1 = ds_name_to_attnum(ds,c->a1);
	int attnum2 = ds_name_to_attnum(ds,c->a2);
	if ( !ds_real_ref_ok(ds,attnum1) )
	  printf("%s is not real-valued.\n",c->a1);
	else if ( !ds_real_ref_ok(ds,attnum2) )
	  printf("%s is not real-valued.\n",c->a2);
    else
	{
	  double fve = fraction_variance_explained(ds,attnum1,attnum2);
	  printf("The fraction variance explained when using %s to predict %s\n"
		     "using linear regression is %g\n",
			 ds_attnum_to_name(ds,attnum1),
			 ds_attnum_to_name(ds,attnum2),
			 fve);
	}
  }
}

#ifdef NEVER 
void dispatch_nearest(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  set_help("nearest <inattributes> <outattribute>: Do inputs predict output well?\n"
	            "Example:\n"
				"  nearest height weight age\n"
				"The above command would compute, via leave-one-out, 1-nearest-neighbor\n"
				"cross-validation, how well height and weight predict age.\n");

  if ( act && attnums_ok(de->ds,c) )
  {
	  ivec *in_attnums = mk_copy_ivec(c->attnums);
	  int out_attnum = ivec_ref(c->attnums,ivec_size(c->attnums)-1);
	  int i;
    double max_error = (double)ds_num_rows(ds);
	  ivec_remove_last_element(in_attnums);
	
    printf("How well do the following attributes...\n  ");
	  for ( i = 0 ; i < ivec_size(in_attnums) ; i++ )
    {
      int attnum = ivec_ref(in_attnums,i);
      printf("%s ",ds_attnum_to_name(ds,attnum));
    }
    printf("\n..predict the value of %s?\n",ds_attnum_to_name(ds,out_attnum));

    printf("The leave one out 1-nearest neighbor error is...\n");
    printf("%g\n",100*(1-loo_nearest_error(ds,in_attnums,out_attnum,max_error)
                  /real_square(ds_att_sdev(ds,out_attnum))));
    free_ivec(in_attnums);
  }
}
#endif

static void dispatch_symbolize(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;

  set_help("symbolize <style> [<num_levels>] <attributes> : make numeric atts symbolic.\n"
"<style> must be equal, uniform, manual, or integer. \n"
"\n"
"If <style> is equal,   unform or manual, <num_levels> MUST be supplied\n"
"If <style> is integer, <num_levels> MUST NOT be supplied.\n"
	   "\n");

  add_to_help("    equal      breaks up the attribute into buckets of equal width. \n"
"\n"
"    uniform    can create buckets of non-equal width, but makes sure that\n"
"               the number of records in the various buckets are not wildly \n"
"               skewed.\n"
"\n"
"    manual     reads in numbers from the user on where to put the breaks.\n"
"\n"
	      "    integer    rounds all the real numbers to their nearest integer value\n");

  add_to_help("               and makes one symbolic value per integer.\n"
"\n"
" <num_levels> must be a number between 2 up to 100\n"
" <attributes> may be \"all\" or a list of attribute names or numbers\n"
"\n"
"This takes every numeric attribute in the list and breaks\n"
"it up into bucketed symbols. How many buckets? That's\n"
"defined by <num_levels>. The buckets are equal sized, with\n"
	      "boundaries at round numbers. The bucket corresponding to the\n");

  add_to_help("lowest value has the name v0:XXX- where XXX is the boundary\n"
"between the smallest and second smallest bucket. \n"
"The integer style simply creates the name to be the integer value.\n");

  if ( act )
  {
    bool uniform=FALSE,equal=FALSE,manual=FALSE,integer=FALSE,ok=TRUE;
    int i,atts_from = 2,*num_levels=NULL;
    char *style;
    string_array *attribs=NULL,*params;
    datset *ds = de->ds;

    params = mk_string_array_4("uniform","equal","manual","integer");
    style = mk_arg_from_command(c,1,"the style",ARG_STRING,0,0,0,params,1);
    if(style){
      uniform = eq_string(c->a1,"uniform");
      equal = eq_string(c->a1,"equal");
      manual = eq_string(c->a1,"manual");
      integer = eq_string(c->a1,"integer");
      if(!integer){
        num_levels = mk_arg_from_command(c,2,"the number of levels",ARG_INT,2,100,3,NULL,1);
        atts_from = 3;
        if(!num_levels) ok = FALSE;
      } else {
        num_levels = AM_MALLOC(int);
        *num_levels = 0;
      }
    } else ok = FALSE;
    free_string_array(params);
    if(ok){
      params = mk_string_array(0);
      for(i=0;i<ds_num_atts(ds);i++)
        add_to_string_array(params,ds_attnum_to_name(ds,i));

      attribs = mk_arg_from_command(c,atts_from,"attributes",ARG_STRING_SET,0,0,0,params,0);
      free_string_array(params);
      if(!attribs) ok = FALSE;
      else ok = attnums_ok_from(de->ds,c,atts_from);
    }

    if(ok)
    {
      int start_secs = global_time();
      for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
      {
        int attnum = ivec_ref(c->attnums,i);
        bool changed = run_symbolize(ds,attnum,uniform,equal,
                                     manual,integer,*num_levels);
        if ( changed ) de->ds_changed = TRUE;
      }
      printf("That took %d sec(s)\n",global_time()-start_secs);
      free_string_array(attribs);
    }
    free_arg(num_levels,ARG_INT);
    free_arg(style,ARG_STRING);
  }
}

static void dispatch_symb2real(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("symb2real <attributes> : Replace symbols with numbers.\n"
"\n"
"   This function will only change an attribute if it is symbolic and\n"
"   has no cached real-values.\n"
"\n"
"   Then it simply sets the ith real value to be a small integer\n"
"   representation of the value. For example if you had a dset with\n"
	   "   one attribute called fruit and it originally was\n");

  add_to_help("\n"
"   fruit\n"
"   -----\n"
"   apple\n"
"   orange\n"
"   apple\n"
"   banana\n"
"   banana\n"
	      "\n");

  add_to_help("   then set of values for the fruit attribute is { apple , banana , orange }\n"
"   and the small-integer value of each attribute is simply 0, 1 and 2\n"
"   respectively (note how the values are sorted alphabetically).\n"
"\n"
"   The result dataset column is:\n"
"\n"
"   fruit\n"
"   -----\n"
"   0\n"
"   2\n"
"   0\n"
"   1\n"
"   1\n"
"\n");

  if ( act )
  {
    int i;
    bool ok;
    string_array *params = mk_string_array(0),*attribs;
    datset *ds = de->ds;
    for(i=0;i<ds_num_atts(ds);i++) if(!ds_real_ref_ok(ds,i)) 
    {
      add_to_string_array(params,ds_attnum_to_name(ds,i));
    }
    attribs = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,
				  params,0);
    free_string_array(params);
    if(!attribs) ok = FALSE;
    else ok = attnums_ok_from(de->ds,c,1);

    if(ok) for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
    {
      int attnum = ivec_ref(c->attnums,i);
      bool changed = run_symb2real(ds,attnum);

      if ( changed )
	de -> ds_changed = TRUE;
    }
    free_arg(attribs,ARG_STRING_LIST);
  }
}

static void dispatch_realize(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("realize <attributes> : Convert certain symbolic attributes to real.\n"
                "  The only attributes this will affect are ones that are\n"
                "  initially symbolic, but all their symbolic values can\n"
                "  be parsed (interpreted) as numbers. So a symbolic attribute \n"
"  with values Andrew, Fred, 3, and 7 would be unchanged. But a symbolic attribute with\n"
	   "  values 0 1 92 17 would be changed into a real attribute in which, for\n");

  add_to_help("  example, those records with value==symbolic 92 previously would now\n"
"  have the real value 92.000.\n"
"\n"
"  You may wonder why the dataset reader does not simply automatically convert\n"
	      "  attributes into real attributes if all the symbolic value names can be\n");

  add_to_help("  parsed as numbers. The reason is that if there are only a small number\n"
"  of distinct numbers, the memory savings of representing as symbols is\n"
"  colossal. To force the dataset loader to automatically convert such\n"
"  attributes as reals, include default_real true on the command line.\n");

  if ( act )
  {
    int i;
    bool ok;
    string_array *params = mk_string_array(0),*attribs;
    datset *ds = de->ds;
    for(i=0;i<ds_num_atts(ds);i++) if(!ds_real_ref_ok(ds,i)) {
      add_to_string_array(params,ds_attnum_to_name(ds,i));
    }
    attribs = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(!attribs) ok = FALSE;
    else ok = attnums_ok_from(de->ds,c,1);

    if(ok) for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
    {
      int attnum = ivec_ref(c->attnums,i);
      bool quietMode = (Verbosity <0);
      (void) run_realize_maybe_silent(ds,attnum, quietMode);
      /* Note we don't count this as changing the dataset */
    }
    free_arg(attribs,ARG_STRING_LIST);
  }
}

static void run_explode(datset *ds,int attnum)
{
  if ( !ds_symbol_ref_ok(ds,attnum) )
  {
    printf("I won't explode %s because it's not symbolic\n",
	   ds_attnum_to_name(ds,attnum));
  }
  else
  {
    if ( ds_real_ref_ok(ds,attnum) )
      run_unrealize(ds,attnum);

    ds_symbolic_binary_explosion(ds,attnum);
  }
}

static void dispatch_explode(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("explode <attributes> : Convert certain symbolic attributes to 1-of-N binary encoding.\n");

  if ( act )
  {
    int i;
    bool ok;
    string_array *params = mk_string_array(0),*attribs;
    datset *ds = de->ds;
    for(i=0;i<ds_num_atts(ds);i++) if(!ds_real_ref_ok(ds,i)) {
      add_to_string_array(params,ds_attnum_to_name(ds,i));
    }
    attribs = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(!attribs) ok = FALSE;
    else ok = attnums_ok_from(de->ds,c,1);

    if(ok) for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
    {
      int attnum = ivec_ref(c->attnums,i);
      printf("Will attempt to explode %s\n",ds_attnum_to_name(ds,attnum));
      run_explode(ds,attnum);
      de->ds_changed = TRUE;
    }
    free_arg(attribs,ARG_STRING_LIST);
  }
}

static void run_normalize(datset *ds,int attnum)
{
  if ( !ds_real_ref_ok(ds,attnum) )
  {
    printf("I won't normalize %s because it's not real-valued\n",
	   ds_attnum_to_name(ds,attnum));
  }
  else
  {
    double mean = ds_att_mean(ds,attnum);
    double sdev = real_max(1e-6,ds_att_sdev(ds,attnum));
    int row;

    if ( ds_symbol_ref_ok(ds,attnum) )
      run_unsymbolize(ds,attnum);

    printf("Am subtracting %g from each element of %s, then dividing by %g\n",
	   mean,ds_attnum_to_name(ds,attnum),sdev);

    for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
    {
      if ( !ds_value_is_missing(ds,row,attnum) )
      {
	double vold = ds_real_ref(ds,row,attnum);
	double vnew = (vold - mean) / sdev;
	ds_real_set(ds,row,attnum,vnew);
      }
    }
  }
}

static void dispatch_normalize(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("normalize <attributes> : Convert certain symbolic attributes to 1-of-N binary encoding.\n");

  if ( act )
  {
    int i;
    bool ok;
    string_array *params = mk_string_array(0),*attribs;
    datset *ds = de->ds;
    for(i=0;i<ds_num_atts(ds);i++) if(!ds_real_ref_ok(ds,i)) {
      add_to_string_array(params,ds_attnum_to_name(ds,i));
    }
    attribs = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(!attribs) ok = FALSE;
    else ok = attnums_ok_from(de->ds,c,1);

    if(ok) for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
    {
      int attnum = ivec_ref(c->attnums,i);
      (void) run_normalize(ds,attnum);
      de->ds_changed = TRUE;
    }
    free_arg(attribs,ARG_STRING_LIST);
  }
}

static void dispatch_symbolize_missing(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("symbolize_missing <attributes> : make missing values a new symbol.\n"
"This command only works on symbolic attributes.\n"
"In order to use it on a real column, you would have to first symbolize that column.\n");

  if ( act )
  {
    int i;
    string_array *params = mk_string_array(0);
    void *args = NULL;
    for(i=0;i<ds_num_atts(de->ds);i++){
      if(ds_symbol_ref_ok(de->ds,i) && ds_attnum_to_num_missing(de->ds,i)>0)
        add_to_string_array(params,ds_attnum_to_name(de->ds,i));
    }
    args = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    free_arg(args,ARG_STRING_SET);

    if(args && attnums_ok(de->ds,c))
    {
      for ( i = 0 ; i < ivec_size(c->attnums) ; i++ )
      {
        int attnum = ivec_ref(c->attnums,i);

        if (!ds_symbol_ref_ok(de->ds,attnum))
	        printf("%s unchanged -- must be pure symbolic to add missing as a symbol\n",
	               ds_attnum_to_name(de->ds,attnum));
        else if (ds_attnum_to_num_missing(de->ds,attnum) == 0)
	         printf("%s unchanged -- it has no missing values.\n",
	                ds_attnum_to_name(de->ds,attnum));
        else if (ds_real_ref_ok(de->ds,attnum))
        {
          bool *cont;
          if(find_index_in_string_array(c->args,"continue")==-1)
	  {
	    char buff[256];

	    sprintf(buff,"Attribute %s has real values and missing values!\n"
                         "The real values will be deleted if you continue.\n",
		    ds_attnum_to_name(de->ds,attnum));
            my_warning(buff);
	  }
          cont = mk_arg_from_command(c,-1,"continue",ARG_BOOL,0,0,0,NULL,TRUE);
          if(cont) convert_missing_to_symbol(de->ds,attnum);
          free_arg(cont,ARG_BOOL);
          if(!attnums_ok(de->ds,c)) my_error("My attnums are all bunked up");
        }
        else {          
    	    convert_missing_to_symbol(de->ds,attnum);
        }
      }
    }
  }
}

static void dispatch_unsymbolize(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  set_help("unsymbolize <attributes> : unsymbolize attributes.\n"
                "  After executing this command, the attributes listed\n"
                "  may no longer be referred to as symbolic attributes\n");

  if ( act )
    {
    int i;
    string_array *params = mk_string_array(0);
    void *args = NULL;
    for(i=0;i<ds_num_atts(de->ds);i++)
      {
      if(ds_symbol_ref_ok(de->ds,i) && ds_attnum_to_num_missing(de->ds,i)>0)
        add_to_string_array(params,ds_attnum_to_name(de->ds,i));
      }
    args = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    free_arg(args,ARG_STRING_SET);

    if(args && attnums_ok(de->ds,c))
      {
      int i2;
      for ( i2 = 0 ; i2 < ivec_size(c->attnums) ; i2++ )
        {
        int attnum = ivec_ref(c->attnums,i2);

        if (!ds_symbol_ref_ok(de->ds,attnum))
	  printf("%s unchanged -- must be symbolic to unsymbolize\n",
		 ds_attnum_to_name(ds,attnum));
	else
          (void) run_unsymbolize(ds,attnum);
        }
      }
    }
}

static void dispatch_unrealize(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("unrealize <attributes> : unrealize attributes.\n"
                "  After executing this command, the attributes listed\n"
                "  may no longer be referred to as real attributes\n");

  if ( act )
  {
    int i;
    string_array *params = mk_string_array(0);
    void *args = NULL;
    for(i=0;i<ds_num_atts(de->ds);i++){
      if(ds_symbol_ref_ok(de->ds,i) && ds_attnum_to_num_missing(de->ds,i)>0)
        add_to_string_array(params,ds_attnum_to_name(de->ds,i));
    }
    args = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    free_arg(args,ARG_STRING_SET);

    if(args && attnums_ok(de->ds,c))
    {
      int i2;
      for ( i2 = 0 ; i2 < ivec_size(c->attnums) ; i2++ )
      {
        int attnum = ivec_ref(c->attnums,i2);

        if (!ds_real_ref_ok(de->ds,attnum))
	  printf("%s unchanged -- must be real-valued to unrealize\n",
		 ds_attnum_to_name(ds,attnum));
	else
          (void) run_unrealize(ds,attnum);
      }
    }
  }
}


static void dispatch_rename(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("rename <attribute> <newname> : Rename.\n"
"  Simply changes the name of the given attribute. The new name must not\n"
"  duplicate another name in the dataset, nor may it be a number.\n");

  if ( act )
  {
    string_array *params = mk_datset_attnames(de->ds,TRUE,FALSE);
    char *newname=NULL;
    char *attname = mk_arg_from_command(c,1,"attribute",ARG_STRING,0,0,0,params,1);
    if(attname) newname = mk_arg_from_command(c,2,"new name",ARG_STRING,0,0,0,NULL,1);
    free_string_array(params);
    if ( newname == NULL || ds_name_to_attnum(de->ds,newname) >= 0 || is_a_number(newname) )
      my_warning("You must supply a new name that's different from\n"
                  "any current name and isn't a number.\n");
    else
    {
      int attnum = ds_name_to_attnum(de->ds,attname);
      ds_rename_attribute(de->ds,attnum,newname);
    }
    free_string(attname);
    free_string(newname);
  }
}

static void dispatch_delete_value(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;

  set_help("delete_value <attribute> <value_name> : delete value.\n"
"<attribute> must be a symbolic attribute. This removes the printed\n"
"name of one of its symbolic values. All records with the named value\n"
"revert to having a missing value for that attribute.\n");

  if ( act )
  {
    char *attname = NULL,*value_name=NULL;
    int i,attnum=-1;
    string_array *params = mk_string_array(0);
    for(i=0;i<ds_num_atts(de->ds);i++) if(!ds_real_ref_ok(de->ds,i))
      add_to_string_array(params,ds_attnum_to_name(de->ds,i));
    attname = mk_arg_from_command(c,1,"attribute",ARG_STRING,0,0,0,params,1);
    free_string_array(params);
    if(attname){
      attnum = ds_name_to_attnum(de->ds,attname);
      if ( ds_real_ref_ok(de->ds,attnum) ) {
        my_warning("Can only delete values for purely symbolic attributes\n"); 
        free_arg(attname,ARG_STRING);
        attname = NULL;
      }
    }
    if(attname){
      params = mk_datset_attvals(de->ds,attnum);
      value_name = mk_arg_from_command(c,2,"old value name",ARG_STRING,0,0,0,params,1);
      free_string_array(params);
    }
    if(value_name)
    {     
      int value = ds_attnum_name_to_value(de->ds,attnum,value_name);
  
      if ( value < 0 ){
        char *buff=mk_printf("Attribute %s has no value called %s.",attname,value_name);
        my_warning(buff);
        free_string(buff);
      }
      else
        ds_delete_value(de->ds,attnum,value);
    }
    free_arg(attname,ARG_STRING);
    free_arg(value_name,ARG_STRING);
  }
}

static void dispatch_rename_value(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;

  set_help("rename_value <attribute> <oldvalname> <newname> : Rename value.\n"
"<attribute> must be a symbolic attribute. This changes the printed\n"
"name of one of its symbolic values. <oldvalname> must be one of\n"
"<attribute>'s values, and <newname> is the name it's given.\n");

  if ( act )
  {
    char *attname = NULL,*oldname=NULL,*newname=NULL;
    int i,attnum=-1;
    string_array *params = mk_string_array(0);
    for(i=0;i<ds_num_atts(de->ds);i++) if(ds_symbol_ref_ok(de->ds,i))
      add_to_string_array(params,ds_attnum_to_name(de->ds,i));

    if (command_num_tokens(c) > 1)
      attname = mk_copy_string(command_arg_ref(c,1));
    else
      attname = mk_arg_from_command(c,1,"attribute",ARG_STRING,0,0,0,params,1);

    free_string_array(params);
    if(attname){
      attnum = ds_name_to_attnum(de->ds,attname);
      if ( ds_real_ref_ok(de->ds,attnum) ) {
        my_warning("Can only rename values for purely symbolic attributes\n"); 
        free_arg(attname,ARG_STRING);
        attname = NULL;
      }
    }
    if(attname)
    {
      params = mk_datset_attvals(de->ds,attnum);

      if (command_num_tokens(c) > 2)
	oldname = mk_copy_string(command_arg_ref(c,2));
      else
	oldname = mk_arg_from_command(c,2,"old value name",ARG_STRING,0,0,0,
				      params,1);
      free_string_array(params);
      if(oldname) 
      {
	if (command_num_tokens(c) > 3)
	  newname = mk_copy_string(command_arg_ref(c,3));
	else
	  newname = mk_arg_from_command(c,3,"new value name",ARG_STRING,0,0,0,
					NULL,1);
      }
    }
    if(newname)
    {     
      int value = ds_attnum_name_to_value(de->ds,attnum,oldname);
  
      if ( value < 0 ){
        char *buff=mk_printf("Attribute %s has no value called %s.",attname,oldname);
        my_warning_no_wait(buff);
        free_string(buff);
      }
      else {
        int newvalue = ds_attnum_name_to_value(de->ds,attnum,newname);

        if ( c->a3 == NULL || newvalue >= 0 ) {
          char *buff = mk_printf("%s already exists!  This will cause %s to merge with "
            "%s.  %s will be deleted.  Continue anyway?",newname,oldname,newname,oldname);
          bool *cont = mk_arg_from_command(c,4,buff,ARG_BOOL,0,0,0,NULL,1);
          if(cont && *cont) ds_merge_value(de->ds,attnum,value,newvalue);
          free_arg(cont,ARG_BOOL);
          free_string(buff);
        }
        else
          ds_rename_value(de->ds,attnum,value,newname);
      }
    }
    free_arg(attname,ARG_STRING);
    free_arg(oldname,ARG_STRING);
    free_arg(newname,ARG_STRING);
  }
}

static void dispatch_set_missing(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;

  set_help("set_missing <attribute> <value> : Undefine values.\n"
"This changes the value of\n"
"<attribute> to be missing in all cases where it currently has <value>.\n"
"If no <value> is given, <attribute> will become undefined in all records.\n");

  if ( act )
  {
    char *attname = NULL,*oldname=NULL;
    int attnum=-1;
    datset *ds = de->ds;
    int row;

    if (command_num_tokens(c) > 1) 
      attname = mk_copy_string(command_arg_ref(c,1));
    else
      attname = NULL;

    if(attname) attnum = ds_name_to_attnum(de->ds,attname);

    if(attnum < 0)
    {
      printf("Attname %s not in datset.\n",(attname)?attname:"NULL");
    }
    else
    {
      int value = -1;
      double realvalue = -888e-888;
      bool checksym = FALSE;
      bool checkreal = FALSE;
      bool doall = FALSE;
      bool parseok = TRUE;

      if (command_num_tokens(c) > 2)
      {
	oldname = mk_copy_string(command_arg_ref(c,2));
	if (ds_symbol_ref_ok(ds,attnum))
	{
	  value = ds_attnum_name_to_value(de->ds,attnum,oldname);
	  if (value >= 0) checksym = TRUE;
	  else printf("Attribute %s has no value called %s.",attname,oldname);
	}
	if (ds_real_ref_ok(ds,attnum))
	{
	  realvalue = double_from_string(oldname,&parseok);
	  if (parseok) checkreal = TRUE;
	}
      }
      else
      {
	doall = TRUE;
      }
  
      for (row=0;row<ds_num_rows(ds);row++)
	if (doall || 
	    (checksym && (ds_symbol_ref(ds,row,attnum) == value)) ||
	    (checkreal && (ds_real_ref(ds,row,attnum) == realvalue)))
	  ds_set_value_missing(ds,row,attnum);
    }
    if (attname) free_string(attname);
    if (oldname) free_string(oldname);
  }
}

static void dispatch_delete(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("delete <attributes> : Remove one or more attributes.\n"
"  Gets rid of the given attribute (and, of course, all its values)\n"
"  from the dataset. There's no undo!\n");
  if ( act )
  {
    int i;
    void *arg;
    string_array *params = mk_datset_attnames(de->ds,TRUE,TRUE);
    arg = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(arg && attnums_ok(de->ds,c))
    {
      for ( i = ivec_size(c->attnums) - 1 ; i >= 0 ; i-- )
      {
        int attnum = ivec_ref(c->attnums,i);
        printf("Will delete %s (attnum %d)\n",ds_attnum_to_name(de->ds,attnum),
               attnum);
        ds_delete_attribute(de->ds,attnum);
        de->ds_changed = TRUE;
      }
    }
    free_arg(arg,ARG_STRING_SET);
  }
}

static void dispatch_copy(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  set_help("copy <attname> <newattname> : Make a copy of an attribute\n");

  if ( act )
  {
    string_array *params = mk_datset_attnames(ds,1,0);
    char *oldname = NULL, *newname = NULL;
    oldname = mk_arg_from_command(c,1,"attribute to copy",ARG_STRING,0,0,0,params,TRUE);
    if(oldname) newname = mk_arg_from_command(c,2,"name of new attribute",ARG_STRING,0,0,0,NULL,FALSE);
    if(newname)
    {
      if(ds_name_to_attnum(ds,newname)>=0){
        char *buff = mk_printf("%s already exists!",newname);
        my_warning(buff);
        free_string(buff);
      }
      else
        ds_copy_attribute(ds,oldname,newname);
    }
    free_arg(newname,ARG_STRING);
    free_arg(oldname,ARG_STRING);
    if (params) free_string_array(params);
  }
}

static void dispatch_copyrows(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  set_help("copyrows [marked] : Makes a copy of all rows, or just the marked rows if requested\n");

  if ( act )
  {
    bool marked = arg_key_exists(c, "marked");
    ivec *src_rows;
    ivec *dst_rows;
    int n = ds_num_rows(ds);

    if (marked) src_rows = mk_ivec_marked_rows(de);
    else        src_rows = mk_identity_ivec(n);
    dst_rows = mk_identity_ivec(ivec_size(src_rows));
    ivec_scalar_plus(dst_rows,n,dst_rows);
    ds_add_rows(ds,ivec_size(src_rows));
    ds_copy_rows(ds,src_rows,ds,dst_rows);
    unmark_all(de);
    free_ivec(src_rows);
    free_ivec(dst_rows);
  }
}

static void dispatch_new(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("new <attname> [real|symbol] [symbolic values] : Make new attribute\n");

  if ( act )
  {
    datset *ds = de->ds;
    char *type=NULL,*attname = mk_arg_from_command(c,1,"attribute name",ARG_STRING,0,0,0,NULL,1);
    void *vals=NULL;
    bool realp=FALSE,symbolp=FALSE,ok=FALSE;
    if(attname){
      string_array *params = mk_string_array_2("real","symbol");
      type = mk_arg_from_command(c,2,"attribute type",ARG_STRING,0,0,0,params,1);
      free_string_array(params);
      if(type){
        realp = eq_string(type,"real");
        symbolp = eq_string(type,"symbol");
        if(symbolp) {
          vals = mk_arg_from_command(c,3,"attribute values",ARG_STRING_LIST,0,0,0,NULL,1);
          ok = (vals!=NULL);
        } else ok = TRUE;
      }
    }
    if(ok)
    {
      if ( ds_name_to_attnum(ds,attname) >= 0 )
        printf("Cannot make an attribute with an already-existing name\n");
      else if ( !realp && !symbolp )
        printf("Must specify whether real or symbolic\n");
      else if ( realp && command_num_tokens(c) > 3 )
        printf("Real attributes do not need to be supplied with values\n");
      else if ( symbolp && command_num_tokens(c) < 4 )
        printf("Symbolic attributes must be supplied with at least 1 value\n");
      else if ( realp )
      {
        ds_add_real_attribute(ds,attname);
        de->ds_changed = TRUE;
      }
      else
      {
        string_array *valnames = mk_string_array(0);
        int j;

        for ( j = 3 ; j < command_num_tokens(c) ; j++ )
          maybe_insert_in_sorted_string_array(valnames,command_arg_ref(c,j));
        ds_add_symbolic_attribute(ds,attname,valnames);
        free_string_array(valnames);
        de->ds_changed = TRUE;
      }
    }
    free_arg(attname,ARG_STRING);
    free_arg(type,ARG_STRING);
    free_arg(vals,ARG_STRING_LIST);
  }
}

static void dispatch_new_value(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("new_value <attname> [symbolic values]: Add symbolic values to attribute\n");

  if ( act )
  {
    datset *ds = de->ds;
    string_array *attvals = NULL;
    string_array *params = mk_datset_attnames(ds,FALSE,TRUE);
    char *attname = mk_arg_from_command(c,1,NULL,ARG_STRING,0,0,0,params,1);
    free_string_array(params);
    if(attname) attvals = mk_arg_from_command(c,2,NULL,ARG_STRING_LIST,0,0,0,NULL,1);
    if(attvals)
    {
      int i, attnum = ds_name_to_attnum(ds,attname);
      for(i=0;i<string_array_size(attvals);i++)
        if(ds_attnum_name_to_value(ds,attnum,string_array_ref(attvals,i))==-1)
          ds_add_value_name(ds,attnum,string_array_ref(attvals,i));
    }
    free_arg(attname,ARG_STRING);
    free_arg(attvals,ARG_STRING_LIST);
  }
}

static void dispatch_define(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("define <attribute> <expression> : (Re)define an attribute's values\n"
	  "	or (equivalent syntax):\n"
	  "<attribute> = <expression>\n"
	  "<expression> may be composed of attribute names, numeric/symbolic values\n"
	   "parentheses and operators\n");

  add_to_help(	  "Available operators (ordered according to decreasing priority):\n");

  add_to_help(	  "power\t(binary) \"power x y\" calculates the value of x raised the power y.\n\tNote the non-use of parentheses for the function call;\n");
  add_to_help(	  "abs\t(unary) calculates absolute value of its argument,\n\tapplicable to real valued attributes only;\n"
	  "sqrt\t(unary) square root of a real, non-negative argument;\n"
	  "sign\t(unary) returns -1.0, 0.0 or +1.0 if its real argument\n\tis respectively negative, zero, or positive;\n"
	  "exp\t(unary) applicable to real valued arguments;\n"
		  "log\t(unary) natural logatithm, applicable to positive\n\treal arguments;\n");

  add_to_help(	  "log10\t(unary) decimal logarithm, applicable to positive\n\treal arguments;\n"
	  "pre\t(unary) applicable to real or symbolic data attributes\n"
		  "\t(not to individual values), it shifts the argument\n");

  add_to_help(	  "\tvalues back, so that if k is the data row index,\n"
	  "\tthen resultant[k] = argument[k-1] for k>0\n"
	  "\tand resultant[0] = argument[0] otherwise;\n"
	  "suc\t(unary) applicable to real or symbolic data attributes\n"
		  "\t(not to individual values), it shifts the argument\n");

  add_to_help(	  "\tvalues forward, so that if k is the data row index,\n"
	  "\t then resultant[k] = argument[k+1] for k<size-1\n"
	  "\tand resultant[size-1] = argument[size-1] otherwise,\n"
		  "\t(size is the number of data rows);\n");

  add_to_help(	  "deriv\t(binary), applicable to real attributes only,\n"
	  "\t\"deriv y x\" computes the first derivative dy/dx;\n"
	  "ranks\t(unary) computes ranks of its argument,\n\tintended for use with real attributes;\n"
	  "sranks\t(binary) \"sranks s r\" computes ranks of symbolic\n"
	  "\tattribute s with regard to the real attribute r,\n"
	  "\tusing a specific permutation of the values of s,\n"
	  "\twhich provides a presumably monotonic behavior of s vs. r.\n"
"year\tassumes its argument is a text string containing a date and returns\n"
		  "\tthe integer year.\n");

  add_to_help("yeardays\tassumes its argument is a text string containing a date (american\n"
          "\tstyle) with only numbers (no month names and returns the"
          "\tthe number of days since Jan 1.\n"
"dayfrac\tassumes its argument is a text string containing a time and\n"
          "\treturns a number between 0 and 1 representing the fraction of\n"
          "\tthe day.\n"
"round\trounds its argument to the nearest integer value\n");

  if ( act )
  {
    datset *ds = de->ds;
    void *arg = NULL;
    string_array *params = mk_datset_attnames(ds,TRUE,FALSE);
    char *attname = mk_arg_from_command(c,1,"attribute",ARG_STRING,0,0,0,params,1);
    int attnum = (attname)? ds_name_to_attnum(ds,attname):-1;
    free_string_array(params);
    if ( attnum < 0 )
      my_warning("Cannot define the values of a non-existent attribute. Use new first.\n");
    else arg = mk_arg_from_command(c,2,"definition",ARG_STRING,0,0,0,NULL,1);
    if(arg)
    {
      string_array *sa = mk_copy_string_array(c->args);
      dex *dx;
      char *errmess;
      string_array_remove(sa,0);
      dx = mk_parse_dex(ds,attnum,sa,&errmess);
      if ( dx == NULL )
      {
        printf("%s\n",errmess);
        free_string(errmess);
      }
      else
      {
        print_dex(ds,dx);
        apply_dex_to_dset(ds,attnum,dx);
        de->ds_changed = TRUE;
        free_dex(dx);
      }
      free_string_array(sa);
    }
    free_arg(attname,ARG_STRING);
    free_arg(arg,ARG_STRING);
  }
}

static void dispatch_select(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("select <attributes> : delete all but the selected attributes.\n"
"  Gets rid of every single attribute in the dataset EXCEPT for those\n"
"  specified in the the list of attributes. There's no UNDO!\n");
  if ( act && attnums_ok(de->ds,c) )
  {
    string_array *params = mk_datset_attnames(de->ds,TRUE,TRUE);
    string_array *args = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    free_string_array(params);
    if(args){
      select_from_string_array(de->ds,args);
      de->ds_changed = TRUE;
    }
    free_arg(args,ARG_STRING_SET);
  }
}

static void dispatch_swap(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("swap <attnum1> <attnum2> : Reposition attributes by swapping columns.\n"
"  Simply swaps the attnum (attribute number) of the two attributes. That\n"
"  will affect the order in which attributes are listed in command such\n"
"  as brief.\n");

  if ( act )
  {
    string_array *params = mk_datset_attnames(de->ds,1,1);
    char *attname1 = mk_arg_from_command(c,1,"first attribute",ARG_STRING,0,0,0,params,1);
    char *attname2 = NULL;
    int attnum1,attnum2;
    if(attname1){
      int i = find_index_in_string_array(params,attname1);
      if(i!=-1) string_array_remove(params,i);
      attname2 = mk_arg_from_command(c,2,"second attribute",ARG_STRING,0,0,0,params,1);
    }
    if(attname2){
      attnum1 = ds_name_to_attnum(de->ds,attname1);
      attnum2 = ds_name_to_attnum(de->ds,attname2);

      if (attnum1 == attnum2)
        printf("The two attributes must be different from each other\n");
      else
        ds_swap_attributes(de->ds,attnum1,attnum2);
      de->ds_changed = TRUE;
    }
    free_string_array(params);
    free_arg(attname1,ARG_STRING);
    free_arg(attname2,ARG_STRING);
  }
}

void dispatch_save(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("save {marked|unmarked|transpose} <filename>.\n"
"  The optional keyword marked or unmarked may be supplied.\n"
"  If marked is supplied, saves ONLY those records that have been\n"
"  marked (see the mark, unmark and togglemarks commands).\n"
"  If unmarked is supplied, saves ONLY those records that have been\n"
"  unmarked. If neither keyword is supplied, saves ALL the records.\n"
	   "\n");
  add_to_help("  transpose dumps the whole datset in a transposed CSV file.\n"
	      "  transpose supercedes the file format selection below.\n"
	      "  the result will not be sprat readable.  Its purpose is to.\n"
	      "  provide an alternative import format for various other\n"
	      "  crippled programs (i.e. Excel) that may choke on the number\n"
	      "  of rows or columns in the original csv file.\n");

  add_to_help("  IMPORTANT: .fds files.\n"
"  There are currently two formats to save. \n"
"\n"
".csv format\n"
"\n"
"   The default format is simple comma separated values (CSV). In SPR or\n"
"   the Auton lab, we usually give these files suffixes of .csv or .ds. The\n"
"   format of these files is that the first non-blank line of the file is\n"
	      "   a list of attribute-names separated by columns, e.g.\n");

  add_to_help("\n"
"     Age,Weight,HomeTown\n"
"\n"
"   and then all other non-blank-lines contain the same number (in the above\n"
"   example, 3) of tokens on each line, with the tokens separated by columns.\n"
"   The k'th such line contains the values of the k'th row (i.e. the\n"
"   k'th record) in the dataset. This is the same file format you get if\n"
"   you save an Excel(TM) spreadsheet in CSV format. For more details, see\n"
	      "   the load command.\n");

  add_to_help("\n"
".fds format\n"
"\n"
"   This format is usually more compact (especially if there are many\n"
"   symbolic attributes) and always much much much faster to load and save.\n"
"   The downside is that it is an SPR/Auton only format, so you can't export\n"
"   of import files in this format between non-SPR/Auton applications. You CAN\n"
"   export them between different SPR/Auton applications, even ones which run\n"
	      "   on different machines. But you can always convert between .fds and\n");

  add_to_help("   .csv formats with this tool.\n"
"  \n"
"   To save a file in .fds format, all you need do is name the file with\n"
"   a .fds suffix.\n"
"");

  if ( act )
  {
    bool marked_only = c->a1 != NULL && eq_string(c->a1,"marked");
    bool unmarked_only = c->a1 != NULL && eq_string(c->a1,"unmarked");
    bool transpose = (c->a1 != NULL && eq_string(c->a1,"transpose"));
    int argnum = (marked_only || unmarked_only || transpose) ? 2 : 1;
    string_array *sa = mk_miner_file_exts();
    char *filename = mk_arg_from_command(c,argnum,NULL,ARG_FILE_SAVE,0,0,0,sa,1);
    free_string_array(sa);

    if ( filename == NULL )
      printf("You must supply a filename\n");
    else
    {
      FILE *s = fopen(filename,"w");
      if ( s == NULL )
        printf("Can't open %s for writing\n",filename);
      else
      {
        fclose(s);
	if (transpose) 
	{
	  FILE *s = fopen(filename,"w");
	  save_datset_transpose(s,de->ds);
	  fclose(s);
	}
        else if ( marked_only || unmarked_only )
        {
          ivec *rows = mk_rows_from_marks(de->row_to_marked,marked_only);
          datset *subds = mk_ds_subset(de->ds,rows,NULL);
          ds_save(filename,subds);
          printf("Saved ONLY THE %sed rows in the dataset.\n",
                 (marked_only) ? "mark" : "unmark");
          free_datset(subds);
          free_ivec(rows);
        }
        else
        {
          if ( de->num_marked != 0 )
          {
            printf("*** Note, I'm saving all rows, whether marked\n"
                   "*** unmarked. To save only the marked rows,\n"
                   "*** enter\n"
                   "***           save marked %s\n"
                   "*** and to save only the unmarked rows, enter\n"
                   "***           save unmarked %s\n",
                   filename,filename);
          }
          ds_save(filename,de->ds);
        }
      }
      free_arg(filename,ARG_STRING);
    }
  }
}

#if 0
/* Just the bytes, ma'am -- so be careful about endianness.
   First two ints are #rows and #columns, then the actual entries
   as doubles, row by row. */
int bwrite_dym(dym *d, char *file)
{
  FILE *f = fopen(file, "w");
  int i, j;
  double el;
  
  if (!f)
    return -1;

  i = dym_rows(d);
  j = dym_cols(d);
  fwrite(&i, sizeof(int), 1, f);
  fwrite(&j, sizeof(int), 1, f);

  for (i = 0; i < dym_rows(d); i++)
    for (j = 0; j < dym_cols(d); j++) {
      el = dym_ref(d, i, j);
      if (1 != fwrite(&el, sizeof(double), 1, f)) {
	fclose(f);
	return -1;
      }
    }

  if (fclose(f))
    return -1;

  return 0;
}
#endif

#ifdef AUTON_UNUSED_CODE
/* Just the bytes, ma'am -- so be careful about endianness.
   First two ints are #rows and #columns, then the actual entries
   as doubles, row by row.  Symbols get converted to small
   integers (stored as doubles).
   Returns: 0 = OK, -1 = file error, -2 = write truncated because
   of missing value. */
static int bwrite_dset(datset *d, char *file)
{
  FILE *f = fopen(file, "w");
  int i, j, m, n;
  double el;
  
  if (!f)
    return -1;

  m = ds_num_rows(d);
  n = ds_num_atts(d);
  fwrite(&m, sizeof(int), 1, f);
  fwrite(&n, sizeof(int), 1, f);

  for (j = 0; j < n; j++) {
    if (ds_real_ref_ok(d, j)) {
      for (i = 0; i < m; i++) {
	if (ds_value_is_missing(d, i, j)) {
	  fclose(f);
	  return -2;
	}
	el = ds_real_ref(d, i, j);
	if (1 != fwrite(&el, sizeof(double), 1, f)) {
	  fclose(f);
	  return -1;
	}
      }
    } else {
      for (i = 0; i < m; i++) {
	if (ds_value_is_missing(d, i, j)) {
	  fclose(f);
	  return -2;
	}
	el = ds_symbol_ref(d, i, j);
	if (1 != fwrite(&el, sizeof(double), 1, f)) {
	  fclose(f);
	  return -1;
	}
      }
    }
  }

  if (fclose(f))
    return -1;

  return 0;
}
#endif /* AUTON_UNUSED_CODE */

static void dispatch_load(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("load <filename> [options] : Load new dataset from CSV or FDS file.\n"
"   <filename> must be the name of a datafile in one of the formats\n"
"              described below.\n"
"   [options] are\n"
"      default_real {true|false} (default value is FALSE)\n"
"         If you specify default_real true on the command line then an\n"
	   "         attribute with only a small number of values, each an integer,\n");

  add_to_help("         will be treated as a real-valued attribute. See the realize \n"
"         command for more discussion.\n"
"\n"
"      ignore_missing {true|false} (default value is FALSE)\n"
"         If there are any missing values in the dataset, and if set to\n"
"         TRUE, the loader will simply ignore any records containing missing\n"
"         values.\n"
"\n"
	      "If filename contains *, then multiple files will be loaded unto 1 datset");

  add_to_help("If the new dataset is loaded successfully, then this tool forgets\n"
"the previous dataset.\n"
"\n"
"File formats. \n"
"If the filename ends with .fds, then this tool will assume that the\n"
"file was created in standard Schenley Park Research compact .fds format,\n"
"and will loadaccordingly. For large files, this is up to hundreds \n"
"of times faster for loading, and so is heavily recommended. \n"
	      "For more discussion, see the save command.\n");

  add_to_help("\n"
"All other file suffixes are loaded with an adaptive loader that can\n"
"attempt to load with comma separated (CSV) format and/or space-separated\n"
"format. It ignores all blank lines and lines beginning with #. The\n"
"first non-ignored line is initially assumed to consist of attribute names.\n"
	      "If there's at least one number amongst them, however, loading assumes\n");

  add_to_help("that this file was created without attribute names. In that case it\n"
"treats the first line as the first record, and generates its own \n"
	      "(unimaginative) attribute names. It automatically decides whether each\n");

  add_to_help("attribute is real or symbolic. If it sees symbolic values wih spaces\n"
"in the symbolic name, it replaces the spaces with - hyphens. \n"
"");

  if ( act )
  {
    string_array *sa = mk_miner_file_exts();
    char *filename = mk_arg_from_command(c,1,NULL,ARG_FILE_OPEN,0,0,0,sa,1);

    free_string_array(sa);

    if(filename != NULL)
    {
      char *errmess;
      datset *new_ds = ds_load_with_options(filename,0,NULL,&errmess);

      if ( errmess != NULL )
      {
	printf("ERROR LOADING FROM FILE:\n%s\n   (Dataset Unchanged)\n",errmess);
	free_string(errmess);
      }
      else
      {
	free_datset(de->ds);
	de->ds = new_ds;
	
	printf("The new datset has %d rows.\n",ds_num_rows(de->ds));
	de->ds_changed = TRUE;
      }
    }
    else
      printf("You must supply a filename\n");

    if ( filename != NULL )
      free_arg(filename,ARG_STRING);
  }
}

static void dispatch_loadmany(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("loadmany <filenames> : Load new datasets from CSV or FDS file.\n"
"   Essentially the same command as load, except you may put more than\n"
"   one filename on the command line. The files will be loaded, concatenated\n"
"   if possible. One additional attribute will be created, called file\n"
"   which, for each record, will contain the filename that it was\n"
"   loaded from.\n");

  if ( act )
  {
    char *errmess;
    datset *ds = mk_datset_from_filenames(c->args,&errmess);
    if ( ds == NULL )
    {
      printf("FAILED TO DO ANY KIND OF LOADING:\n%s\n",errmess);
      free_string(errmess);
    }
    else
    {
      free_datset(de->ds);
      de->ds = ds;
      printf("The new datset has %d rows.\n",ds_num_rows(de->ds));
      de->ds_changed = TRUE;
    }
  }
}

static void dispatch_ls(void *env, command *c, bool act)
{
  set_help("ls:  lists the current working directory\n");

  if(act)
  {
#ifdef PC_PLATFORM
    FILE* fp;
    char buff[1000];
    char *filename = "C:\\ls.junk";
    char *command = mk_printf("dir > %s",filename);

    system(command);
    
    fp = fopen("C:\\ls.junk","r");
    if ( fp == NULL )
      printf("dispatch_ls can't make it's temporary file\n");
    else
    {
      while(fgets(buff,1000,fp)!=NULL) printf(buff);
      fclose(fp);
    }

    free_string(command);
    command = mk_printf("del %s",filename);
    system(command);
    free_string(command);
#else
    char *s = mk_string_from_string_array(c->args);
    char *command_to_run = mk_printf("ls %s",s);
    system(command_to_run);
    free_string(command_to_run);
    free_string(s);
#endif
  }
}

static void dispatch_sortmissing(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("sortmissing : Display attributes sorted by number missing rows.\n"
"A trivial utility that prints out the attribute names in order,\n"
"so that those with the most missing values are printed first.\n"
"");
  if ( act )
  {
    fprint_attributes_sorted_by_num_missing(stdout,de->ds);
  }
}

static void dispatch_join(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("join <file(s)> key(s) <keycode(s)> : Extra attributes from other file(s)\n"
"    Examples:\n"
"      1.  join population.fds key state\n"
"      2.  join population.fds latitude.fds key state\n"
"      3.  join population.fds key homestate/state\n"
"      4.  join age.fds keys firstname lastname\n"
	   "Suppose the current dataset in memory is\n");

  add_to_help("\n"
"city state crimerate homestate\n"
"pit  pa    100       pa\n"
"ny   ny    230       pa\n"
"phil pa    120       oh\n"
"new  nj    150       pa\n"
"\n"
"And suppose population.fds is\n"
	      "\n");

  add_to_help("state population\n"
"ha     3\n"
"nj    10\n"
"ny    25\n"
"oh    18\n"
"pa    15\n"
"\n"
"Then after example 1 the dataset in memory will be\n"
	      "\n");

  add_to_help("city state crimerate homestate population\n"
"pit  pa    100       pa        15\n"
"ny   ny    230       pa        25\n"
"phil pa    120       oh        15\n"
"new  nj    150       pa        10\n"
"\n"
"The newly added column gives you the population of the state mentioned\n"
"as the 2nd attribute in the memory dataset. Where does the population value\n"
	      "come from? From the file population.fds\n");

  add_to_help("\n"
"Example 2 allows you to merge with more than one file at once. It's\n"
"exactly equivalent to running the following commands in succession:\n"
"   join population.fds key state\n"
"   join latitude.fds key state\n"
"\n"
"Example 3 shows what we'd have to do if we wanted the population column to\n"
"be associated with the homestate in the in-memory file even though the\n"
	      "matching record we're using in population.fds is called state.\n");

  add_to_help("\n"
"Sometimes one key is not enough to disambiguate which record in\n"
"the new file contains the values you want. For example, 4 would be\n"
"useful in this case:\n"
"\n"
"ORIGINAL DATASET....                               AGE.FDS...\n"
"firstname lastname  height weight year              firstname lastname age\n"
"andrew    moore     75     135    1985              andrew    moore     34\n"
	      "jeff      lee       65     235    1999              andrew    lee       33\n");

  add_to_help("andrew    moore     75     155    1998              jeff      lee       32\n"
"andrew    lee       35      55    1998       \n"
"\n"
"when the result would be\n"
"firstname lastname  height weight year age\n"
"andrew    moore     75     135    1985 34\n"
"jeff      lee       65     235    1999 32\n"
"andrew    moore     75     155    1998 34\n"
	      "andrew    lee       35      55    1998 33\n");

  add_to_help("\n"
"Notes: * In all cases, all attributes from <filename> are included into the \n"
"         in-memory dataset. Delete those you don't want. \n"
"       * All key attributes must be symbolic\n"
"       * Key attributes must match (have the same values) in memory & in\n"
"         loaded datasets\n");

  if ( act )
  {
    datset *ds = de->ds;
    int old_num_atts = ds_num_atts(ds);
    run_join_command(ds,c);
    if ( ds_num_atts(ds) != old_num_atts )
      de -> ds_changed = TRUE;
  }
}

static void dispatch_sortrows(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("sortrows <attributes> : Re-oder the dataset records\n"
    "Takes one or more attributes as an argument. Sorts using\n"
    "the first attribute as the primary key, the next as the secondary\n"
    "key and so on.\n"
    "Good new for sort algorithm afficionados: The sort algorithm is\n"
    "careful not to unnecessarily reorder rows that are equal according\n"
    "to the specified attributes.\n");

  if ( act )
  {
    string_array *params = mk_datset_attnames(ds,1,1);
    void *arg = mk_arg_from_command(c,1,"attributes",ARG_STRING_LIST,0,0,0,params,1);
    free_string_array(params);
    if(arg)
    {
      ivec *attnums = 
        mk_unsorted_attnums_from_command(ds,c,1,command_num_tokens(c));
      printf("Sorting...\n");
      ds_sort(ds,attnums);
      my_info("The dataset has been sorted.");
      free_ivec(attnums);
    }
    free_arg(arg,ARG_STRING_LIST);

    de -> ds_changed = TRUE;
  }
}

/* re-order the attributes to put them in alphabetical order */
static void dispatch_sortatts(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  int i,j;

  set_help("sortatts: Re-oder the dataset columns\n"
	   "using the attribute name as the key\n");

  /* sorry about the bubble sort, I didn't feel like digging underneath the
     API, or writing my own version of qsort with ds_swap_attributes at the
     center -- JGS 9-10-02
  */
  if ( act )
  {
    for (i=0;i<ds_num_atts(ds);i++)
      for (j=0;j<ds_num_atts(ds)-i-1;j++)
      {
	char *name1 = ds_attnum_to_name(ds,j);
	char *name2 = ds_attnum_to_name(ds,j+1);
	if (strcmp(name1,name2) > 0) ds_swap_attributes(ds,j,j+1);
      }
    de -> ds_changed = TRUE;
  }
}

static void dispatch_removerows(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("removerows [marked|unmarked] : Remove some rows.\n"
"    removerows marked: Deletes all the marked records.\n"
"    removerows unmarked: Deletes all the unmarked records.\n"
"  For help on marking and unmarking records, see the mark command\n");

  if ( act && arg_exists_ok(c,1) )
  {
    bool marked = caseless_eq_string(c->a1,"marked");
    bool unmarked = caseless_eq_string(c->a1,"unmarked");
    if ( !marked && !unmarked )
      my_warning("Hey, that's no good! The argument should be \"marked\"\n"
	     "or \"unmarked\".\n");
    else
    {
      ivec *rows = mk_rows_from_marks(de->row_to_marked,marked);
      int old_num_rows = ds_num_rows(ds);
      int new_num_rows;
      char *buff;
      printf("Removing %smarked records...\n",(marked)?"":"un");
      ds_delete_rows(ds,rows);
      new_num_rows = ds_num_rows(ds);
      buff=mk_printf("Dataset has had %d records removed and now has %d records.\n",
	     old_num_rows - new_num_rows,new_num_rows);
      my_info(buff);
      free_string(buff);
      free_ivec(rows);
      de -> ds_changed = TRUE;
    }
  }

}

static void dispatch_concatenate(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("concatenate <file(s)> [rename] : Add records from other file(s)\n"
"    Examples:\n"
"      1.  concatenate population.fds\n"
"      2.  concatenate population.fds latitude.fds\n"
"      3.  concatenate population.fds rename\n"
"Suppose the current dataset in memory is\n"
"\n"
"city state crimerate homestate\n");

  add_to_help("pit  pa    100       pa\n"
"ny   ny    230       pa\n"
"phil pa    120       oh\n"
"new  nj    150       pa\n"
"\n"
"And suppose population.fds is\n"
"\n"
"city state crimerate homestate\n"
"new  pa     40       pa\n");

  add_to_help("gulf al     30       fl\n"
"\n"
"Then after example 1 the dataset in memory will be\n"
"\n"
"city state crimerate homestate\n"
"pit  pa    100       pa\n"
"ny   ny    230       pa\n"
"phil pa    120       oh\n"
"new  nj    150       pa\n");

  add_to_help("new  pa     40       pa\n"
"gulf al     30       fl\n"
"\n"
"All that happens is that the records from population.fds are appended\n"
"at the end of the current dataset.\n"
"\n"
"Example 2 allows you to merge with more than one file at once. It's\n"
"exactly equivalent to running the following commands in succession:\n"
"   concatenate population.fds\n");

  add_to_help("   concatenate latitude.fds\n"
"\n"
"Example 3 shows the rename option. If the columns in the population.fds\n"
"had different names than those in the current dataset, then the names\n"
"in population.fds will be ignored. (By default, if you don't include\n"
"rename on the command, the system will moan at you if it finds two\n"
"attributes with names that diagree).\n");

  if ( act )
  {
    datset *ds = de->ds;
    run_concatenate_command(ds,c);
    de -> ds_changed = TRUE;
  }
}

static void dispatch_removeduplicates(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("removeduplicates <attributes> : Remove duplicate rows.\n"
"    The simplest use of this command is: removeduplicates all.\n"
"        In this case we simply search for pairs of records that are\n"
"	identical. Whenever we find such a pair we remove the later\n"
"        of the records (needless to say, the implementation is much more\n");

  add_to_help("	efficient than this description makes it sound).\n"
"				 \n"
"    A slightly more complex use of the command is when you specify a\n"
"    set of attributes other than \"all\". In this case it considers\n"
"    two records identical if they match on all the attributes you mentioned\n"
"    even if they don't match on the attributes you don't mention.\n");

  if ( act )
  {
    string_array *params = mk_datset_attnames(ds,1,1);
    void *arg = mk_arg_from_command(c,1,"attributes",ARG_STRING_SET,0,0,0,params,0);
    ivec *attnums = arg?
      mk_unsorted_attnums_from_command(ds,c,1,command_num_tokens(c)) : mk_ivec(0);
    int old_num_rows = ds_num_rows(ds);
    int new_num_rows;
    char *buff;
    printf("Removing duplicates...\n");
    ds_remove_duplicates(ds,attnums);
    new_num_rows = ds_num_rows(ds);
    buff=mk_printf("Dataset has had %d duplicates removed and now has %d records.\n",
	   old_num_rows - new_num_rows,new_num_rows);
    my_info(buff);
    free_string(buff);
    free_ivec(attnums);
    free_string_array(params);
    free_arg(arg,ARG_STRING_SET);

    de -> ds_changed = TRUE;
  }
}

static void dispatch_batchrename(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("batchrename <filename> : Rename attributes from a file.\n"
"<filename> must be a textfile containing exactly the same number of\n"
"tokens as there are attributes in the dataset. The tokens must be separated\n"
"by spaces and/or on separate lines. The i'th token in the file \n"
"becomes the name of the i'th attribute in the dataset.\n"
"");
  if ( act )
  {
    char *filename = c->a1;
    if ( filename == NULL )
      printf("You must supply a filename\n");
    else
    {
      rename_from_file(de->ds,filename);
    }
  }
}

static void dispatch_togglemarks(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("togglemarks : Make marked rows unmarked and vice versa.\n"
"See the mark command for more detail about marks.\n"
"");

  if ( act )
  {
    int i;
    int num_rows = ds_num_rows(de->ds);
    char *buff;
    de->num_marked = num_rows - de->num_marked;
    for ( i = 0 ; i < num_rows ; i++ )
      ivec_set(de->row_to_marked,i,1-ivec_ref(de->row_to_marked,i));
    buff=mk_printf("%d%s currently marked\n",de->num_marked,
                     (de->num_marked==1)?" row is":" rows are");
    my_info(buff);
    free_string(buff);
  }
}

static void dispatch_showmarked(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("showmarked <number> <attributes> : Show a subset of the data.\n"
"This function allows you to see a subset of the rows and attributes\n"
"of the dataset. It displays the first <number> marked rows in the\n"
"dataset, one per line of output but the only attribute values that\n"
"appear on each line of output are those specified in <attributes>.\n");

  add_to_help("If you wanted to see values of attribute height and age for records\n"
"numbers 40 through 49 you could do\n"
"   mark all\n"
"   unmark row < 40\n"
"   show 10 height age\n"
"See the mark command for more details about marking.\n"
"");

  if ( act )
  {
    string_array *params = mk_datset_attnames(de->ds,1,1);
    int *numrows = mk_arg_from_command(c,1,"number of records",ARG_INT,1,ds_num_rows(de->ds),10,NULL,1);
    string_array *attnames = (numrows)? mk_arg_from_command(c,2,"attributes",ARG_STRING_SET,0,0,0,params,0):NULL;
    free_string_array(params);
    if(!attnames){}
    else if ( !attnums_ok_from(de->ds,c,2) || ivec_size(c->attnums) <= 0 )
      my_warning("You should type: showmarked <number> <att1> <att2> .. <attn>\n"
             "Where <att1> .. <attn> are a list of attribute names. I will\n"
             "then show you the first <number> of records which are marked.\n");
    else
    {
      int n = *numrows;
      int i,j;
      int num_seen = 0;
      printf("%6s ","Row");
      for ( j = 0 ; j < ivec_size(c->attnums) ; j++ )
        printf("%10s ",ds_attnum_to_name(de->ds,ivec_ref(c->attnums,j)));
      printf("\n");
      for ( i = 0 ; i < ds_num_rows(de->ds) && num_seen < n ; i++ )
      {
        if ( ivec_ref(de->row_to_marked,i) == 1 )
        {
          printf("%6d ",i);
          for ( j = 0 ; j < ivec_size(c->attnums) ; j++ )
          {
            int attnum = ivec_ref(c->attnums,j);
            char *name = mk_row_attnum_to_name(de->ds,i,attnum);
            printf("%10s ",name);
            free_string(name);
          }
          printf("\n");
          num_seen += 1;
        }
      }
      printf("I showed you the first %d of the %d marked row%s.\n",
             num_seen,de->num_marked,(de->num_marked==1)?"":"s");
    }
    free_arg(attnames,ARG_STRING_SET);
    free_arg(numrows,ARG_INT);
  }
}

static void process_marked_rows(datset *ds,dex *dx,bool mark,bool submark,
                         ivec *row_to_marked,int *num_marked)
{
  int row;
  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    bool row_already_marked = ivec_ref(row_to_marked,row) == 1;

    /* submark ignores records currently not marked */
    if (submark && !row_already_marked) continue;

    if ( mark && !row_already_marked  ) /* the mark command */
    {
      bool missing;
      double eval = eval_dex(ds,dx,row,&missing);
      if ( !missing && eval != 0.0 )
      {
        ivec_set(row_to_marked,row,1);
        *num_marked += 1;
      }
    }
    else if ( !mark && row_already_marked ) /* the unmark command */
    {
      bool missing;
      double eval = eval_dex(ds,dx,row,&missing);
      if ( !missing && eval != 0.0 )
      {
        ivec_set(row_to_marked,row,0);
        *num_marked -= 1;
      }
    }
    else if ( submark && mark && row_already_marked ) /* the submark command */
    {
      bool missing;
      double eval = eval_dex(ds,dx,row,&missing);
      if ( !missing && eval == 0.0 )
      {
      	ivec_set(row_to_marked,row,0);
	      *num_marked -= 1;
      }
    }
  }
}

/* There are now 3 modes for this command:
   mark    - go through the entire dataset and mark all records that match the
             criterion (leaving all others unchanged)
   unmark  - go through the entire dataset and unmark all records that match
             the criterion (leaving all others unchanged)
   submark - go through the only the set of marked records and unmark those
             that don't match the criterion
*/
static void dispatch_mark_or_unmark(void *env,command *c)
{
  dsenv *de = (dsenv *) env;
  bool submark = eq_string(c->s,"submark");
  bool mark = submark || eq_string(c->s,"mark");
  char *errmess;
  dex *dx;
  char *arg = mk_arg_from_command(c,1,"constraint",ARG_STRING,0,0,0,NULL,1);
  if(arg) {
    dx = mk_parse_dex(de->ds,-1,c->args,&errmess);
    if ( dx == NULL )
    {
      my_warning(errmess);
      free_string(errmess);
    }
    else
    {
      char *buff;
      process_marked_rows(de->ds,dx,mark,submark,de->row_to_marked,
			  &de->num_marked);
      free_dex(dx);
      buff=mk_printf("%d%s currently marked\n",de->num_marked,
             (de->num_marked==1)?" row is":" rows are");
      my_info(buff);
      free_string(buff);
    }
  }
  free_arg(arg,ARG_STRING);
}

static void dispatch_mark(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("mark <constraint> : Set all rows matching constraint to be marked.\n"
"When the program starts, or when a new dataset is loaded, all rows are \n"
"initially unmarked.\n"
"You can mark subsets of the rows at any time. And then, if you wish,\n"
"you can unmark subsets at any time. You can then save only the marked, or\n"
"only the unmarked rows if you wish (see the save command).\n"
"To mark all the rows matching some constraint, just type mark <constraint>.\n");

  add_to_help("Legal constraints are...\n"
"    all                - matches every single row\n"
"    missing            - matches only rows with missing value(s)\n"
"<attribute> == <value> - matches only rows in which given SYMBOLIC attribute\n"
"                         has the given SYMBOLIC value.\n"
	      "<attribute> < <number> - matches only rows in which given REAL attribute\n");

  add_to_help("                         has a numeric value strictly below threshold.\n"
"<attribute> > <number> - matches only rows in which given REAL attribute\n"
"                         has a numeric value strictly above threshold.\n");

  add_to_help("row < <number>         - matches only the rows with row_numbers strictly\n"
"                         below <number>. The first (top) row in the dataset\n"
"                         has row_number 0 etc.\n"
"row > <number>         - matches only the rows with row_numbers strictly \n"
"						  greater than <number>\n" 	
"");

  if ( act ) dispatch_mark_or_unmark(de,c);
}

static void dispatch_unmark(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("unmark <constraint> : Set all rows matching constraint to be unmarked.\n"
"Unmarks a row. See the mark command for more details.\n"
"");

  if ( act ) dispatch_mark_or_unmark(de,c);
}

static void dispatch_submark(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("submark <constraint> : Leave only marked records matching constraint marked.\n"
"Exactly the same syntax and usage as "
"the mark command, except that it looks only at records that are already "
"marked and leaves them marked if they meet the given condition, and unmarks "
"them if they don't."
"");

  if ( act ) dispatch_mark_or_unmark(de,c);
}

#ifdef AUTON_UNUSED_CODE
static void dispatch_edit(void *env,command *c,bool act)
{
  set_help("edit: Allows editing of all properties of all attributes at once.\n");

  if(act) {
#ifndef PC_MVIS_PLATFORM
    printf("The edit command is not supported in the console version.\n");
#else
    char *str;
    static edit_attribs_dlg* last_dlg_info = NULL;
    if(Num_Iterations==0) DlgInfo = mk_edit_attribs_dlg(((dsenv*)env)->ds);
    else DlgInfo = last_dlg_info;
    gui_prompt("",CLI_PROCESSING,TRUE);
    gui_dialog(IDD_EDIT_ATTRIBS,process_edit_attribs);
    str = ((edit_attribs_dlg*)DlgInfo)->str;
    if(str){
      last_dlg_info = ((edit_attribs_dlg*)DlgInfo);
      Next_Command = mk_command_from_string(((edit_attribs_dlg*)DlgInfo)->str);
      free_string(str);
      ((edit_attribs_dlg*)DlgInfo)->str = NULL;
    } else {
      Stop_Commands = TRUE;
      free_edit_attribs_dlg(DlgInfo);
    }
    gui_unprompt();
#endif
  }
}
#endif /* AUTON_UNUSED_CODE */

static void dispatch_shuffle(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("shuffle : Randomly re-order the dataset rows.\n"
"The row_numbers of all the records in the dataset change randomly, and when\n"
"saved, will be saved in this different order. This is especially useful\n"
"for creating training and test sets. For example, suppose you want to\n"
"save a test-set of a random subset of rows, and you want that test-set to\n"
"be of size 1000 rows, and you want the remainder of the records to be\n");

  add_to_help("saved to a training set, you could do the following...\n"
"\n"
"  shuffle\n"
"  mark row < 1000\n"
"  save marked test.csv\n"
"  save unmarked train.csv\n"
"");

  if ( act )
  {
    datset *newds = mk_ds_random_subset(de->ds,ds_num_rows(de->ds));
    free_datset(de->ds);
    de->ds = newds;
    printf("You now have the same dataset, but the rows\n"
           "are in randomized order\n");
    de -> ds_changed = TRUE;
  }
}

static void dispatch_compat(void *env,command *c,bool act)
{
  set_help("compat <file1> <file2> : enables 2 compatible files to be used as training and test\n"
"and validation data together.\n"
"");

  if ( act )
  {
    char *errmess = NULL;
    string_array *params = mk_miner_file_exts();
	char *filename_a = mk_arg_from_command(c,1,"First file to open",ARG_FILE_OPEN,0,0,0,params,1);
  char *filename_b = (filename_a)? mk_arg_from_command(c,2,"Second file to open",ARG_FILE_OPEN,0,0,0,params,1):NULL;
  string_array_remove(params,3);
  string_array_remove(params,3);
  if(filename_b)
  {
	  char *compat_filename_a = mk_arg_from_command(c,3,"First file to save",ARG_FILE_SAVE,0,0,0,params,1);
    char *compat_filename_b = (compat_filename_a)? mk_arg_from_command(c,4,"Second file to save",ARG_FILE_SAVE,0,0,0,params,1):NULL;
    bool *rename_atts = (compat_filename_b)? mk_arg_from_command(c,5,"rename conflicting attributes",ARG_BOOL,0,0,FALSE,NULL,1):NULL;
	  datset *dsa=NULL, *dsb=NULL;
    if(rename_atts)
    {
	    dsa = ds_load_with_options(filename_a, /*argc=*/0,
										    /*argv=*/NULL,&errmess);

	    if ( !errmess )
	      dsb = ds_load_with_options(filename_b, /*argc=*/0,
										      /*argv=*/NULL,&errmess);

	    if ( !errmess )
  	    ds_compat(dsa, dsb, *rename_atts, &errmess);

      if ( !errmess ) 
      {
	      ds_save(compat_filename_a,dsa);
	      ds_save(compat_filename_b,dsb);
	      my_info("Successfully enabled compatibility and saved");
      } 
      else 
      {
        my_warning(errmess);
        free_string(errmess);
      }
    }
    free_arg(compat_filename_a,ARG_FILE_SAVE);
    free_arg(compat_filename_b,ARG_FILE_SAVE);
    free_arg(rename_atts,ARG_BOOL);
	  if(dsa) free_datset(dsa);
	  if(dsb) free_datset(dsb);
  }
  free_arg(filename_a,ARG_FILE_OPEN);
  free_arg(filename_b,ARG_FILE_OPEN);
  free_string_array(params);
  }
}	

#ifdef OBSCURE
static void dispatch_binary_explosion(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  set_help("binary_explosion <attributes> : Create new attributes from bits\n"
                "For each attribute mentioned, creates new attributes, one of each\n"
                "binary bit of the original attribute. For example, if there is\n"
                "a real valued attribute called num_cars which can values between\n"
                "0 to 5, would create three new attributes called num_cars_bit_0,\n"
                "num_cars_bit_1 and num_cars_bit_2. _bit_0 corresponds to the\n"
                "LLEAST significant bit in the original attribute.\n");

  if ( act && attnums_ok(de->ds,c) )
  {
    ds_binary_explosion_of_attnums(de->ds,c->attnums);
    printf("The dataset now has %d attributes\n",ds_num_atts(de->ds));
  }
}
#endif /* OBSCURE */
static void dispatch_frequent(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;
  char *errmess = NULL;
  datset *newds;

  set_help("frequent <freq_atts> <match_atts> <setsize> <mincount> <filename>: make frequent datset\n"
"  freq_atts  - the attributes of which you want to find frequent tuples\n"
"  match_atts - the attributes that indicate which records should be considered\n"
"               together in finding frequent attribute tuples\n");

  add_to_help("  setsize    - the size of the tuple to be found (e.g. 2 indicates find all\n"
"               frequent pairs)\n"
"  mincount   - the minimum number of occurences before a tuple will be included\n" 
"               in the final result\n"
"  filename   - file in which to save the resulting frequent dataset \n");

  if (act)
  {
    string_array *freqstring  = (command_num_tokens(c) > 1) ? 
                                mk_broken_quoteless_string(command_arg_ref(c,1)) : NULL;
    string_array *matchstring = (command_num_tokens(c) > 2) ?
                                mk_broken_quoteless_string(command_arg_ref(c,2)) : NULL;
    int setsize               = (command_num_tokens(c) > 3) ?
                                atoi(command_arg_ref(c,3)) : 2;
    int minsetcount           = (command_num_tokens(c) > 4) ?
                                atoi(command_arg_ref(c,4)) : 0;
    char *out_file            = (command_num_tokens(c) > 5) ?
                                command_arg_ref(c,5) : "default.csv";

    ivec *freqatts = (freqstring) ? 
                     mk_unsorted_attnums_from_string_array(ds,freqstring,&errmess) : NULL;

    ivec *matchatts = (matchstring && (string_array_size(matchstring) > 0)) ? 
                      mk_unsorted_attnums_from_string_array(ds,matchstring,&errmess) : 
                      NULL;

    if (errmess)
    {
      printf("Error: %s\n",errmess);
      free_string(errmess);
    }
    
    newds = mk_frequent_datset(ds,matchatts,freqatts,setsize,minsetcount);
    
    if (newds) 
    {
      ivec *sort_key = mk_ivec(2);
      ivec_set(sort_key,0,ds_num_atts(newds)-2); /* max ratio */
      ivec_set(sort_key,1,ds_num_atts(newds)-1); /* count     */
      ds_sort(newds,sort_key);
      printf("\nThe new frequent datset was created (%d rows).\n",ds_num_rows(newds));
      ds_save(out_file,newds);
      free_datset(newds);
      free_ivec(sort_key);
    }

    if (matchstring) free_string_array(matchstring);
    if (freqstring) free_string_array(freqstring);
    if (matchatts) free_ivec(matchatts);
    if (freqatts) free_ivec(freqatts);
  }
}

static void dispatch_transpose(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("transpose NEEDS DOCUMENTING");

  if ( act )
  {
    datset *newds = mk_run_transpose(c,ds);
    if ( newds != NULL )
    {
      free_datset(de->ds);
      de -> ds = newds;
      de -> ds_changed = TRUE;
    }
  }
}

static void dispatch_shear(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("shear rowatt <att> colatt <att> cellatt <att>\n"
    "rowatt must be a symbolic attribute\n"
    "colatt must be a symbolic attribute\n"
    "cellatt must be a real attribute\n"
"\n"
    "Replaces the new dataset by one in which the number of rows\n"
    "equals the original arity of rowsatt and the number of columns\n");

  add_to_help(    "equals one plus the arity of colatt. The i+1'th attribute in the new\n"
    "dataset is given the same name as the i'th value of colatt in the\n"
    "original dataset. The value in the r'th row of the new dataset\n"
    "in the first (leftmost) column is symbolic: its the symbolic value\n"
    "of the r'th value of rowatt. The value in the i+1'th column and\n"
    "the r'th row is the value that cellatt had in the original\n"
    "dataset in the first record in which rowatt==r and colatt==i.\n"
"    \n"
"EXAMPLE: Old dataset:\n");

  add_to_help("\n"
    "GENDER STATE EYES  AGE HEIGHT\n"
    "M      PA    Blue  17  6\n"
    "M      CA    Brown 39  5\n"
    "F      MA    Blue  54  6\n"
    "M      MA    Grey  16  6\n"
    "F      PA    Grey  17  4\n"
"\n"
    "If you run the command:\n");

  add_to_help(      "shear rowatt GENDER colatt EYES cellatt AGE\n"
    "the dataset would become:\n"
"\n"
    "ROWNAME Blue Brown Grey\n"
    "M       17   39    16\n"
    "F       54   ?     17\n"
"\n"
    "...where ? denotes \"missing value\"\n"
"\n");

  if ( act )
  {
    datset *newds = mk_sheared_from_command(c,ds);
    if ( newds != NULL )
    {
      free_datset(de->ds);
      de -> ds = newds;
      de -> ds_changed = TRUE;
    }
  }
}

static void dispatch_cartesian(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("cartesian <new_name> <attnums>: create a new attribute called new_name whose values are the cartesian product of the values of attnums");

  if ( act )
  {
    bool changed = run_add_cartesian_attribute(c,ds);
    if ( changed )
      de -> ds_changed = TRUE;
  }
}

static void dispatch_nomissing(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  set_help("nomissing NEEDS DOCUMENTING");

  if ( act )
  {
    if ( !ds_has_missing_values(ds) )
      printf("Nothing changed. ds has no missing values\n");
    else
    {
      int old_rows = ds_num_rows(ds);
      int old_atts = ds_num_atts(ds);

      ds_remove_missing_rows_and_atts(ds);

      {
        int new_rows = ds_num_rows(ds);
        int new_atts = ds_num_atts(ds);
        printf("I removed %d row(s) and %d attribute(s).\n"
               "There are now no missing values in the dataset\n",
               old_rows-new_rows,old_atts-new_atts);
      }

      de->ds_changed = TRUE;
    }
  }
}

#ifdef NOVPORT
void dispatch_noise(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  Menu_ID = ID_PETFOOD_YESTERDAY_NOISINESS;
  if(act)
  {
    string_array *params = mk_datset_attnames(ds,0,0);
    string_array *tattnames = NULL, *gattnames = NULL;
    char *attname = NULL;
    int *samplesize = NULL;
    ivec *tattnums = NULL,*gattnums = NULL;
    ivec *ds_attnums = mk_identity_ivec(ds_num_atts(ds));
    int attnum;
    int def = find_index_in_string_array(params,"day");
    if(def==-1) def = 0;
    attname = mk_arg_from_command(c,1,"attname",ARG_STRING,0,0,0,params,1);
    if(attname){
      attnum = ds_name_to_attnum(ds,attname);
      samplesize = mk_arg_from_command(c,2,"samplesize",ARG_INT,100,100000,1000,NULL,1);      
    }
    if(samplesize){
      free_string_array(params);
      params = mk_datset_attnames(ds,1,1);
      tattnames = mk_arg_from_command(c,-1,"test",ARG_STRING_LIST,0,0,def,params,1);
    }
    if(tattnames){
      tattnums = mk_attnums_from_command_keyword(ds,c,ds_attnums,"test",NULL);
      free_string_array(params);
      params = mk_datset_attnames(ds,0,1);
      gattnames = mk_arg_from_command(c,-1,"group",ARG_STRING_SET,0,0,0,params,1);
    }
    free_string_array(params);

    if(gattnames){
      gattnums = mk_attnums_from_command_keyword(ds,c,ds_attnums,"group",NULL);
      if(gattnums)
      {
        run_ds_noise(ds,attnum,tattnums,gattnums,*samplesize);
      }
    }

    free_ivec(ds_attnums);
    free_arg(attname,ARG_STRING);
    free_arg(samplesize,ARG_INT);
    free_arg(tattnames,ARG_STRING_LIST);
    free_arg(gattnames,ARG_STRING_SET);
    if(tattnums) free_ivec(tattnums);
    if(gattnums) free_ivec(gattnums);
  }
}

void dispatch_clump(void *env,command *c,bool act)
{
  dsenv *de = (dsenv *) env;
  datset *ds = de->ds;

  Menu_ID = ID_PETFOOD_YESTERDAY_CLUMPTESTING;
  if(act)
  {
    string_array *params = mk_datset_attnames(ds,0,0);
    string_array *tattnames = NULL, *gattnames = NULL;
    char *attname;
    ivec *tattnums = NULL,*gattnums = NULL;
    ivec *ds_attnums = mk_identity_ivec(ds_num_atts(ds));
    int attnum;
    int def = find_index_in_string_array(params,"day");
    if(def==-1) def = 0;
    attname = mk_arg_from_command(c,1,"attname",ARG_STRING,0,0,0,params,1);
    if(attname){
      attnum = ds_name_to_attnum(ds,attname);
      free_string_array(params);
      params = mk_datset_attnames(ds,1,1);
      tattnames = mk_arg_from_command(c,-1,"test",ARG_STRING_LIST,0,0,def,params,1);
    }
    if(tattnames){
      tattnums = mk_attnums_from_command_keyword(ds,c,ds_attnums,"test",NULL);
      free_string_array(params);
      params = mk_datset_attnames(ds,0,1);
      gattnames = mk_arg_from_command(c,-1,"group",ARG_STRING_SET,0,0,0,params,1);
    }
    free_string_array(params);

    if(gattnames){
      gattnums = mk_attnums_from_command_keyword(ds,c,ds_attnums,"group",NULL);
      if(gattnums)
      {
        run_ds_clump_size(ds,attnum,tattnums,gattnums,TRUE);
        run_ds_clump_size(ds,attnum,tattnums,gattnums,FALSE);
      }
    }

    free_ivec(ds_attnums);
    free_arg(tattnames,ARG_STRING_LIST);
    free_arg(gattnames,ARG_STRING_SET);
    if(tattnums) free_ivec(tattnums);
    if(gattnums) free_ivec(gattnums);
  }
}
#endif

static void editenv_command(void *env,command *c)
{
  generic_dispatch(env,c,"symbolize",dispatch_symbolize);
  generic_dispatch(env,c,"realize",dispatch_realize);
  generic_dispatch(env,c,"explode",dispatch_explode);
  generic_dispatch(env,c,"normalize",dispatch_normalize);
  generic_dispatch(env,c,"unsymbolize",dispatch_unsymbolize);
  generic_dispatch(env,c,"unrealize",dispatch_unrealize);
  generic_dispatch(env,c,"symb2real",dispatch_symb2real);
  generic_dispatch(env,c,"rename",dispatch_rename);
  generic_dispatch(env,c,"set_missing",dispatch_set_missing);
  generic_dispatch(env,c,"rename_value",dispatch_rename_value);
  generic_dispatch(env,c,"delete_value",dispatch_delete_value);
  generic_dispatch(env,c,"delete",dispatch_delete);
  generic_dispatch(env,c,"new",dispatch_new);
  generic_dispatch(env,c,"new_value",dispatch_new_value);
  generic_dispatch(env,c,"define",dispatch_define);
  generic_dispatch(env,c,"copy",dispatch_copy);
  generic_dispatch(env,c,"copyrows",dispatch_copyrows);
}

void unmark_all_if_ds_changed(dsenv *de)
{
  if ( de->ds_changed &&
       (de->num_marked > 0 || ivec_size(de->row_to_marked) != ds_num_rows(de->ds)) )
  {
    printf("*** Because the dataset set has changed, I am erasing all\n"
           "*** the marks. No dataset records are marked any longer.\n");
    unmark_all(de);
  }
}

void dsenv_command(void *env,command *c)
{
  dsenv *de = (dsenv *) env;
  de -> ds_changed = FALSE;

  basic_command((void *)NULL,c);
  editenv_command(env,c);
  generic_dispatch(env,c,"brief",dispatch_brief);
  generic_dispatch(env,c,"info",dispatch_info);
  generic_dispatch(env,c,"symbolize_missing",dispatch_symbolize_missing);
  generic_dispatch(env,c,"select",dispatch_select);
  generic_dispatch(env,c,"swap",dispatch_swap);
  generic_dispatch(env,c,"save",dispatch_save);
  generic_dispatch(env,c,"load",dispatch_load);
  generic_dispatch(env,c,"loadmany",dispatch_loadmany);
  generic_dispatch(env,c,"sortmissing",dispatch_sortmissing);
  generic_dispatch(env,c,"batchrename",dispatch_batchrename);
  generic_dispatch(env,c,"togglemarks",dispatch_togglemarks);
  generic_dispatch(env,c,"showmarked",dispatch_showmarked);
  generic_dispatch(env,c,"mark",dispatch_mark);
  generic_dispatch(env,c,"unmark",dispatch_unmark);
  generic_dispatch(env,c,"submark",dispatch_submark);
  generic_dispatch(env,c,"shuffle",dispatch_shuffle);
#ifdef OBSCURE
  generic_dispatch(env,c,"binary_explosion",dispatch_binary_explosion);
  generic_dispatch(env,c,"nearest",dispatch_nearest);
#endif
  generic_dispatch(env,c,"correlate",dispatch_correlate);
  generic_dispatch(env,c,"compat",dispatch_compat);
  generic_dispatch(env,c,"ls",dispatch_ls);
  generic_dispatch(env,c,"join",dispatch_join);
  generic_dispatch(env,c,"concatenate",dispatch_concatenate);
  generic_dispatch(env,c,"sortatts",dispatch_sortatts);
  generic_dispatch(env,c,"sortrows",dispatch_sortrows);
  generic_dispatch(env,c,"removerows",dispatch_removerows);
  generic_dispatch(env,c,"removeduplicates",dispatch_removeduplicates);
  generic_dispatch(env,c,"frequent",dispatch_frequent);
  generic_dispatch(env,c,"shear",dispatch_shear);
  generic_dispatch(env,c,"transpose",dispatch_transpose);
  generic_dispatch(env,c,"cartesian",dispatch_cartesian);
  generic_dispatch(env,c,"nomissing",dispatch_nomissing);

  unmark_all_if_ds_changed(de);
}

void ds_cli_edit(datset **ds,int argc, char *argv[])
{
  dsenv *de = mk_dsenv_incfree_datset(*ds);
  generic_cli((void *)de,"dscli> ",dsenv_command,argc, argv);
  *ds = mk_datset_free_dsenv(de);
}

void edit_main(int argc,char *argv[])
{
  const char *infname = string_from_args("in",argc,argv,"default.ds");
  datset *ds = ds_load_with_options_simple(infname,argc,argv);
  ds_cli_edit(&ds,argc,argv);
  free_datset(ds);
}

void dset_main(int argc,char *argv[],const char *filename)
{
  datset *ds = ds_load_with_options_simple(filename,argc,argv);
  ds_cli_edit(&ds,argc,argv);
  free_datset(ds);
}

