/*
   File:        join.c
   Author:      Andrew W. Moore
   Created:     Sun Jul 18 09:17:40 EDT 1999
   Description: Joing of two datasets through 1 or more linked attributes

   Copyright 1999, Schenley Park Research
*/

#include "join.h"


char *mk_string_from_attnum_and_value(datset *ds,int attnum,int value)
{
  return mk_printf("%s=%s",ds_attnum_to_name(ds,attnum),
                   ds_attnum_value_to_name(ds,attnum,value));
}

string_array *mk_query_string_array_from_attnums_and_values(datset *ds,
                                                            ivec *attnums,
                                                            ivec *values)
{
  int i;
  string_array *sa = mk_string_array(0);
  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    int value = ivec_ref(values,i);
    char *s = mk_string_from_attnum_and_value(ds,attnum,value);
    add_to_string_array(sa,s);
    free_string(s);
  }
  return sa;
}

char *mk_query_string_from_attnums_and_values(datset *ds,ivec *attnums,
                                              ivec *values)
{
  string_array *sa = 
    mk_query_string_array_from_attnums_and_values(ds,attnums,values);
  char *s = mk_string_from_string_array(sa);
  free_string_array(sa);
  return s;
}

ivec *mk_values_from_attnums_and_row(datset *ds,int row,ivec *attnums)
{
  int size = ivec_size(attnums);
  int i;
  ivec *values = mk_ivec(size);
  for ( i = 0 ; i < size ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    int value = ds_symbol_ref(ds,row,attnum);
    ivec_set(values,i,value);
  }
  return values;
}

char *mk_query_string_from_attnums_and_row(datset *ds,int row,ivec *attnums)
{
  ivec *values = mk_values_from_attnums_and_row(ds,row,attnums);
  char *s = mk_query_string_from_attnums_and_values(ds,attnums,values);
  free_ivec(values);
  return s;
}

/* As below, exceopt only does it for rows in rows1 of ds1 and rows2 of ds2,
   and only tries to match on attributes mentioned in index k and above of
   attnums1 and attnums2 */
void ds_join_helper(datset *ds1,datset *ds2,
                    ivec *attnums1,ivec *attnums2,
                    ivec *rows1,ivec *rows2,int k,
                    char **r_errmess)
{
  bool bailout = FALSE;

  if ( ivec_size(rows1) > 0 )
  {
    if ( *r_errmess == NULL && ivec_size(rows2) <= 0 )
    {
      int row1 = ivec_ref(rows1,0);
      char *s = mk_query_string_from_attnums_and_row(ds1,row1,attnums1);
      printf("There is nothing in the right-hand dataset that matches %s on row %d (0-indexed) of the first dataset.\nIts values will be left as default\n",s,row1);
      free_string(s);
      bailout = TRUE;
    }
    if ( *r_errmess == NULL && k == ivec_size(attnums1) && !bailout )
    {
      if ( ivec_size(rows2) > 1 )
      {
        int row2_0 = ivec_ref(rows2,0);
        int row2_1 = ivec_ref(rows2,1);
        char *s = mk_query_string_from_attnums_and_row(ds2,row2_0,attnums2);
        *r_errmess = mk_printf("There's more than one record in the second "
                               "dataset matching\n"
                               "%s (records %d and %d, 0-indexed)\n",s,
                               row2_0,row2_1);
        free_string(s);
      }
      if ( *r_errmess == NULL )
      {
        int num_new_atts = ds_num_atts(ds2) - ivec_size(attnums2);
        int ds1_start_new = ds_num_atts(ds1) - num_new_atts;
        int row2 = ivec_ref(rows2,0);
        int j;
        for ( j = 0 ; j < ivec_size(rows1) ; j++ )
        {
          int row1 = ivec_ref(rows1,j);
          int ds1_attnum = ds1_start_new;
          int ds2_attnum = 0;
          while ( ds2_attnum < ds_num_atts(ds2) )
          {
            if ( !is_in_ivec(attnums2,ds2_attnum) )
            {
              ds_copy_value(ds2,row2,ds2_attnum,ds1,row1,ds1_attnum);
              ds1_attnum += 1;
            }
            ds2_attnum += 1;
          }
        }
      }
    }   
    else if ( *r_errmess == NULL && !bailout )
    {
      ivec_array *subrows1 = mk_subrows(ds1,rows1,ivec_ref(attnums1,k));
      ivec_array *subrows2 = mk_subrows(ds2,rows2,ivec_ref(attnums2,k));
      int i;
      int subsize = ivec_array_size(subrows1);
      if ( subsize != ivec_array_size(subrows2) )
        my_error("ajnxduciowbcdou");

      for ( i = 0 ; *r_errmess == NULL && i < subsize ; i++ )
        ds_join_helper(ds1,ds2,attnums1,attnums2,
                       ivec_array_ref(subrows1,i),
                       ivec_array_ref(subrows2,i),
                       k+1,r_errmess);


      free_ivec_array(subrows2);
      free_ivec_array(subrows1);
    }
  }
}

/* Incomplete... (AWM)
bool ds_values_of_first_are_subset_of_second(datset *ds1,int attnum1,
					     datset *ds2,int attnum2)
{
  sarr *names1 = mk_value_names_from_attnum(ds1,attnum1);
  sarr *names2 = mk_value_names_from_attnum(ds2,attnum2);
  bool result = first_sa_is_subset(names1,names2);
  free_sarr
*/

/* PRE: attnums1 is a list (not just set) of attnums in ds1
        attnums2 is a list (not just set) of attnums in ds2
        order matters in both sets of attnums
        both sets of attnums must be symbolic
        both sets must be the same size
        For any i, Let
           a1 = attnums1[i]
           a2 = attnums2[i]
        Then we insist that the a1'th attribute of ds1 has the same shape
          as the a2'th attribute of ds2: i.e. the same arity, and the k'th
          value of the a1'th attribute of ds1 must have the same name as 
          the k'th value of the a2'th attribute of ds2.
        Finally we insist that in ds2 no two records have the same joint
        values of the attnums2 attributes.

   POST: For each attribute that exists in ds2 that is NOT included in
         attnums2, a new attribute with the same name will be added to ds1.
         Suppose attnum is an attribute in ds2 that's not mentioned in attnums2.
         Then attnum will be a new attribute in ds1 and the value of it on the
         k'th record in ds1 will be the same as the value that attnum has in
         the unique record in ds2 for which the values of all the things in attnums2
         are bthe same as all the things in attnums1.
*/
void ds_join(datset *ds1,datset *ds2,
             ivec *attnums1,ivec *attnums2,
             char **r_errmess)
{
  int size = ivec_size(attnums1);
  int i;

  *r_errmess = NULL;

  if ( size != ivec_size(attnums2) )
    *r_errmess = mk_printf("Num attributes in the joint specification for the\n"
                           "first dataset different from the number in the second\n"
                           "dataset.\n");
  for ( i = 0 ; *r_errmess == NULL && i < size ; i++ )
  {
    int attnum1 = ivec_ref(attnums1,i);
    int attnum2 = ivec_ref(attnums2,i);
    if ( *r_errmess == NULL && !ds_attnum_to_symbolicp(ds1,attnum1) )
      *r_errmess = mk_printf("Can't join with real attribute %s in first dataset.",
                             ds_attnum_to_name(ds1,attnum1));
    if ( *r_errmess == NULL && !ds_attnum_to_symbolicp(ds2,attnum2) )
      *r_errmess = mk_printf("Can't join with real attribute %s in second dataset.",
                             ds_attnum_to_name(ds2,attnum2));
    if (!(*r_errmess) && !ds_attnums_have_same_shape(ds1,attnum1,ds2,attnum2)) 
      ds_compat_symbolic_attribute_values(ds1,attnum1,ds2,attnum2,r_errmess);
  }

  if ( *r_errmess == NULL )
  {
    ivec *rows1 = mk_identity_ivec(ds_num_rows(ds1));
    ivec *rows2 = mk_identity_ivec(ds_num_rows(ds2));
    int num_new_atts = ds_num_atts(ds2) - ivec_size(attnums2);
    int ds2_attnum = 0;
    while ( ds2_attnum < ds_num_atts(ds2) )
    {
      if ( !is_in_ivec(attnums2,ds2_attnum) )
      {
	printf("Adding new attribute %s to current dataset\n",
	       ds_attnum_to_name(ds2,ds2_attnum));

        if ( ds_attnum_to_symbolicp(ds2,ds2_attnum) )
        {
          string_array *value_names = 
            mk_value_names_from_attnum(ds2,ds2_attnum);
          ds_add_symbolic_attribute(ds1,ds_attnum_to_name(ds2,ds2_attnum),
                                    value_names);
          free_string_array(value_names);
        }
        else
          ds_add_real_attribute(ds1,ds_attnum_to_name(ds2,ds2_attnum));
      }
      ds2_attnum += 1;
    }

    ds_join_helper(ds1,ds2,
                   attnums1,attnums2,
                   rows1,rows2,0,r_errmess);
    
    if ( *r_errmess != NULL )
    {
      for ( i = 0 ; i < num_new_atts ; i++ )
        ds_delete_attribute(ds1,ds_num_atts(ds1)-1);
    }

    free_ivec(rows1);
    free_ivec(rows2);
  }
}

ivec *mk_attnums_from_attnames_for_join(datset *ds,string_array *attnames,
                                        char **r_errmess,char *ds_describe)
{
  ivec *attnums = mk_sorted_attnums_from_string_array(ds,attnames,r_errmess);
  if ( attnums == NULL )
  {
    char *temp = mk_printf("In the %s dataset: %s",ds_describe,*r_errmess);
    free_string(*r_errmess);
    *r_errmess = temp;
  }
  return attnums;
}

void ds_join_with_names(datset *ds1,datset *ds2,
                        string_array *attnames1,string_array *attnames2,
                        char **r_errmess)
{
  ivec *attnums1 = mk_attnums_from_attnames_for_join(ds1,attnames1,
                                                     r_errmess,"first");
  ivec *attnums2 = (attnums1==NULL) ? NULL :
                       mk_attnums_from_attnames_for_join(ds2,attnames2,
                                                         r_errmess,"second");
  if ( *r_errmess == NULL )
    ds_join(ds1,ds2,attnums1,attnums2,r_errmess);
  if ( attnums1 != NULL ) free_ivec(attnums1);
  if ( attnums2 != NULL ) free_ivec(attnums2);
}

void make_parse_join_keys(char *key,char **r_attname1,char **r_attname2)
{
  int i;
  for ( i = 0 ; key[i] != '\0' && key[i] != '/' ; i++ )
  {
    /* skip */
  }
  if ( key[i] == '/' )
  {
    *r_attname1 = mk_substring(key,0,i);
    *r_attname2 = mk_substring(key,i+1,(int)strlen(key));
  }
  else
  {
    *r_attname1 = mk_copy_string(key);
    *r_attname2 = mk_copy_string(key);
  }
}
    
void ds_join_with_keys(datset *ds1,datset *ds2,
                       string_array *keys,char **r_errmess)
{
  int size = string_array_size(keys);
  string_array *attnames1 = mk_string_array(size);
  string_array *attnames2 = mk_string_array(size);
  int i;
  for ( i = 0 ; i < size ; i++ )
  {
    char *attname1;
    char *attname2;
    char *key = string_array_ref(keys,i);
    make_parse_join_keys(key,&attname1,&attname2);
    string_array_set(attnames1,i,attname1);
    string_array_set(attnames2,i,attname2);
    free_string(attname1);
    free_string(attname2);
  }
  ds_join_with_names(ds1,ds2,attnames1,attnames2,r_errmess);
  free_string_array(attnames1);
  free_string_array(attnames2);
}

string_array *mk_miner_file_exts(void){
  string_array *sa = mk_string_array(0);
  /*First the default extension*/
  add_to_string_array(sa,"fds");
  /*Now all the extension types*/
  add_to_string_array(sa,"Fast Dataset Files (*.FDS)");
  add_to_string_array(sa,"*.fds");
  add_to_string_array(sa,"Comma-Separated Value Files (*.CSV)");
  add_to_string_array(sa,"*.csv");
  add_to_string_array(sa,"All Files (*.*)");
  add_to_string_array(sa,"*.*");
  return sa;
}

void run_join_command(datset *ds,command *c)
{
  int num_args = string_array_size(c->args);
  int keyarg = int_max(find_index_in_string_array(c->args,"key"),
                       find_index_in_string_array(c->args,"keys"));
  if( num_args < 3 || keyarg < 1 || keyarg >= num_args-1 ){
    string_array *newc = mk_string_array_1("join");
    string_array *params = mk_miner_file_exts();
    void *file = NULL, *keycodes = NULL;
    rebuild_command_from_string_array(c,newc);
    free_string_array(newc);
    file = mk_arg_from_command(c,1,NULL,ARG_FILE_OPEN,0,0,0,params,1);
    free_string_array(params);
    if(file) 
    {
      newc = mk_copy_string_array(c->line);
      add_to_string_array(newc,"keys");
      rebuild_command_from_string_array(c,newc);
      free_string_array(newc);
      keycodes = mk_arg_from_command(c,3,"keycodes",ARG_STRING_LIST,0,0,0,NULL,1);
      if(keycodes)
      {
        num_args = string_array_size(c->args);
        keyarg = 1;
      }
    }
    free_arg(file,ARG_FILE_OPEN);
    free_arg(keycodes,ARG_STRING_LIST);
  }
  if ( ! (num_args < 3 || keyarg < 1 || keyarg >= num_args-1 ) )
  {
    string_array *keys = mk_string_array(0);
    int i;
    for ( i = keyarg+1 ; i < num_args ; i++ )
      add_to_string_array(keys,string_array_ref(c->args,i));
    for ( i = 0 ; i < keyarg ; i++ )
    {
      char *filename = string_array_ref(c->args,i);
      char *errmess = NULL;
      datset *ds2 = ds_load_with_options(filename, /*argc=*/ 0, /*argv=*/ NULL,
                                         &errmess);
      if ( ds2 == NULL )
        my_warning(errmess);
      else
      {
        ds_join_with_keys(ds,ds2,keys,&errmess);
        if ( errmess != NULL ){
          char *buff = mk_printf("Error joining current dataset (first dataset) with second dataset\n"
                                  "file %s:\n%s\n",filename,errmess);
          my_warning(buff);
          free_string(buff);
        }
      }
      if ( errmess != NULL ) free_string(errmess);
      if ( ds2 != NULL ) free_datset(ds2);
    }
    free_string_array(keys);
  }
}

void ivec_swap_elements(ivec *iv,int i,int j)
{
  if ( i != j )
  {
    int temp = ivec_ref(iv,i);
    ivec_set(iv,i,ivec_ref(iv,j));
    ivec_set(iv,j,temp);
  }
}

bool is_a_permute(ivec *permute)
{
  int size = ivec_size(permute);
  bool result = ivec_min(permute) == 0 && ivec_max(permute) == size-1;
  if ( result )
  {
    ivec *anti_permute = mk_constant_ivec(size,-1);
    int i;
    for ( i = 0 ; i < size ; i++ )
      ivec_set(anti_permute,ivec_ref(permute,i),i);
    result = ivec_min(anti_permute) == 0;
    free_ivec(anti_permute);
  }
  return result;
}

void check_is_a_permute(ivec *permute)
{
  if ( !is_a_permute(permute) )
    my_error("ivec is not a permute");
}

ivec *mk_anti_permute(const ivec *permute)
{
  int size = ivec_size(permute);
  ivec *anti_permute = mk_ivec(size);
  int i;

  for ( i = 0 ; i < size ; i++ )
    ivec_set(anti_permute,ivec_ref(permute,i),i);
  return anti_permute;
}

/* Re-orders the rows of the datset according to permute.
   Afterwards, the i'th row of the dataset will contain
   the permute[i]'th row of the dataset on entry.

  PRE: permute must be a permutation (i.e. some ordering
  of the integers 0, 1, 2 , .. ds_num_rows(ds)-1 ) */
void ds_permute(datset *ds,ivec *permute)
{
  ivec *p = mk_copy_ivec(permute);
  int i;
  int num_rows = ds_num_rows(ds);
  ivec *anti_permute = mk_anti_permute(p);

#ifndef AMFAST
  if ( ds_num_rows(ds) != ivec_size(p) )
    my_error("pemute size is different from number of rows");
  check_is_a_permute(p);
  check_is_a_permute(anti_permute);
#endif

  i = 0;
  while ( i < num_rows )
  {
    int next = ivec_ref(p,i);
    if ( next == i )
      i += 1;
    else if ( next < i )
      my_error("Can't happen");
    else
    {
      int j = ivec_ref(anti_permute,i);
      ds_swap_rows(ds,i,next);
      /* Let j be such that p[j] = i
	 We now want p[j] to be next.
	 And for neatness, lets say p[i] = i.

	 We will also want to set
	 anti_permute[i] == i
	 and 
	 anti_permute[next] == j
      */
      ivec_set(p,i,i);
      ivec_set(anti_permute,i,i);
      ivec_set(p,j,next);
      ivec_set(anti_permute,next,j);
    }
  }

#ifndef AMFAST
  check_is_a_permute(p);
  check_is_a_permute(anti_permute);

  if ( !is_sivec(p) || !is_sivec(anti_permute) )
    my_error("bad permute end");
#endif
  free_ivec(anti_permute);
  free_ivec(p);
}

datset *Being_sorted = NULL;
ivec *Sort_attnums = NULL;

int ds_value_compare(datset *ds,int row1,int row2,int attnum)
{
  int result;
  int missing_1 = ds_value_is_missing(ds,row1,attnum);
  int missing_2 = ds_value_is_missing(ds,row2,attnum);

  if ( missing_1 )
    result = (missing_2) ? 0 : 1;
  else if ( missing_2 )
    result = -1;
  else if ( ds_attnum_to_symbolicp(ds,attnum) )
  {
    int v1 = ds_symbol_ref(ds,row1,attnum);
    int v2 = ds_symbol_ref(ds,row2,attnum);
    result = (v1 < v2) ? -1 : (v1 > v2) ? 1 : 0;
  }
  else
  {
    double v1 = ds_real_ref(ds,row1,attnum);
    double v2 = ds_real_ref(ds,row2,attnum);
    result = (v1 < v2) ? -1 : (v1 > v2) ? 1 : 0;
  }
  return result;
}

bool ds_sort_row_compare(datset *ds,int row1,int row2,ivec *attnums)
{
  bool result = 0;
  int i;
  for ( i = 0 ; result == 0 && i < ivec_size(attnums) ; i++ )
    result = ds_value_compare(ds,row1,row2,ivec_ref(attnums,i));
  if ( result == 0 )
    result = (row1 < row2) ? -1 : (row1 > row2) ? 1 : 0;
  return result;
}

/* see __cdecl note at int_comp */
#ifdef WIN32
int __cdecl compare_ds_rows(const void *i_ptr, const void *j_ptr)
#else
int compare_ds_rows(const void *i_ptr, const void *j_ptr)
#endif
{
  int *ip = (int *) i_ptr;
  int i = *ip;
  int *jp = (int *) j_ptr;
  int j = *jp;
  return ds_sort_row_compare(Being_sorted,i,j,Sort_attnums);
}

/* Returns permute such that if ds was permuted according to
   permute it would be in lexicographic sorted order where the
   primary lexicographic key is attnums[0], 2ndary is
   attnums[1] etc. 

   Rows which are equal according to this lexicography remain
   in the same relative order. */
ivec *mk_sorted_datset_permute(datset *ds,ivec *attnums)
{
  int i;
  int size = ds_num_rows(ds);

  int *indices = AM_MALLOC_ARRAY(int, size);
  ivec *permute;

  for (i=0; i<size; i++) 
    indices[i] = i;

  Being_sorted = ds;
  Sort_attnums = attnums;

  qsort((char *)indices, size, sizeof(int), compare_ds_rows);

  Being_sorted = NULL;
  Sort_attnums = NULL;

  permute = mk_ivec_from_iarr(indices,size);

  AM_FREE_ARRAY(indices, int, size);

  return permute;
}

/* Sorts ds in lexicographic order
   primary lexicographic key is attnums[0], 2ndary is
   attnums[1] etc. 

   Rows which are equal according to this lexicography remain
   in the same relative order. */
void ds_sort(datset *ds,ivec *attnums)
{
  ivec *permute = NULL;

  if (ds_num_rows(ds) > 0) /* some permute operation chokes on size 0 */
  {
    permute = mk_sorted_datset_permute(ds,attnums);
    ds_permute(ds,permute);
    free_ivec(permute);
  }
}

bool ds_values_are_same(datset *ds,int row1,int row2,int attnum)
{
  bool result;
  if ( ds_value_is_missing(ds,row1,attnum) )
    result = ds_value_is_missing(ds,row2,attnum);
  else if ( ds_value_is_missing(ds,row2,attnum) )
    result = FALSE;
  else if ( ds_attnum_to_symbolicp(ds,attnum) )
    result = ds_symbol_ref(ds,row1,attnum) == ds_symbol_ref(ds,row2,attnum);
  else
    result = ds_real_ref(ds,row1,attnum) == ds_real_ref(ds,row2,attnum);
  return result;
}

bool rows_are_identical_in_attributes(datset *ds,ivec *attnums,
				      int row1,int row2)
{
  bool result = TRUE;
  int i;
  for ( i = 0 ; result && i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    result = ds_values_are_same(ds,row1,row2,attnum);
  }
  return result;
}

/* Deletes all the records mentioned in "rows", thus reducing
   the dataset size by ivec_size(rows).

    Note that the relative ordering of the remaining records is
    unchanged.
*/
/* rows may be null meaning "delete all records" */
/* changed to eliminate expensive permuting -- 03-19-01 JGS */
void ds_delete_rows(datset *ds,ivec *rows)
{
  int num_rows = ds_num_rows(ds);
  ivec *all_rows = mk_identity_ivec(num_rows);
  ivec *die = (rows==NULL) ? mk_copy_ivec(all_rows) :
                             mk_sivec_from_ivec(rows);
  ivec *live = mk_sivec_difference(all_rows,die);
  /*
  ivec *append = mk_append_ivecs(live,die);
  ds_permute(ds,append);
  */
  ds_keep_subset_rows(ds,live);
  /*  ds_delete_last_n_rows(ds,ivec_size(die)); */
  free_ivec(all_rows);
  free_ivec(live);
  free_ivec(die);
  /*  free_ivec(append); */
}

/* Consider the k'th record in the dataset. If there is some i such
   that
     i < k
   and
     forall j = 0 , 1 , .. n-1
        ds[i,attnums[j]] == ds[k,attnums[j]]

   (in English, if there is some earlier record that matches the j'th
    record in each of the attributes mentioned in attnums....)

    ....then the k'th record is removed. 

    This process occurs for all k. Thus at the end, there will
    be no duplications in combined values for attnums[0]...attnums[S-1]
    between any pair of records.

    Note that the relative ordering of the remaining records is
    unchanged.
*/
void ds_remove_duplicates(datset *ds,ivec *attnums)
{
  ivec *permute = mk_sorted_datset_permute(ds,attnums);
  ivec *die = mk_ivec(0);
  int i;
  int num_rows = ds_num_rows(ds);
  for ( i = 0 ; i < num_rows-1 ; i++ )
  {
    int row = ivec_ref(permute,i);
    int next_row = ivec_ref(permute,i+1);
    if ( rows_are_identical_in_attributes(ds,attnums,row,next_row) )
      add_to_ivec(die,next_row);
  }
  ds_delete_rows(ds,die);
  free_ivec(permute);
  free_ivec(die);
}

/* Increases the length of ds1 by ading in copies of the entire
   contents of ds2. 

   ds2 is almost unchanged. (some of the attributes might be compatibalized
   with ds1).

   If succeeds, sets *r_errmess to NULL.

   If fails, sets *r_errmess to a null terminated string (which must be
   free_string()'d) explaining why it couldn't concatenate.

   It will fail if the datsets have different shapes.

   If rename_ds2_atts is FALSE it will fail if the i'th attribute of 
   ds2 has a different name than the i'th attribute of ds1.

   If rename_ds2_atts is TRUE it will succeed if they have different
   shapes provided the attributes can be compatibalized.
 
   (see ds_compat documentation to see what it means for two attributes
   to be compatibalized).
*/
void ds_concatenate(datset *ds1,datset *ds2,
		    bool rename_ds2_atts,char **r_errmess)
{
  ds_compat(ds1,ds2,rename_ds2_atts,r_errmess);
  if ( *r_errmess == NULL )
  {
    int num_rows1 = ds_num_rows(ds1);
    int num_rows2 = ds_num_rows(ds2);
    ivec *source_rows = mk_identity_ivec(num_rows2);
    ivec *dest_rows = mk_sequence_ivec(num_rows1,num_rows1+num_rows2);
    ds_add_rows(ds1,num_rows2);
    ds_copy_rows(ds2,source_rows,ds1,dest_rows);
    free_ivec(source_rows);
    free_ivec(dest_rows);
  }
}

/* Just the same as ds_concatenate except my_error()'s if there's a problem.
   This does rename attributes if necessary to ensure compatibility. */
void ds_concatenate_simple(datset *ds1,datset *ds2)
{
  char *errmess = NULL;
  ds_concatenate(ds1,ds2, /*rename_ds2_atts=*/TRUE,&errmess);
  if ( errmess != NULL )
    my_error(mk_printf("ds_concatenate_simple: %s\n",errmess));
}

void run_concatenate_command(datset *ds,command *c)
{
  string_array *sa = mk_copy_string_array(c->args);
  bool my_rename = find_and_remove(sa,"rename");
  int num_args = string_array_size(sa);
  if ( num_args < 1 ) 
  {
    string_array *params = mk_miner_file_exts();
    string_array *newc = mk_string_array_1("concatenate");
    char *file = NULL;
    rebuild_command_from_string_array(c,newc);
    file = (char *)mk_arg_from_command(c,1,NULL,ARG_FILE_OPEN,0,0,0,params,1);
    free_string_array(params);
    free_string_array(newc);

    if(file)
    {
      bool *p_rename;
      free_string_array(sa);
      sa = mk_copy_string_array(c->args);
      p_rename = (my_rename)? &my_rename:(bool *)mk_arg_from_command(c,2,"rename",ARG_BOOL,0,0,FALSE,NULL,1);
      find_and_remove(sa,"rename");
      num_args = string_array_size(sa);
      if(p_rename) my_rename = *p_rename;
      free_arg(p_rename,ARG_BOOL);
    }
    free_arg(file,ARG_FILE_OPEN);
  }
  
  if(num_args>=1)
  {
    int i;
    for ( i = 0 ; i < num_args ; i++ )
    {
      char *filename = string_array_ref(sa,i);
      char *errmess = NULL;
      datset *ds2 = ds_load_with_options(filename, /*argc=*/0, /*argv=*/NULL,
                                         &errmess);
      if ( ds2 == NULL )
        my_warning(errmess);
      else
      {
        ds_concatenate(ds,ds2,my_rename,&errmess);
        if ( errmess != NULL ){
          char *buff = mk_printf("Error concatenating current dataset (first dataset)\n"
		                             "with second dataset from file %s:\n%s\n",filename,errmess);
          my_warning(buff);
          free_string(buff);
        }
      }
      if ( errmess != NULL ) free_string(errmess);
      if ( ds2 != NULL ) free_datset(ds2);
    }
  }
  free_string_array(sa);
}

/* Updates ds by adding from "filename". Alters ds's shape if necessary
   so that it's compatted with filename. If impossible to concat, does 
   nothing, but sets *r_errmess to be an AM_MALLOCKED string (that
   must be freed with free_string). If no problem, sets *r_errmess
   to be NULL */
void ds_concatenate_from_file(datset *ds,const char *filename,char **r_errmess)
{
  int argc = 0;
  char **argv = NULL;
  datset *new_ds = ds_load_with_options(filename,argc,argv,r_errmess);

  if(*r_errmess == NULL)
    ds_compat(ds, new_ds, 1, r_errmess);

  if(*r_errmess == NULL)
    ds_concatenate(ds, new_ds, 1, r_errmess);

  if ( new_ds != NULL ) free_datset(new_ds);
}

/* Loads from filenames[index], then adds a new attribute called
   file, which has symbolic values consisting of all the strings in
   filenames, but in which all the actual values of file in this
   datasets are set to filenames[i] */
datset *ds_load_adding_filenames_attribute(string_array *filenames,int idx,
					   char **r_errmess)
{
  char *fname = string_array_ref(filenames,idx);
  datset *ds = ds_load_with_options(fname,0,NULL,r_errmess);
  if ( ds != NULL )
  {
    char *attname = "file";
    int attnum;
    int value;
    int row;
    ds_add_symbolic_attribute(ds,attname,filenames);
    attnum = ds_name_to_attnum(ds,attname);
    value = ds_attnum_name_to_value(ds,attnum,fname);
    /* Notes that in general value != index because value names
       will have been sorted */
    if ( attnum < 0 || value < 0 ) my_error("Can't happen");
    for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
      ds_symbol_set(ds,row,attnum,value);
  }
  return ds;
}

/* Sets *r_errmess to NULL if no problem.

   Returns NULL and sets *r_errmess to a string that
   must eventually be free_string'd if there's a problem. */
datset *mk_datset_from_filenames(string_array *filenames,char **r_errmess)
{
  datset *ds = NULL;

  if(string_array_size(filenames) == 0)
  {
    *r_errmess = mk_printf("You must give me at least one file for loading");
  }
  else
  {
    string_array *sorted_filenames = mk_sort_string_array(filenames);
    int i;
    printf("Trying to load the following file(s):\n");
    fprintf_string_array(stdout, "", sorted_filenames,"\n");

    ds = ds_load_adding_filenames_attribute(sorted_filenames,0,r_errmess);

    for ( i = 1 ; *r_errmess == NULL && i < string_array_size(sorted_filenames); i++ )
    {
      datset *newds = 
	ds_load_adding_filenames_attribute(sorted_filenames,i,r_errmess);
      if ( *r_errmess == NULL )
      {
	ds_concatenate(ds,newds,TRUE,r_errmess);
	free_datset(newds);
      }
      else
      {
	char *s = mk_printf("This problem happened when concatenating file\n"
			    "number %d (called %s):\n%s\n",
			    i+1,string_array_ref(sorted_filenames,i),*r_errmess);
	free_string(*r_errmess);
	*r_errmess = s;

	if ( newds != NULL )
	  my_error("newds should be NULL");
      }
    }

    if ( *r_errmess != NULL && ds != NULL )
    {
      free_datset(ds);
      ds = NULL;
    }
    free_string_array(sorted_filenames);
  }

  return ds;
}


/* Returns an attnum or else -1 denoting "no attribute".

   Default may be -1 denoting "default is none"

   options is a string:

      if contains 'n' allows the user to state "none"
      if contains 'r' allows the user to select real attribute
      if contains 's' allows the user to select symbol attribute
      if contains 'm" allows the user to select attribute with missing values
    If on entry, *okay is FALSE, this function does nothing, and returns -77

    If user cancels, or not possible satisfy constraints, set *okay to FALSE,
    prints an error message and returns -77.
*/
int attnum_from_command(command *c,datset *ds,char *key,char *options,
			int default_attnum,bool *okay)
{
  bool none_ok = char_is_in(options,'n');
  bool symbol_ok = char_is_in(options,'s');
  bool real_ok = char_is_in(options,'r');

  int attnum = 7777;
  if ( *okay )
  {
    default_attnum = safe_default_attnum(ds,options,default_attnum,okay);
    if ( !(*okay) )
      printf("I cannot run this command because there is no\n"
             "possible legal choice of the \"%s\" attribute.\n",key);
  }
  if ( *okay )
  {
    char *default_string = (default_attnum < 0) ? "none" : 
                           ds_attnum_to_name(ds,default_attnum);
    *okay = FALSE;

    while ( !(*okay) )
    {
      char *message = 
        mk_printf("Enter %s attribute %sfor %s",
                  (symbol_ok && real_ok) ? "an" :
                  (symbol_ok) ? "a symbolic" : "a real",
                  (none_ok) ? "(or \"none\") " : "",key); 
      char *string = mk_string_from_command_with_message(c,key,message,default_string);

      attnum = ds_name_to_attnum(ds,string);
      if ( attnum < 0 && !caseless_eq_string(string,"none") )
        printf("There is no such attribute as %s\n",string);
      else
      {
        char *errmess = mk_attnum_choice_errmess(ds,options,attnum);
        
        if ( errmess == NULL )
          *okay = TRUE;
        else
        {
          printf("Sorry. I can't accept %s:\n%s\n",string,errmess);
          free_string(errmess);
        }
      }
      free_string(string);
      free_string(message);
    }
  }

  return attnum;
}

datset *mk_sheared_datset(datset *base,int row_attnum,int col_attnum,int cell_attnum)
{
  datset *shear = mk_empty_datset();
  int shear_num_rows = ds_attnum_to_arity(base,row_attnum);
  int base_num_rows = ds_num_rows(base);
  int shear_num_cols = ds_attnum_to_arity(base,col_attnum);
  int i,j;
  string_array *row_names = mk_value_names_from_attnum(base,row_attnum);

  ds_add_symbolic_attribute(shear,ds_attnum_to_name(base,row_attnum),row_names);

  for ( i = 0 ; i < shear_num_cols ; i++ )
    ds_add_real_attribute(shear,ds_attnum_value_to_name(base,col_attnum,i));
   
  ds_add_rows(shear,shear_num_rows);

  for ( i = 0 ; i < shear_num_cols ; i++ )
    for ( j = 0 ; j < shear_num_rows ; j++ )
      ds_set_value_missing(shear,j,i+1);

  for ( j = 0 ; j < shear_num_rows ; j++ )
    ds_symbol_set(shear,j,0,j);

  for ( i = 0 ; i < base_num_rows ; i++ )
  {
    int shear_row = ds_symbol_ref(base,i,row_attnum);
    int shear_col = ds_symbol_ref(base,i,col_attnum);
    double cell_value = ds_real_ref(base,i,cell_attnum);

    ds_real_set(shear,shear_row,shear_col+1,cell_value);
  }

  free_string_array(row_names);

  return shear;
}


#ifdef AUTON_UNUSED_CODE
ivec *mk_missing_rows(datset *ds)
{
  ivec *rows = mk_ivec(0);
  int i;
  for ( i = 0 ; i < ds_num_rows(ds) ; i++ )
  {
    bool missing = ds_row_to_num_missing(ds,i) > 0;
    if ( missing )
      add_to_ivec(rows,i);
  }
  return rows;
}
#endif /* AUTON_UNUSED_CODE */

int missingest_attnum(datset *ds,double *r_frac)
{
  int result = -1;
  int attnum;

  *r_frac = -77.7;

  for ( attnum = 0 ; attnum < ds_num_atts(ds) ; attnum++ )
  {
    double n = ds_attnum_to_num_missing(ds,attnum);
    double frac = n / int_max(1,ds_num_rows(ds));
    if ( result < 0 || frac > *r_frac )
    {
      result = attnum;
      *r_frac = frac;
    }
  }

  return result;
}

int missingest_row(datset *ds,double *r_frac)
{
  int result = -1;
  int row;

  *r_frac = -77.7;

  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    double n = ds_row_to_num_missing(ds,row);
    double frac = n / int_max(1,ds_num_atts(ds));
    if ( result < 0 || frac > *r_frac )
    {
      result = row;
      *r_frac = frac;
    }
  }

  return result;
}

void ds_delete_row(datset *ds,int row)
{
  ivec *rows = mk_ivec_1(row);
  ds_delete_rows(ds,rows);
  free_ivec(rows);
}

void ds_remove_missing_rows_and_atts(datset *ds)
{
  while ( ds_has_missing_values(ds) )
  {
    double attnum_frac;
    int attnum = missingest_attnum(ds,&attnum_frac);
    double row_frac;
    int row = missingest_row(ds,&row_frac);

    if ( attnum_frac >= row_frac )
    {
      printf("Deleting attribute %s (%g%% missing)\n",
             ds_attnum_to_name(ds,attnum),attnum_frac*100);
      ds_delete_attribute(ds,attnum);
    }
    else
    {
      char *string = mk_row_attnum_to_name(ds,row,0);
      printf("Deleting row %s=%s (%g%% missing)\n",
             ds_attnum_to_name(ds,0),string,attnum_frac*100);
      ds_delete_row(ds,row);
      free_string(string);
    }
  }
}

datset *mk_sheared_from_command(command *c,datset *ds)
{
  bool okay = TRUE;
  int row_attnum = attnum_from_command(c,ds,"rowatt","s",0,&okay);
  int col_attnum = attnum_from_command(c,ds,"colatt","s",0,&okay);
  int cell_attnum = attnum_from_command(c,ds,"cellatt","r",0,&okay);
  datset *result = (okay) ? mk_sheared_datset(ds,row_attnum,col_attnum,cell_attnum) :
                            NULL;

  return result;
}

char *mk_cartesian_name(datset *ds,ivec *attnums,ivec *values)
{
  char *name = mk_copy_string("");
  int i;
  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    int value = ivec_ref(values,i);
    char *valname = ds_attnum_value_to_name(ds,attnum,value);
    char *newname = (i==0) ?
                    mk_copy_string(valname) :
                    mk_printf("%s-%s",name,valname);
    free_string(name);
    name = newname;
  }
  return name;
}

void add_to_cart_names(datset *ds,ivec *attnums,ivec *values,string_array *cart_names)
{
  if ( ivec_size(values) == ivec_size(attnums) )
  {
    char *name = mk_cartesian_name(ds,attnums,values);
    add_to_string_array(cart_names,name);
    free_string(name);
  }
  else
  {
    int attnum = ivec_ref(attnums,ivec_size(values));
    int value;
    for ( value = 0 ; value < ds_attnum_to_arity(ds,attnum) ; value++ )
    {
      add_to_ivec(values,value);
      add_to_cart_names(ds,attnums,values,cart_names);
      ivec_remove_last_element(values);
    }
  }
}

#ifdef AUTON_UNUSED_CODE
string_array *mk_cartesian_names_combinatorially(datset *ds,ivec *attnums)
{
  string_array *cart_names = mk_string_array(0);
  ivec *values = mk_ivec(0);
  add_to_cart_names(ds,attnums,values,cart_names);
  free_ivec(values);
  sort_string_array(cart_names);
  return cart_names;
}
#endif /* AUTON_UNUSED_CODE */

ivec *mk_unused_values(datset *ds,int attnum)
{
  ivec *result = mk_ivec(0);
  ivec *freqs = mk_ds_frequencies(ds,attnum);
  int i;
  for ( i = 0 ; i < ivec_size(freqs) ; i++ )
  {
    if ( ivec_ref(freqs,i) == 0 )
      add_to_ivec(result,i);
  }
  free_ivec(freqs);
  return result;
}

void delete_values(datset *ds,int attnum,ivec *values)
{
  int i;
  for ( i = ivec_size(values)-1 ; i >= 0 ; i-- )
  {
    int value = ivec_ref(values,i);
    ds_delete_value(ds,attnum,value);
  }
}

void ds_remove_unused_values(datset *ds,int attnum)
{
  ivec *unused_values = mk_unused_values(ds,attnum);
  delete_values(ds,attnum,unused_values);
  free_ivec(unused_values);
}

/* Makes an ivec is same size at attnums.

   result[i] = ds_symbol_ref(ds,row,attnums[i])

   result is NULL if any of the above elements were missing.

   PRE: all attnums must be symbol_ref_ok */
ivec *mk_ds_values_ivec(datset *ds,int row,ivec *attnums)
{
  ivec *result = NULL;
  bool none_missing = TRUE;
  int i;

  for ( i = 0 ; i < ivec_size(attnums) ; i++ )
  {
    int attnum = ivec_ref(attnums,i);
    bool missing = ds_value_is_missing(ds,row,attnum);
    if ( missing )
      none_missing = FALSE;
  }

  if ( none_missing )
  {
    result = mk_ivec(ivec_size(attnums));
    for ( i = 0 ; i < ivec_size(attnums) ; i++ )
      ivec_set(result,i,ds_symbol_ref(ds,row,ivec_ref(attnums,i)));
  }
  return result;
}

void ds_add_cartesian_attribute(datset *ds,char *name,ivec *attnums)
{
  stringhash *sh = mk_empty_stringhash();
  string_array *cart_names = NULL;
  int row;
  int new_attnum;

  if ( ivec_size(attnums) < 2 )
    my_error("Must have at least two attnums!");

  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    ivec *values = mk_ds_values_ivec(ds,row,attnums);
    if ( values != NULL )
    {
      char *valname = mk_cartesian_name(ds,attnums,values);
      add_to_stringhash(sh,valname); /* Note, this will do
                                        nothing if valname already in sh */
      free_string(valname);
      free_ivec(values);
    }
  }
  cart_names = mk_sorted_string_array_from_stringhash(sh);
  ds_add_symbolic_attribute(ds,name,cart_names);
  new_attnum = ds_name_to_attnum(ds,name);

  for ( row = 0 ; row < ds_num_rows(ds) ; row++ )
  {
    ivec *values = mk_ds_values_ivec(ds,row,attnums);
    if ( values == NULL )
      ds_set_value_missing(ds,row,new_attnum);
    else
    {
      int value;
      char *valname = mk_cartesian_name(ds,attnums,values);
      value = ds_attnum_name_to_value(ds,new_attnum,valname);

      if ( value < 0 )
        my_error("We should have got this name on the first pass");

      ds_symbol_set(ds,row,new_attnum,value);
      free_string(valname);
      free_ivec(values);
    }
  }

  free_string_array(cart_names);
  free_stringhash(sh);
}

ivec *mk_attnums_from_command(command *c,datset *ds,int start_arg,char *options)
{
  ivec *attnums = mk_ivec(0);
  bool ok = TRUE;
  int i;
  for ( i = start_arg ; ok && i < command_num_tokens(c) ; i++ )
  {
    char *arg = command_arg_ref(c,i);
    int attnum = ds_name_to_attnum(ds,arg);
    if ( attnum < 0 )
    {
      ok = FALSE;
      printf("There is no such attribute as %s\n",arg);
    }
    else
    {
      char *errmess = mk_attnum_choice_errmess(ds,options,attnum);
      if ( errmess == FALSE )
        add_to_ivec(attnums,attnum);
      else
      {
        printf("%s\n",errmess);
        free_string(errmess);
        ok = FALSE;
      }
    }
  }
  if ( !ok )
  {
    free_ivec(attnums);
    attnums = NULL;
  }
  return attnums;
}

/* Returns TRUE if changes datset */
bool run_add_cartesian_attribute(command *c,datset *ds)
{
  bool result = FALSE;

  if ( arg_exists_ok(c,2) )
  {
    char *name = c->a1;
    if ( ds_name_to_attnum(ds,name) >= 0 )
      printf("Usage: cartesian <new_name> <attnums>: but the first\n"
	     "token on the your command line is a pre-existing\n"
	     "attribute name (%s). I will do nothing.\n",name);
    else
    {
      ivec *attnums = mk_attnums_from_command(c,ds,2,"s");
      if ( attnums != NULL )
      {
	if ( ivec_size(attnums) < 2 )
	  printf("Usage: cartesian <new_name> <attnums>: and you must\n"
		 "give me at least two attnums.\n");
	else
	{
	  ds_add_cartesian_attribute(ds,name,attnums);
	  result = TRUE;
	}
	free_ivec(attnums);
      }
      else
	result = FALSE;
    }
  }
  return result;
}

int ds_num_real_atts(datset *ds)
{
  int result = 0;
  int i;
  for ( i = 0 ; i < ds_num_atts(ds) ; i++ )
  {
    if ( ds_real_ref_ok(ds,i) )
      result += 1;
  }
  return result;
}

/* naming_attnum must be symbolic or else can be -1
   meaning "just invent names" */
datset *mk_datset_transpose(datset *start,int naming_attnum)
{
  datset *end = mk_empty_datset();
  int start_row;
  int start_attnum;
  int start_num_rows = ds_num_rows(start);
  int start_num_atts = ds_num_atts(start);
  int end_num_rows = ds_num_real_atts(start);

  ds_add_rows(end,end_num_rows);

  for ( start_row = 0 ; start_row < start_num_rows ; start_row++ )
  {
    int end_attnum = start_row;
    char *name = (naming_attnum < 0) ?
                 mk_printf("x%d",ds_num_atts(end)) :
                 mk_row_attnum_to_name(start,start_row,naming_attnum);
    int end_row = 0;

    ds_add_real_attribute(end,name);

    for ( start_attnum = 0 ; start_attnum < start_num_atts ; start_attnum++ )
    {
      if ( ds_real_ref_ok(start,start_attnum) )
      {
        if ( ds_value_is_missing(start,start_row,start_attnum) )
          ds_set_value_missing(end,end_row,end_attnum);
        else
        {
          double value = ds_real_ref(start,start_row,start_attnum);
          ds_real_set(end,end_row,end_attnum,value);
        }
        end_row += 1;
      }
    }

    if ( end_row != ds_num_rows(end) ) my_error("{asdlkjnvcka");

    free_string(name);
  }

  return end;
}

/* Returns NULL if there's a problem */
datset *mk_run_transpose(command *c,datset *ds)
{
  datset *transpose = NULL;
  bool okay = TRUE;
  char *arg = command_arg_ref(c,1);
  int naming_attnum = -1;

  if (arg) naming_attnum = ds_name_to_attnum(ds,arg);

  if (naming_attnum < 0)
    naming_attnum = attnum_from_command(c,ds,"naming_attribute","sn",0,&okay);

  if ( okay )
  {
    transpose = mk_datset_transpose(ds,naming_attnum);
    printf("New dataset has %d row(s) and %d attribute(s)\n",
           ds_num_rows(transpose),ds_num_atts(transpose));
  }

  return transpose;
}



