/* *
   File:        compat.c
   Author:      Merle Romesberg
   Created:     1999
   Description: Enables compatible files to be used together.
                                        ie training / validation sets with the same value numbering system
   Copyright 1999, SPR
*/

#include "dsut.h"
#include "compat.h"

#define MAX_STR_SIZE 1000


void compat_main(int argc,char *argv[]);

/* Private prototypes */
static string_array *mk_possible_value_names(datset *train,int train_attnum, datset *test,int test_attnum);
static string_array *mk_row_to_valname(datset *ds, int attnum);
static void fill_in_values(datset *ds, int attnum, string_array *row_to_valname);
static void ds_save_loudly(char *fname,datset *ds);


/*construct a sorted array of all the value_names for attnum 
  occuring in either train or test*/
static string_array *mk_possible_value_names(datset *train,int train_attnum,
				      datset *test,int test_attnum)
{
  int trainmax = ds_attnum_to_arity(train, train_attnum);
  int testmax = ds_attnum_to_arity(test, test_attnum);
  int traini = 0;
  int testi = 0;
  string_array *possible_value_names = mk_string_array(0);
  
  while( (traini < trainmax) && (testi < testmax) )
  {
    char *trainname =strdup(ds_attnum_value_to_name(train, train_attnum, traini));
    char *testname =strdup(ds_attnum_value_to_name(test, test_attnum, testi)); 
          
    int lex = strcmp(trainname, testname);
    if(lex == 0)  /*they both have the same name*/
    {
      traini++; testi++;
      add_to_string_array(possible_value_names,trainname);
    }
    else  if(lex<0) /*trainname < testname lexicographically*/
    {
      traini++;
      add_to_string_array(possible_value_names,trainname);
    }
    else /*trainname > testname lexicographically*/
    {
      testi++;
      add_to_string_array(possible_value_names,testname);
    }
  }
  while( testi < testmax )
  {
    char *testname =strdup(ds_attnum_value_to_name(test, test_attnum, testi)); 
    add_to_string_array(possible_value_names,testname);
    testi++;
  }
  while(traini < trainmax)
  {
    char *trainname =strdup(ds_attnum_value_to_name(train, train_attnum, traini)); 
    add_to_string_array(possible_value_names,trainname);
    traini++;
  }
  return possible_value_names;
}

/*mk a string_array with the names of values of attnum for each row in 
  ds in order.*/ 
static string_array *mk_row_to_valname(datset *ds, int attnum)
{
    int R = ds_num_rows(ds);
    string_array *row_to_valname = mk_string_array(R);
    int r;

    for(r=0; r<R; r++)
    {
        char* value_name = NULL;
        if ( !ds_value_is_missing(ds,r,attnum) )
          value_name = ds_attnum_value_to_name(ds, attnum,          
            ds_symbol_ref(ds, r, attnum));
        string_array_set(row_to_valname,r,value_name);
    }
    return row_to_valname;
}

static void fill_in_values(datset *ds, int attnum, string_array *row_to_valname)
{
  int R = ds_num_rows(ds);
  int r;

  for ( r = 0 ; r < R ; r++ )
  {
    char *value_name = string_array_ref(row_to_valname, r); 
    
    if ( value_name != NULL )
    {
      int val = ds_attnum_name_to_value(ds,attnum,value_name);
      ds_symbol_set(ds,r,attnum, val);
    }
    else
      ds_set_value_missing(ds,r,attnum);
  }
}

void ds_new_value_names(datset *ds, int attnum, string_array *value_names)
{
  string_array *row_to_valname = mk_row_to_valname(ds, attnum);
  ds_set_value_names(ds, attnum, value_names);
  fill_in_values(ds, attnum, row_to_valname);
  free_string_array(row_to_valname);
}

/* Two atts have the same shape iff

     They're both real-valued

         or

     They're both symbolic with the same arity and each of their
     values has the same name (in the same order).

   The attributes themselves do NOT need to have the same name
   in order to have the same shape. */
bool ds_attnums_have_same_shape(datset *ds1,int attnum1,
                                datset *ds2,int attnum2)
{
  bool same = TRUE;
  bool symbolic1 = ds_symbol_ref_ok(ds1,attnum1);
  bool symbolic2 = ds_symbol_ref_ok(ds2,attnum2);
  bool real1 = ds_real_ref_ok(ds1,attnum1);
  bool real2 = ds_real_ref_ok(ds2,attnum2);

  if ( same ) same = (symbolic1 == symbolic2);
  if ( same ) same = (real1 == real2);

  if ( same && symbolic1 )
  {
    same = symbolic2;
    if ( same )
    {
      int arity1 = ds_attnum_to_arity(ds1,attnum1);
      int arity2 = ds_attnum_to_arity(ds2,attnum2);
      same = arity1 == arity2;
      if ( same )
      {
        int v;
        for ( v = 0 ; same && v < arity1 ; v++ )
          same = eq_string(ds_attnum_value_to_name(ds1,attnum1,v),
                           ds_attnum_value_to_name(ds2,attnum2,v));
      }
    }
  }

  return same;
}

/* this is like ds_compat_attribute only different.  there no attempt to
   make the attribute names match.  there is no attempt to handle real values.
   it merely makes the symbolic value lists match (presumably so they can be
   joined in some way.
   */
void ds_compat_symbolic_attribute_values(datset *train, int train_attnum, 
					 datset *test, int test_attnum,
					 char **r_errmess)
{
  if (!ds_symbol_ref_ok(train,train_attnum) || 
      !ds_symbol_ref_ok(test,test_attnum))
  {
    add_to_error_message(r_errmess,"ds_compat_symbolic_attribute_values: must be run on symbolic attributes.\n");
  }
  else if (!ds_attnums_have_same_shape(train,train_attnum,test,test_attnum))
  {
    string_array *possnames = mk_possible_value_names(train,train_attnum,
						      test,test_attnum);

    ds_new_value_names(train, train_attnum, possnames);                
    ds_new_value_names(test, test_attnum, possnames);
    free_string_array(possnames);
  }
}

/* This makes the specified attributes in the two datasets have
   the same shape (by the defintion of shape used in
   ds_attnums_have_same_shape() above. It does not rename
   either attribute.  */
void ds_compat_attribute(datset *train, int train_attnum, 
			 datset *test, int test_attnum,
			 bool rename_ok,
			 char **r_errmess)
{
  char *train_name = ds_attnum_to_name(train,train_attnum);
  char *test_name = ds_attnum_to_name(test,test_attnum);

  if ( eq_string(train_name,"PRODROME") )
    my_breakpoint();

  if ( !eq_string(train_name,test_name) && !rename_ok )
    *r_errmess = mk_printf("I can't compatibalize attribute %s from the\n"
			   "first dataset and %s from the second unless they\n"
			   "are renamed",
			   train_name,test_name);
  else if ( ds_attnums_have_same_shape(train,train_attnum,
				       test,test_attnum) )
  {
    /* skip */
  }
  else
  {
    if ( ds_real_ref_ok(train,train_attnum) ||
	 ds_real_ref_ok(test,test_attnum) )
    {
      run_realize(train,train_attnum);
      run_realize(test,test_attnum);
    }

    if ( ds_real_ref_ok(train,train_attnum) && 
	 ds_real_ref_ok(test,test_attnum) )
    {
      ds_set_coltype(train,train_attnum,REAL_COL,UNDEFINED_COL);
      ds_set_coltype(test,test_attnum,REAL_COL,UNDEFINED_COL);
    }
    else if ( ds_symbol_ref_ok(train,train_attnum) && 
	      ds_symbol_ref_ok(test,test_attnum) )
    {
      string_array *possnames = 
      	mk_possible_value_names(train,train_attnum,test,test_attnum);

      ds_set_coltype(train,train_attnum,SYMBOL_COL,UNDEFINED_COL);
      ds_set_coltype(test,test_attnum,SYMBOL_COL,UNDEFINED_COL);

      printf("Compatibalizing %s in the first dataset and %s in the second\n",
	           train_name,test_name);
      ds_new_value_names(train, train_attnum, possnames);                
      ds_new_value_names(test, test_attnum, possnames);
      free_string_array(possnames);
    }
    else if ( ds_real_ref_ok(train,train_attnum) )
    	*r_errmess = mk_printf("attribute %s is real in datset a, "
			                       "but attribute %s is symbolic in datset b\n",
			                       train_name,test_name);
    else
    	*r_errmess = mk_printf("attribute %s is symbolic in datset a, "
			                       "but attribute %s is real in datset b\n",
			                       train_name,test_name);
  }
}

void ds_compat(datset *train, datset *test, bool rename_ok, 
	       char **r_errmess)
{
  int ai;
  int numatts_train = ds_num_atts(train);
  int numatts_test = ds_num_atts(test);  

  *r_errmess = NULL;

  if( numatts_train != numatts_test ) 
  {
    *r_errmess = mk_printf("Different number of attributes in the "
                           "two files! Not compatible\n");
  }

  for(ai=0; *r_errmess == NULL && ai<numatts_train; ai++)
    ds_compat_attribute(train,ai,test,ai,
			rename_ok,r_errmess);
}

static void ds_save_loudly(char *fname,datset *ds)
{
  ds_save(fname,ds);
  printf("Saved compatibalized dataset in %s\n",fname);
}

void compat_main(int argc,char *argv[])
{
  datset *ds1 = mk_datset_from_args("ds1",argc,argv);
  datset *ds2 = mk_datset_from_args("ds2",argc,argv);
  char *errmess;
  ds_compat(ds1,ds2,TRUE,&errmess);

  if ( errmess != NULL )
    printf("Error doing compat:\n%s\n",errmess);
  else
  {
    ds_save_loudly("ds1-saved.csv",ds1);
    ds_save_loudly("ds1-saved.fds",ds1);
    ds_save_loudly("ds2-saved.csv",ds2);
    ds_save_loudly("ds2-saved.fds",ds2);
  }

  if ( errmess != NULL ) free_string(errmess);
  free_datset(ds1);
  free_datset(ds2);
}
