/*
   File:        newdset.h
   Author:      Andrew W. Moore
   Created:     Sat Aug 14 12:49:28 EDT 1999
   Description: Header for SPR/Auton DSET Structure

   Copyright 1999, Schenley Park Research
*/


#ifndef DSET_H
#define DSET_H
#include "shape.h"
#include "column.h"

/* The only important public data structure is a datset. This is
   a memory-sparing representation of a machine learning dataset
   that can have symbolic and real attributes.

   For people interested in the public API, look ahead in this
   file for the section labeled DATSET API.
*/



/* This represents a complete dataset. Information about the names
   shapes and actual values of all the things in the data file. */
typedef struct datset
{
  int num_rows; 
  shape *sh;

  int array_size;
  column **cols_array; /* cols[i] gives the actual values of the
                             ith attribute: 0 <= i < sh->num_atts.
                             The columns all have length "num_rows". */
} datset;


/* DATSET API */

/*
   Here is an example of a dataset represented as a kind of spreadsheet.
   It will provide all the nomenclature we'll need:

     height   weight   gender   marital_status  age  state
    ------------------------------------------------------
      60       150    female      single        43    PA
      65       189     male       married       56    PA
      72       110    female      divorced      27    OH
      52        98    female      married       34    NY
      
   This is a datset with six attributes (each corresponding to one
   column). Each attribute has a name: attribute 0 (the leftmost)
   is named "height". Attribute 1 is named "weight" etc.

   Three of the attributes (height weight age) are numeric, which means
   all their values are numbers.

   The other three are all symbolic, which means they have a discrete
   set of values, each with its own value_name. The number of different
   possible values of an attribute is its arity. "gender" has an arity of
   two. "marital_status" has an arity of three.

   Based on this whirlwind tour, here are the public operations
   on datsets.
*/

/* Frees the datset and all its subcomponents. */
void free_datset(datset *x);

shape *ds_shape(const datset *ds);
bool ds_shape_equal(datset *ds,datset *test_ds);

/************ MK_COPY operations ********/

/* Builds and returns a copy of x with in turn contains copies
   of all of x's subcomponents. */
datset *mk_copy_datset(datset *x);

/* Sends an ASCII (detailed) printout of all of datset's
   substructures to the specified stream. Use stdout if you
   want to send to the console. */
void fprintf_datset(FILE *s,char *m1,datset *x,char *m2);

/* Creates a datset of the given shape in which each value of
   each symbolic attribute is 0 (i.e. the first of the list of
   attribute-value-names for the given columns) and each real-valued
   value is 0.0 */
datset *mk_zero_datset(shape *sh,int num_rows);

/* Makes a datset with no rows and no columns.  Use the add ...
   functions to give it rows and columns. */
datset *mk_empty_datset(void);

/* Adds a new attribute (column) to the datset, of type REAL_COL.
   The initial values in the column are all 0.0 */
void ds_add_real_attribute(datset *ds,char *attname);

/* Makes a new symbolic attribute. The arity of the new attribute is
   string_array_size(value_names), and the name of the i'th value is
   string_array_ref(value_names,i).

   The name of the new attribute is attname.

   The initial values in the column are all 0 (corresponding
   to the first element in value_names).

   PRE: value_names must be in INCREASING lexicographic order. */
void ds_add_symbolic_attribute(datset *ds,char *attname,
                               string_array *value_names);

/* Adds a new attribute called newname which is identical to oldname
   (Copies in values of rows as well)*/
void ds_copy_attribute(datset *ds,char *oldname,char *newname);

/* Use this function to mess with a dataset coltype. 

   It will cause a my_error if you attempt to set major_coltype
   to be UNDEFINED_COL or to the same value as minor_coltype.

   The major_type of the attribute will be what is detected by
   ds_attnum_to_symbolicp(...).

   The minor type, if not undefined, will allow both real and symbolic
   refs to the attribute. 

   This function never changes stored values of stored value names,
   except to delete the real component should the real
   component become undefined or to delete the symbolic component
   should the symbolic component become undefined.

   If a symbolic component is created, it has arity 0 and any
   call to ds_symbol_ref will return zero. Its arity can be changed
   by a call to ds_add_value_name or ds_set_value_names.

   If a real component is created, all its values are 0.0.

   THE CASUAL PROGRAMMER SHOULD NOT USE THIS FUNCTION: TO ADD
   SYMBOLIC/REAL COMPONENTS YOU SHOULD USE ONE OF THE COLUMN TRANSFORM
   OPS LISTED BELOW. THIS RESTRICTION IS IN ORDER TO PREVENT
   SYMBOLIC COMPONENTS THAT DISAGREE WITH REAL COMPONENTS.

   To find out what is the major_coltyp of an attribute, use
   ds_attnum_to_symbolicp...if TRUE, the major_coltype is SYMBOL_COL.
   If FALSE the major_coltype is REAL_COL.

   To find out there's a symbol component somewhere use ds_symbol_ref_ok
   To find out there's a real component somewhere use ds_real_ref_ok
*/
void ds_set_coltype(datset *ds,int attnum,int major_coltype,int minor_coltype);

/* Returns TRUE iff the major or minor mode of the given 
   attribute is SYMBOL_COL. As the name suggests, you may call
   ds_symbol_ref(ds,attnum) if and only if ds_symbol_ref_ok(ds,attnum).
*/
bool ds_symbol_ref_ok(const datset *ds,int attnum);

/* Returns TRUE iff the major or minor mode of the given 
   attribute is REAL_COL. As the name suggests, you may call
   ds_real_ref(ds,attnum) if and only if ds_real_ref_ok(ds,attnum).
*/
bool ds_real_ref_ok(const datset *ds,int attnum);

void check_ds_symbol_ref_ok(const datset *ds,int attnum);

void check_ds_real_ref_ok(datset *ds,int attnum);

/* This adds a new value name to the given attribute. After adding, 
   some of the small integer representations of values may change.

   This function ensures that after calling, all value_names for the
   given attribute remain in sorted increasing lexicographic order.

   PRE: ds_symbol_ref_ok(ds,attnum)

   This function is more efficient if the new value has a larger
   lexicographic value_name than any previous value_names, but despite
   that, if possible use ds_set_value_names instead of this if possible and
   if you don't care about losing the previous values.
*/
void ds_add_value_name(datset *ds,int attnum,char *value_name);

/* This function obliterates any previous values and redefines the
   value names (and implicitly the arity) of the attribute. After this
   call, all symbolic attribute values are 0.

   PRE: ds_symbol_ref_ok(ds,attnum)
*/
void ds_set_value_names(datset *ds,int attnum,string_array *value_names);

void ds_set_attname(datset *ds,int attnum,char *new_attname);

/* Very efficiently changes all the value names of the 
   given (symbolic) attribute. If the i'th value name was
   previously "xxx" it now becomes "<prefix>xxx" */
void ds_valnames_prepend(datset *ds,int attnum,char *prefix);

/* Returns the number of rows of data in the datset. In the ds.h example
   would return 4. */
int ds_num_rows(const datset *ds);

/* Returns the number of attributes (i.e. columns) in the dataset.
   In the example would return 6 */
int ds_num_atts(const datset *ds);


/* Returns the small-integer-value of the attnum'th attribute on
   the row'th row. In our example, ds_symbol_ref(ds,0,2) == 0,
   because the two symbol values are { "female" "male" }. We know
   they are in that order internally because female is before male
   in the alphabet. 

   PRE: ds_symbol_ref_ok(ds,attnum)
   PRE: ds_value_is_missing(ds,row,attnum) must be FALSE
*/
int ds_symbol_ref(const datset *ds,int row,int attnum);

/* Returns the real value of the attnum'th attribute in the
   row'th row. In our example ds_real_ref(3,1) == 98.0.
   PRE: ds_real_ref_ok(ds,attnum)
   PRE: ds_value_is_missing(ds,row,attnum) must be FALSE
*/
double ds_real_ref(const datset *ds,int row,int attnum);

/* Sets the small-integer-value of the attnum'th attribute on
   the row'th row. 

   PRE: ds_symbol_ref_ok(ds,attnum)
   Note that after this, ds_value_is_missing(ds,row,attnum) will be
   FALSE no matter whether or not it was missing before.
*/
void ds_symbol_set(datset *ds,int row,int attnum,int value);

/* Sets the real value of the attnum'th attribute in the
   row'th row.
   PRE: ds_real_ref_ok(ds,attnum)
   Note that after this, ds_value_is_missing(ds,row,attnum) will be
   FALSE no matter whether or not it was missing before.
 */
void ds_real_set(datset *ds,int row,int attnum,double value);

/* The name of the attnum'th attribute */
char *ds_attnum_to_name(const datset *ds, int attnum);

/* Which of the symbolic value of the specified
   (symbolic) attribute has the given name? That's
   what this function returns. If no values have this
   name, the function returns -1 */
int ds_attnum_name_to_value(const datset *ds,int attnum,char *s);

/* The arity of the attnum'th attribute. attnum MUST BE symbolic.  */
int ds_attnum_to_arity(const datset *ds,int attnum);

/* TRUE iff the attnum'th attribute is symbolic */
bool ds_attnum_to_symbolicp(const datset *ds,int attnum);

/* The name of the value'th value (alphabetically) of the
   attnum'th attribute. attnum MUST BE symbolic. */
char *ds_attnum_value_to_name(const datset *ds,int attnum,int value);

/*correctly delete a set of attributes whose name is given in sa*/
void ds_delete_attributes(datset *ds, string_array *sa);

/*delete a single attribute, referred by attnum
  careful: each delete will change other attributes'
  attnums*/
void ds_delete_attribute(datset *ds,int attnum);

/* This swaps the order of the attributes with attnums att1 and att2.
   Precondition: att1 != att2, and both are valid attnums for ds. */
void ds_swap_attributes(datset *ds,int att1,int att2);

/* This re-orders the attributes in ds, according to the
   corresponding values in the dyv.  If incp is TRUE, it
   will sort by increasing order; otherwise it will sort
   by decreasing order.  

   Thus if incp is TRUE and values[i] < values[j],
   then attribute i will be placed before attribute j.

   *WARNING!* This function modifies the values dyv.
*/
void ds_sort_by_values(datset *ds,dyv *values,bool incp);

/* This sorts the attributes by arity.  If incp is TRUE, it sorts
   them by increasing order of arity.  Otherwise it sorts them
   by decreasing arity. */
void ds_sort_by_arity(datset *ds,bool incp);

void ds_rename_attribute(const datset *ds,int attnum,char *newname);

/*PRE: newvalue must be a legal value for attribute attnum
  This takes all occurences of oldvalue and makes them into newvalue.
  It then deletes oldvalue.*/
void ds_merge_value(datset *ds,int attnum,int oldvalue,int newvalue);

void ds_rename_value(datset *ds,int attnum,int value,char *newname);

/* This deletes the value from the given attribute. After adding, 
   some of the small integer representations of values may change.

   This function ensures that after calling, all value_names for the
   given attribute remain in sorted increasing lexicographic order.

   PRE: ds_symbol_ref_ok(ds,attnum)

   Any datapoint in the dataset that previously had the given value,
   now has a missing value.
*/
void ds_delete_value(datset *ds,int attnum,int value);

void column_set_from_string(loadinst *ldi,colshape *cs,column *col,
			    int row,char *s,char **r_errmess);

void fill_datset_row(loadinst *ldi,datset *ds,string_array *tokens,
                     int row,char **r_errmess);

void fill_datset_from_file(const char *filename,loadinst *ldi,datset *ds,
                           char **r_errmess);

datset *mk_datset_from_file_and_shape(const char *filename,loadinst *ldi,
                                      shape *sh,int num_rows,
                                      char **r_errmess);

/* Loads a file from the specified file, using instructions from the
   loadinst *ldi structure for deciding things like whether the lines
   are space separated, comma separated (or should be automatically decided)
   and whether records with missing values are to be ignored and if not,
   what the default missing value should be.

   If there are no problems loading, builds and returns the datset.
   and sets *r_errmess == NULL.

   If there are problems loading returns NULL, but makes and returns
   a string in *r_errmess describing the loading problem. The caller
   must eventually free *r_errmess. */
datset *mk_datset_from_file(const char *filename,loadinst *ldi,
			    char **r_errmess);

/* Just like mk_datset_from_file_simple except uses the standard
  *r_errmess error message formalism.
*/

datset *mk_datset_from_file_simplish(const char *filename,
				     int argc,char *argv[],char **r_errmess);

/* Loads a file from the specified file, assuming the datset file format
   described at the bottom of datset.h. 
   If there are no problems loading, builds and returns the datset.
   If there are problems, prints an error message to stderr and aborts
   the entire program. 

   You can set argc == 0 and argv == NULL for default simple behavior.

   If you use a "real" command-line argc and argv, then the loader
   will search for the following command-line parameters:

     [format <string>] where string is one of

          space : use whitespace as the separator between values
          comma : use comma's as the separator (.csv style)
          auto : automatically choose between the above two (this is default)

     [ignore_missing <string>] where string is one of
          true  : Throw away any rows that contain missing data. (this is
                  the default)
          false : Load rows with missing data. Symbolic attributes
                  get assigned the special value "?". Numeric
                  attributes get given a default value <missing_numeric_value>
                  which by default is -9999.0

    [missing_numeric_value <number>] See above.
*/
datset *mk_datset_from_file_simple(const char *filename,int argc,char *argv[]);

/* Takes a symbolic attribute.
   And makes a numeric column
   with the appropriate numbers throughout. 
   real_value = symbol_val_to_real_val[symbol_value]

   PRE: ds_symbol_ref_ok(ds,attnum) and NOT ds_real_ref_ok(ds,attnum)
        to start with.

   POST: Attnum is real-valued major coltype, symbol-valued minor
         coltype. 
*/
void ds_define_real(datset *ds,int attnum,dyv *symbol_val_to_real_val);

/* Takes a symbolic attribute in which all the values
   have names that are numbers. And makes a numeric column
   with the appropriate numbers throughout. 

   PRE: All the values must be parseable as numbers. attnum must be symbolic
        to start with.
   PRE: ds_symbol_ref_ok(ds,attnum) and NOT ds_real_ref_ok(ds,attnum)
        to start with.

   POST: Attnum is real-valued major coltype, symbol-valued minor
         coltype. 
*/
void ds_realize(datset *ds,int attnum);

bool string_array_is_strictly_sorted(string_array *sa);

void check_value_names_increasing(string_array *value_names);

/* PRE: ds_real_ref_ok(ds,attnum) && value_names is sorted in lexicographically
        increasing order.

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

	 forall row,
          ds_symbol_ref(ds,row,attnum) == symbol_value_fn(ds,row,attnum,data)

	 and the symbolic value names are value_names
*/  
void ds_define_symbolic(datset *ds,int attnum,
			string_array *value_names,
			int (*symbol_value_fn)(datset *ds,
					       int row,
					       int attnum,
					       void *data),
			void *data);

/* PRE: ds_real_ref_ok(ds,attnum) && value_names is sorted in lexicographically
        increasing order.
        string_array_size(value_names) == dyv_size(breaks) + 1
	breaks is in strictly increasing order

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

forall row,
   ds_symbol_ref(ds,row,attnum) == 0 if ds_real_ref(ds,row,attnum) < breaks[0]
else
   ds_symbol_ref(ds,row,attnum) == 
     1 + min_i such that ds_real_ref(ds,row,attnum) >= breaks[i]
*/  
void ds_symbolize_with_named_breaks(datset *ds,int attnum,
				                            string_array *value_names,dyv *breaks);

string_array *mk_breaks_value_names(dyv *breaks);

/* PRE: ds_real_ref_ok(ds,attnum) 
	breaks is in strictly increasing order

   POST: ds_symbol_ref_ok(ds,attnum) && ds_real_ref_ok(ds,attnum)
         ds_real values are unchanged and

forall row,
   ds_symbol_ref(ds,row,attnum) == 0 if ds_real_ref(ds,row,attnum) < breaks[0]
else
   ds_symbol_ref(ds,row,attnum) == 
     1 + min_i such that ds_real_ref(ds,row,attnum) >= breaks[i]

   And the value_name of the i'th symbol is a name denoting the i'th bucket.
*/  
void ds_symbolize_with_breaks(datset *ds,int attnum,dyv *breaks);

void save_datset_fast(FILE *s,datset *ds);

/* The version number was added to handled multiple kinds of files.  The
   versions existing so far are:
   version 0 - crashes and/or ignores missing values
   version 1 - handles missing values
   Except for version 0, the version number is given in the first line of
   the file.
*/
datset *mk_datset_from_stream_fast(FILE *s,int *line,char **r_errmess);

datset *mk_datset_from_file_fast(const char *filename,char **r_errmess);

datset *mk_datset_from_file_fast_simple(const char *filename);

datset *mk_datset_from_xml_stream(FILE* s, int *line);

/***** Basic Operations Related To Missing Values *****/

/* MISSING VALUES: For every element in the dataset (i.e. every location
                   that is indexed by a row-attribute pair) it is possible
                   that instead of having a value, the element has a "MISSING
                   VALUE". The next few functions allow the programmer to
                   test for and manipulate missing values. The most important
                   thing about missing values is that you cannot call
                   "ds_symbol_ref" or "ds_real_ref" on a location with missing
                   value. If you do, there will be a my_error. We recommend
                   that if you are writing code that doesn't want to consider
                   missing values, you simply put a call like this....

                           if ( ds_has_missing_values(ds) )
                             my_error("No missing values allowed for D-trees");

                   at the start of your top-level function.
*/

/* Returns TRUE if and only if the specified attribute
   on the specified row has a missing value. */
bool ds_value_is_missing(const datset *ds,int row,int attnum);

/* Tells you how many rows in the datset have the specified
   attribute number missing. */
int ds_attnum_to_num_missing(datset *ds,int attnum);

/* Sets the datset so that the specified row/attribute gets
   given a missing value. After doing this, you may not use
   ds_symbol_ref or ds_real_ref to access this element of the
   datset. If you want to eventually make this element non-missing,
   you must eventually give it a value with ds_symbol_set or
   ds_symbol_ref */
void ds_set_value_missing(datset *ds,int row,int attnum);

/* True if and only if the datset has missing values. This is useful
   for you to call just before you run an algorithm that will choke
   on missing values. */
bool ds_has_missing_values(datset *ds);

/* The number of missing values of the given row */
int ds_row_to_num_missing(datset *ds,int row);
bool ds_row_has_missing_values(datset *ds,int row);

void ds_delete_last_n_rows(datset *ds,int n);

void ds_keep_subset_rows(datset *ds, ivec *subset);


/* Adds the specified number of rows at the end of the dataset. Sets the
   value of all new symbolic entries to value == (int) 0, and for all
   real entries to value == 0.0 */
void ds_add_rows(datset *ds,int num_rows);

#define MISSING_VALUE_NAME "*MissingValue*"

/* If its not already present, adds a new symbol for MISSING_VALUE.
   Returns the index of this symbol.  If you call this with a non-symbolic
   column, you will suffer miserably.

   Currently does not ensure that *missingvalue* goes in at the right
   lexicographic position. But soon will.
*/
int add_missing_symbol(datset *ds,int attnum);

/* If you call this with a non-symbolic column, you will suffer miserably. */
int missing_symbol_value(datset *ds,int attnum);

/* For the attnum'th symbolic attribute, add a new symbol value 
   MISSING_VALUE_NAME and change all the missing values to have that value.
   WARNING!! any real values WILL BE DELETED!
*/
void convert_missing_to_symbol(datset *ds,int attnum);


column *ds_attnum_to_column(const datset *ds,int attnum);

/* works with symbolic atts */
extern void replace_missingvals_with_symbol(datset *ds,int attnum,char *symbol);
/* works with real atts */
extern void replace_missingvals_with_number(datset *ds,int attnum,double val);

extern void replace_symbol_with_missingvals(datset *ds,int attnum,char *symbol);
extern void replace_number_with_missingvals(datset *ds,int attnum,double val);

#ifdef USE_IN_CPP
}
#endif
#endif /* DSET_H */

