/* *
   File:        dsut.h
   Author:      Andrew W. Moore
   Created:     Mon Jun  2 19:58:44 EDT 1997
   Description: Header for Utilities on datsets

   Copyright 1997, Carnegie Mellon University
*/


#ifndef DSUT_H
#define DSUT_H
#include "utils.h"
#include "ds.h"
#include "amdym.h"
#include "randomization.h"

/* Read ds.h first, to understand the semantics of datsets.

   This file contains additional useful operations with datsets. All 
   operations in this file use the API to datsets provided in ds.h. Thus
   if internal representations change to datsets, the implementation in
   this file will remain unchanged. 
*/

void copy_part_of_datset_row_to_dyv( datset *ds, int row, ivec *atts, dyv *dv);
void copy_dyv_to_real_column( datset *ds, dyv *dv, int att);
void copy_ivec_to_symbol_column( datset *ds, ivec *iv, int att);

double ds_att_mean(const datset *ds,int attnum);

double ds_att_sdev(const datset *ds,int attnum);

double ds_att_min(const datset *ds,int attnum);

double ds_att_max(const datset *ds,int attnum);

/* Same as ds_att_min and ds_att_max, except, it only finds the extreme
   values of the given rows, instead of all of the rows.  If rows == NULL,
   it searches all rows (ie, same functionally as ds_att_max/min) */
double ds_att_max_rows(const datset *ds, int attnum, ivec *rows);
double ds_att_min_rows(const datset *ds, int attnum, ivec *rows);


/* PRE: attnum is symbolic 
   result[i] = number of rows in ds with value i */
ivec *mk_ds_frequencies(const datset *ds,int attnum);

/* Same as above except only does min/max if small dataset */
char *mk_not_slow_ds_att_explain_string(datset *ds,int attnum);

/* Makes a brief (approx 40 char); null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_att_explain_string(datset *ds,int attnum);

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_real_att_explain_string_basic(datset *ds,int attnum,
					  bool do_min_and_max);

/* Makes a brief (approx 40 char) null-terminated string, explaining
   the attribute briefly. Its type, its arity, its range and possibly
   other brief information. 
*/
char *mk_ds_symbolic_att_explain_string_basic(datset *ds,int attnum);

/* Prints a summary of the attributes in ds to stdout */
void explain_datset(datset *ds);

/* Takes all the attributes that are symbolic but have all their
   values as numbers, and turns them into real attributes. */
void ds_make_as_real_as_possible(datset *ds);

/* Returns TRUE if and only if all attributes in ds can be
   accessed with ds_real_ref */
bool ds_all_real(datset *ds);

bool ds_all_symbolic_values_are_numbers(datset *ds,int attnum);

/* Makes a dyv x in which dyv_ref(x,i) == ds_real_ref(ds,i,attnum)
   dyv_size(x) == ds_num_rows(ds).
  PRECONDITION: attnum is real-valued. */
dyv *mk_dyv_from_real_column(datset *ds,int attnum);
ivec *mk_ivec_from_symbol_column(datset *ds,int attnum);
dyv *mk_dyv_from_symbolic_col( datset *ds, int attnum );
dyv *mk_dyv_from_column(const datset *ds,int attnum);

/* For use when calling ds_symbolize_with_breaks() (which is
   defined in ds.h) */
dyv *mk_sensible_breaks(datset *ds,int attnum,int arity);

dyv *mk_uniform_breaks(datset *ds,int attnum,int num_levels);

string_array *mk_value_names_from_attnum(datset *ds,int attnum);

string_array *mk_names_from_attnums(const datset *ds,ivec *attnums);

/* This is a somewhat special purpose combination of the previous two
   functions.  It is used when doing regressions of both symbolic (using
   1-of-n encodings) and real attributes.  The purpose is to generate
   the "names" of each term in the regression.  Real valued attributes
   get just the attribute name.  Symbolic valued attributes get the direct
   concatenation of the attribute name and the attribute value.
   */
string_array *mk_names_values_from_attnums(datset *ds,ivec *attnums);

char *mk_value_names_string_from_attnum(datset *ds,int attnum);

/* Returns TRUE if the symbolize succeeded. Returns FALSE if it failed
   for some reason (in which case an error message is printed to stdout).

   The details of the kinds of symbolization available are given in
   the documentation of dispatch_symbolize in dscli.c 

   PRE: ds_real_ref_ok(ds,attnum)
   POST: ds_real_ref_ok(ds,attnum), ds_symbol_ref_ok(ds,attnum) and
         ds_attnum_to_symbolicp(ds,attnum) 
*/
bool run_symbolize(datset *ds,int attnum,bool uniform,bool equal,
		   bool manual,bool integer,int num_levels);

bool run_clever_symbolize(datset *ds,int attnum);

/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_symbol_ref_ok(ds,attnum)
   and if !ds_real_ref_ok(ds,attnum) and if all symbol values are
   numeric.

   Then it simply sets the ith real value to be the parsed number
   associated with the ith symbol value's name. */
bool run_realize(datset *ds,int attnum);
bool run_realize_maybe_silent(datset *ds,int attnum, bool silent);

/* Returns TRUE if the unsymbolize succeeded. Returns FALSE if it failed
   for some reason (in which case an error message is printed to stdout).

   PRE: ds_symbol_ref_ok(ds,attnum)
   POST: !ds_symbol_ref_ok(ds,attnum) and !ds_attnum_to_symbolicp(ds,attnum) 
*/
bool run_unsymbolize(datset *ds,int attnum);

/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_real_ref_ok(ds,attnum).

   Then if ds_symbol_ref_ok() it sets the major column type to symbolic
   If !symbol_ref_ok() then it ignores this because then the column would
   have no type
   
*/
bool run_unrealize(datset *ds,int attnum);


/* PRE: ds_real_ref_ok(ds,attnum), ds_att_min(ds,attnum) >= 0.
         
   POST: Any previous symbolic values are destroyed, and instead the
         i'th row's new symbolic value is associated with a small
         string representation of the number. The only string-representation
         numbers that exist as value_names in the new symbolic attribute
         are for real-values that actually exist in the dataset.

         Everything is converted to the nearest integer.

   POST: ds_attnum_to_symbolicp(ds,attnum),
         ds_real_ref_ok(ds,attnum),
	 ds_symbol_ref_ok(ds,attnum)
*/
void ds_integer_symbolize(datset *ds,int attnum);

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format.

   Uses loadinst (described in shape.h) to decide how to interpret the
   file. It even tries to make sure things conform with loadinst even
   if loading from fast .fds file format.

   You can have default loading behavior with

   loadinst *ldi = mk_default_loadinst();
   char *errmess;
   datset *ds = mk_datset_from_loadinst(filename,ldi,&errmess);
   ...
   free_loadinst(ldi);

   If you want fancier loading though, then between making the default
   loadinst and calling mk_datset_from_loadinst you can request one
   of the special loadinst options using the loadinst API defined
   in shape.h.

   Errors are handled in the standard Auton way. 

   If all goes well, this returns a datset and sets *r_errmess to NULL

   If there's a problem, returns NULL and sets *r_errmess to a copy of a
   null-terminated error string which must be eventaually freed with
   free_string(...).
*/
datset *mk_datset_from_loadinst(const char *filename,
				loadinst *ldi,char **r_errmess);

/* Just like mk_datset_from_loadinst except simply calls my_error
   and aborts the entire program with a text message to stdout if
   there's a problem. */
datset *mk_datset_from_loadinst_simple(const char *filename,loadinst *ldi);

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format.

   Reads extra options from the command

   You can set argc == 0 and argv == NULL for default simple behavior.

   If you use a "real" command-line argc and argv, then the loader
   will search for the following command-line parameters:

     [format <string>] where string is one of

          space : use whitespace as the separator between values
          comma : use comma's as the separator (.csv style)
          auto : automatically choose between the above two (this is default)

     [ignore_missing <string>] where string is one of
          true  : Throw away any rows that contain missing data. (this is
                  the default)
          false : Load rows with missing data. Symbolic attributes
                  get assigned the special value "?". Numeric
                  attributes get given a default value <missing_numeric_value>
                  which by default is -9999.0

    [missing_numeric_value <number>] See above.
*/
datset *ds_load_with_options(const char *filename,int argc,char *argv[],
			     char **r_errmess);

/* Just like ds_load_with_options except my_errors if problem. */
datset *ds_load_with_options_simple(const char *filename, 
				    int argc, char * argv[]);

/* Uses the filename to decide whether to load it in fast or slow format.
   If ends with .fds, loads in fast format. 
   If ends with anything else, loads in slow (ASCII) format. 

   Uses the default options (described in comments for ds_load_with_options())
*/
datset *ds_load(const char *filename);

/* Just like ds_load except it ensures that all inputs are real-valued
   and the output is binary-valued with values "0" and "1". It also
   assumes that the output is the rightmost column. */
datset *ds_load_for_binary_outputs(char *filename);

string_array *directory_pattern_to_filenames(char *dir, char *pattern);

string_array *directory_to_filenames(char *dir);

/* Uses the filename to decide whether to save it in fast or slow format.
   If ends with .fds, saves in fast format. 
   If ends with anything else, saves in slow (ASCII) format. */
void ds_save(const char *filename,datset *ds);
void save_datset_transpose(FILE *s,datset *ds);

/* Saves the dataset to the given stream in CSV (not fast) format */
void save_datset_csv(FILE *s,datset *ds);

/* Returns a such that eq_string( ds_attnum_to_name(ds,a) , s )
   Returns -1 if no such attribute */
int ds_name_to_attnum(datset *ds,const char *s);

/* Just like ds_name_to_attnum except my_error's if no attribute
   exists with given name. So never returns -1 */
int ds_name_to_attnum_simple(datset *ds,char *attname);

/* This function doesn't care whether it's given a real,
   symbolic or missing value. In all cases, it constructs 
   a null-terminated string representation of the number,
   symbol or missingness and returns it. */
char *mk_row_attnum_to_name(datset *ds,int row,int attnum);

/* This function insists that attnum is symbolic referencable.
   It does NOT malloc a string, just returns a pointer to a
   string buried in ds that specifies the name of the value
   of the given entry in the datset. */
char *ds_row_attnum_to_name(datset *ds,int row,int attnum);

ivec *mk_unsorted_attnums_from_string_array(datset *ds,
					    string_array *sa,
					    char **r_errmess);

ivec *mk_sorted_attnums_from_string_array(datset *ds,
                                          string_array *sa,char **r_errmess);

ivec *mk_unsorted_attnums_from_string(datset *ds,const char *s,char **r_errmess);

ivec *mk_unsorted_attnums_from_string_simple(datset *ds,char *s);

ivec *mk_sorted_attnums_from_string(datset *ds,const char *s,char **r_errmess);

ivec *mk_sorted_attnums_from_string_simple(datset *ds,const char *s);

ivec *mk_sorted_attnums_from_args(char *key,datset *ds,int argc,char * argv[]);

void explain_attribute_detailed(FILE *s,datset *ds,int attnum);

void explain_datset_detailed(FILE *s,datset *ds);

/* Result ivec is of size "number of attributes".
   result[i] = number of records which with row-numbers
   included in "rows" in which attribute "attnum" has value i.

   Note rows==NULL means "use all rows". */
ivec *mk_slow_ds_counts(datset *ds,ivec *rows,int attnum);

/* Returns -z * log_2(z), being sensible and robust 
   when z == 0.0 */
double minus_plog2p(double z);

/* Assuming that value i occurs counts[i] times, and assuming
   thus that the probability of value i is

      p_i = counts[i] / sum-of-all-counts

   this computes the entropy of this signal as

     sum_over_i = -plog2p p_i */
double entropy_from_counts(const ivec *counts);

/* return a dyv with the entropy of each corresponding row in the
   given dym */
dyv *mk_entropy_column(const dym *P);

/* This calculates and returns the entropy of the given attribute
   without using an adtree. Note that if rows==NULL, means
   "use all rows".*/
double ds_slow_attribute_entropy(datset *ds,ivec *rows,int attnum);

/* Partitions "rows" into NA subrows where 
     NA = ds_attnum_to_arity(ds,attnum)
   and where r is a member of 
     ivec_array_ref(result,v) 
   if and only if 
   
        r is in "rows" 
           AND
        the value of the attnum'th attribute on
        the r'th row is v.

   It's fine to call this with rows==NULL, denoting "just
   use the whole damn dataset!"
*/
ivec_array *mk_subrows(datset *ds,ivec *rows,int attnum);

/* PRE: The source and dest must have the same shape.
   Copies the entire contents of the source_row'th row of source_ds to
     to the dest_row'th row of dest_ds.

   It is fine for source_ds and dest_ds to be the same datset. It
   does nothing if source_ds = dest_ds and source_row = dest_row. */
void ds_copy_row(datset *source_ds,int source_row,datset *dest_ds,int dest_row);

/* PRE: The source and dest must have the same shape.
   For each i, copies the entire contents of the source_rows[i]'th row 
   of source_ds to the dest_row[i]'th row of dest_ds.

   It is fine for source_ds and dest_ds to be the same datset. It
   does nothing whenever
   source_ds = dest_ds and source_row[i] = dest_row[i]. */
void ds_copy_rows(datset *source_ds,ivec *source_rows,
		  datset *dest_ds,ivec *dest_rows);
 
/* Updates the (row2,attnum2) cell of ds2 to be the same as
   the (row1,attnum1) cell of ds1.

   The copying includes the ability to copy missing-value-ness

   PRE: The shape of attnum1 in ds1 must be exactly the same as the
        shape of attnum2 in ds2.
*/
void ds_copy_value(const datset *ds1,int row1,int attnum1,
                   datset *ds2,int row2,int attnum2);

/* Creates a new datset called dsnew in which the i'th row
   of dsnew is the rows[i]'th row of ds. Thus dsnew constists
   of a subset of the rows of ds, and contains ivec_size(rows)
   such rows.

   It also contains only the specified subset of attributes.

   rows may be NULL denoting "use all rows"

   attnums may be NULL denoting "use all attributes" */
datset *mk_ds_subset(const datset *ds,ivec *rows,ivec *attnums);

/* Makes a datset containing a randomly chosen subset
   of the rows of the original datset, in random order. */
datset *mk_ds_random_subset(datset *ds,int num_rows);

/* ivec_ref(result,i) = the attribute with the i'th
   fewest missing values (i == 0 means least missing) */
ivec *mk_attnums_sorted_by_num_missing(datset *ds);

/* Prints a sorted list of the attributes that have missing
   values */
void fprint_attributes_sorted_by_num_missing(FILE *s,datset *ds);

/* Looks at the set of tokens (space and/or line separated
   strings in the file, with blank lines or lines beginning
   with a # symbol ignored) in the file. Renames the nth attribute
   to be the nth token in the file. If two few tokens in the file
   only adjusts the FIRST k attributes. If too many, ignores 
   all but the first N. */
void rename_from_file(datset *ds,char *filename);

/* Creates logN new binary attributes called name_bit_L, name_bit_L-1, ..
   name_bit_1, .. name_bit_0 giving the bits of attribute attnum. L equals 
   or is larger than logN where the attnum'th attribute has maxiimum
   value N. */
/* PRE: attnum is real-valued */
void ds_binary_explosion(datset *ds,int attnum);

/* Ignores symbolic attributes that are mentioned in attnums */
void ds_binary_explosion_of_attnums(datset *ds,ivec *attnums);

/* Creates N new binary attributes called name_is_v1, name_is_vN, ..
   name_is_vi is 1 iff the value of the attnum attribue is equal to vi
   N is the number of values the symbolic attribute attnum takes. */
/* PRE: attnum is symbolic */
void ds_symbolic_binary_explosion(datset *ds,int attnum);

void ds_partial_shuffle(datset *ds, int shuffle_size);
void ds_shuffle(datset *ds);

bool filename_is_fast(const char *filename);
bool filename_is_row(const char* filename);

/* Replaces given attribute name with their downcased versions. */
void downcase_attname(datset *ds,int attnum);

/* Replaces all attribute names with their downcased versions. */
void downcase_all_attnames(datset *ds);

/* Returns -1 if all attributes are symbolic. Else returns attnum
   such that 0 <= attnum < ds_num_atts(ds) and such that
   ds_attnum_to_symbolicp(ds,attnum) is FALSE and such that forall
   i < attnum, ds_attnum_to_symbolicp(ds,i) is TRUE */
int find_leftmost_real_attnum(datset *ds);

/* Returns the fraction variance explained when using attnum1 to predict
   attnum2 using linear regression.

   PRECONDITION: attnum1 and attnum2 must be real-valued (this function will
   call my_error() [defined in ambs.h] is that's violated. */
double fraction_variance_explained(datset *ds,int attnum1,int attnum2);

/* Returns a ds with the same shape, but in which all real values
   are randomly chosen uniformly between min and max of their
   attribute, and all symbol values are uniformly randomly chosen.
   There are no missing values in the result. */
datset *mk_random_ds_of_like_shape(datset *ds);

bool check_and_warn_ds_has_missing_values(datset *ds);

void ds_swap_values(datset *ds,int row_i,int row_j,int attnum);

void ds_swap_rows(datset *ds,int i,int j);

/*Returns the index of value s of symbolic attribute attnum,
   where s is an ascii index or a name*/
int ds_value_from_string(datset *ds,int attnum,char *s);
/*Same as above, except if s is both a valid index and name, it assumes the name was meant*/
int ds_value_from_string_no_ambiguity(datset *ds,int attnum,char *s);

/* return dyvs of attribute values from selected rows */
/* robust versions return dyvs of the same size, but which may contain
   irrelevant (in fact zeroed) values of the components which correspond to the rows
   in which there are missing values. 
   Nb. a list of these row indices are returned in missing_rows.
   If missing_rows is NULL, then there were no missing values in the rows.
   Remember to free missing_rows after use. */
extern dyv *mk_dyv_from_rows(const datset *ds, int attnum, ivec *rows );
extern dyv *mk_dyv_from_rows_robust( datset *ds, int attnum, ivec *rows, ivec **missing_rows );

extern ivec *mk_ivec_from_symbol_rows(const datset *ds, int attnum, ivec *rows );

extern dyv *mk_dyv_from_symbolic_rows( datset *ds, int attnum, dyv *output, ivec *marked_rows );
extern dyv *mk_dyv_from_symbolic_rows_robust( datset *ds, int attnum, dyv *output, 
                                       ivec *marked_rows, ivec **missing_rows );

/* PRE: All attributes must be real_ref_ok.
   Returns a matrix whose (i,j)'th element has the same value as the
   j'th attribute in the i'th row of the dataset */
dym *mk_dym_from_datset(datset *ds);

/* Loads a dym from a file named in filename. Uses the fast auton
   formats if the filename so indicates (e.g. if it ends with .fds). But
   uses slow simple .csv (comma separated) format otherwise.

   Signals an error and exits the whole program if can't find the file
   or there's a syntax error while loading. But in that case prints a 
   message to the console explain what was wrong. */
dym *mk_dym_from_ds_file(const char *filename,int argc,char * argv[]);

/* Searches for key on the command line, and uses subsequent token
   as the filename. MAKES and returns the column names in *r_attnames.

   Can handle .fds files and .csv files (plus space separated or bar
   separated)

   Automatically realizes any attributes that are symbolic but which
   have symbolic values that can be parsed as real.

   If it's a csv file with no attribute names at the top, invents
   attribute names x0 , x1 , x2 .. */
dym *mk_dym_and_attnames_from_args(char *key,int argc,char *argv[],
				   string_array **r_attnames);

/* Computes the average size of a "cluster" (i.e. a trend in the values of
   attnum) when all the attributes in time_attnums are treated as a single
   independent value.  These are grouped according to the values in attnums.
   A cluster is hereby defined as an interval of time_attnums over which 
   attnum is either consistently above or consistently below its mean.  

   Example: if attnum=="eaten", time_attnums==<"week","day">, and attnums=={"cat"}:
   A new attribute (call it "time") will be created s.t. time = (week,day).
   For each cat, it will compute the average length of time spent continuously
   above (for up_cluster) or below (for down_cluster) the mean eaten per day 
   for that cat.*/
void run_ds_clump_size(datset *ds,int attnum,ivec *tattnums,ivec *gattnums,bool up);

void run_ds_noise(datset *ds,int attnum,ivec *tattnums,ivec *gattnums,int samplesize);

/*
   PRE: attnum is symbol_ref_ok and NOT real_ref_ok
   POST: attnum is real_ref_ok and NOT symbol_ref_ok, and any
         new value is simply the small-integer representation of
         the old value.
*/
void ds_symb2real(datset *ds,int attnum);


/* Returns TRUE iff the datset gets changed. Returns FALSE if for
   some reason it's illegal to do the change.

   This function will only change an attribute if ds_symbol_ref_ok(ds,attnum)
   and if !ds_real_ref_ok(ds,attnum).

   Then it simply sets the ith real value to be a small integer
   representation of the value. For example if you had a dset with
   one attribute called fruit and it originally was

   fruit
   -----
   apple
   orange
   apple
   banana
   banana

   then set of values for the fruit attribute is { apple , banana , orange }
   and the small-integer value of each attribute is simply 0, 1 and 2
   respectively (note how the values are sorted alphabetically).

   The result dataset column is:

   fruit
   -----
   0
   2
   0
   1
   1
*/
bool run_symb2real(datset *ds,int attnum);

/* Makes a dym from the file "filename". If filename is in .fds
   format, loads it fds-style. (.fds is the Auton-style fast-load-and-save
   format of file).

   my_error()'s and exits if there's a missing file or syntax error
*/
dym *mk_dym_from_any_file_format(char *filename);

/* Makes a dym from the file "filename". If filename is in .fds
   format, loads it fds-style. (.fds is the Auton-style fast-load-and-save
   format of file).

   my_error()'s and exits if there's a missing file or syntax error

   If num_points < 0 or num_points >= number of points in the datafile,
   returns the full file. But if 0 <= num_points < full file size, returns
   a random sample of the rows in the file.
*/ 
dym *mk_dym_from_any_file_format_maybe_sample(char *filename,int num_points);

shape *mk_all_symbolic_shape_simple(int num_atts);

shape *mk_all_real_shape_simple(int num_atts);

datset *mk_datset_from_dym(dym *x);

/* Saves dym to the specified filename. If a special auton name (e.g.
   .fds format) saves it in fast efficient easy-to-load format. Else saves
   it in a simple .csv format */
void save_dym_to_any_file_format(char *filename,dym *x);

void datset_change_arity(datset *ds,int attnum,int arity);
void datset_integerize(datset *ds,int attnum);

/* This is a less disruptive version of datset_symbolize_all.  It only
   operates on attributes that don't already have symbol values.  It is
   assumed that if they have symbol values, its because the user wanted
   them that way.*/
void datset_symbolize_all_not_symbolized(datset *ds);

/* This function makes sure that every attribute has symbolic
   values. It also makes sure that if an attribute is symbolic,
   but all its values can be parsed as numbers and its arity
   is greater than two, that the symbolic values are then
   obtained by quantization to two levels */
void datset_symbolize_all(datset *ds);

/* When a datset is loaded there can be some attributes
   which only have symbolic attributes, but for which those
   symbolic values are all strings that can be parsed as numbers.
   This function "cures" all such attributes, and makes them
   primarily real-valued, but retaining the original symbolic
   values. */
void datset_realize_all_possible(datset *ds);

string_array *mk_string_array_from_attnums(datset *ds,ivec *attnums);

char *mk_string_from_attnums(datset *ds,ivec *attnums);

char *filename_from_args(int argc,char * argv[]);

datset *mk_datset_from_argc_argv(int argc, char *argv[]);

/* In all of the following functions, rows may be NULL, denoting "use all
   rows". And attnums may be NULL denoting "USE ALL ATTRIBUTES".

   Implicit precondition that all values referenced in the dataset are
   not-missing.

   For an automatic conversion to 1-of-N coding method that handles
   symbolic attributes, see Jeff's mk_dym_from_datset_portion in regress.h 
*/
dym *mk_dym_from_part_of_datset(datset *ds,ivec *rows,ivec *attnums);

dym *mk_dym_from_part_of_datset_column(datset *ds,ivec *rows,int attnum);

dyv *mk_dyv_from_part_of_datset_column(datset *ds,ivec *rows,int attnum);

ivec *mk_ivec_from_part_of_datset_column(datset *ds,ivec *rows,int attnum);

/* attnums may be null denoting use all attnums */
dyv *mk_dyv_from_part_of_datset_row(datset *ds,int row,ivec *attnums);

/* dyv_array_ref(result,i) = dyv obtained from the row[i]'th
   row of the dataset */
/* rows may be NULL denoting use all rows */
/* attnums may be null denoting use all attnums */
dyv_array *mk_dyv_array_from_part_of_datset(datset *ds,ivec *rows,
					    ivec *attnums);

/* Loads a datset in which all attributes are forced to be
   real (and not symbolic), and in which there are no missing values.

   If it discovers this is impossible (for example if one attribute
   has values that cannot be parsed as numbers) my_error()'s with a detailed
   error message.

   If it loads from fds format and finds some attributes are symbolic
   but with all parseable-as-numbers values, does the conversion 
   automatically without complaint. */
datset *mk_real_datset_from_file(char *filename);

dyv_array *mk_dyv_array_from_file(char *filename);

dyv_array *mk_dyv_array_from_args(char *key,int argc,char *argv[]);

/* Returns all rows such that row_to_marked[row] = do_marked */
ivec *mk_rows_from_marks(ivec *row_to_marked,bool do_marked);

/* Returns all rows such that row_to_marked[row] = TRUE */
ivec *mk_rows_from_row_to_marked(ivec *row_to_marked);

dym *mk_simple_dym_from_ds_file(const char *filename, int argc, char *argv[]);

void ds_realize_all(datset *ds);

dym *mk_simple_dym_from_datset(const datset *ds);

/* The following three functions use the "options" attribute specirfication
   method described in the below documentation for "attnum_from_string" */
/* NULL if no problem */
char *mk_attnum_choice_errmess(datset *ds,char *options,int attnum);

bool attnum_choice_ok(datset *ds,char *options,int attnum);

int safe_default_attnum(datset *ds,char *options,int default_attnum,
			bool *okay);


string_array *mk_valnames(const datset *ds,int attnum);

/* result[i] = valname of values[i] in attnum'th attribute of ds */
string_array *mk_valnames_from_values(datset *ds,int attnum,ivec *values);

/* If every element in valnames can be parsed as a legal value of
   the given attribute then returns a result ivec in which

     result[i] = value of attnum corresponding to name valnames[i] 

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_values_from_valnames(datset *ds,int attnum,string_array *valnames,
			      char **r_errmess);

/*
Makes an ivec of all the rows in ds in which attnum has a value
 included in the ivec "values".

It is NOT required that values is a sivec.
 */
ivec *mk_rows_from_attnum_and_values(datset *ds,int attnum,ivec *values);

/*
Makes an ivec of all the rows in ds in which attnum has a value
whose name is include in "valnames"

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_rows_from_attnum_and_valnames(datset *ds,int attnum,
				       string_array *valnames,char **r_errmess);

/* If every element in valnames can be parsed as a legal value of
   the given attribute then returns a result sivec (set of integers)
   in which 

     v is in result if and only if v's name is in valnames

   If there's a parsing problem, returns NULL and MAKES and returns
   error in r_errmess.

   *r_errmess is set to NULL if no parse problems */
ivec *mk_values_sivec_from_valnames(datset *ds,int attnum,
				    string_array *valnames,char **r_errmess);

/* Returns an attnum or else -1 denoting "no attribute".

   options is a string:

      if contains 'n' allows the user to state "none"
      if contains 'r' allows the user to select real attribute
      if contains 's' allows the user to select symbol attribute
      if contains 'm" allows the user to select attribute with missing values
*/
int attnum_from_string(datset *ds,const char *name,char *options,
                       char **r_errmess);

/* Returns an attnum matching the constraints using the command-line.

   my_error's if user enters something illegal or if it can't find an
   attribute to satisfy the command line.

   options is a string:

      if contains 'n' allows the user to state "none"
      if contains 'r' allows the user to select real attribute
      if contains 's' allows the user to select symbol attribute
      if contains 'm" allows the user to select attribute with missing values
*/
int attnum_from_args(datset *ds,const char *key,char *options,
		     int argc, char *argv[]);

/* Returns a value for the given attribute. my_error's if there's a problem.
*/
int value_from_args(datset *ds,int attnum,
		    const char *key,int argc, char *argv[]);

/* NULL attnums means "use all atributes" */
string_array *mk_attnames(datset *ds,ivec *attnums);

/* Loads a datset in which the caller forces, via the "assume" string,
   exactly what the attributes are called and what coltypes (REAL or SYMBOLIC)
   each attribute has.

   For example, 

    datset *ds = ds_load_assuming("plop.csv","height/r gender/s age/r");

   Loads a datset from the file plop.csv, but insists that there are
   exactly three attributes. The first attribute must be interpretable
   as real-valued (and will be loaded as
   real valued), and must be called height. The second will be loaded
   as symbolic and must be called gender. The third will be called age
   and must be interpretable as real-valued (and will be loaded as
   real valued).
*/
datset *ds_load_assuming(char *filename,char *assume);

/* Loads a dym in which the caller forces, via the "assume" string,
   exactly what the attributes must be called within the CSV file.

   For example, 

    dym *d = mk_dym_from_file_assuming("plop.csv","height age");

   Loads a dym from the file plop.csv, but insists that there are
   exactly two attributes called height and age respectively. Note that
   each attribute corresponds to one column of the dym.
*/
dym *mk_dym_from_file_assuming(char *filename,char *assume);

/* Returns TRUE if and only if the string representation of the
   value of ds_symbol_ref(ds,row,attnum) is equal to name.

   Returns FALSE if value is missing.

   PRE: attnum must be symbol_ref_ok */
bool ds_row_attnum_has_name(datset *ds,int row,int attnum,char *name);

void explain_datapoint(FILE *s,datset *ds,int row,char *id);

void ds_remove_minor_coltypes(datset *ds);

ivec *mk_remove_adjacent_duplicates(ivec *in); /* XXX Should be public, in amiv*/

datset *mk_datset_from_args(char *key,int argc,char *argv[]);

datset *mk_datset_from_double_arr(double *data, int cols, int rows);

/* Forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
void datset_ensure_symbolic_nomissing(datset *ds,int arity);

/* Loads a datset and forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
datset *mk_symbolic_datset_from_args(int argc,char *argv[]);


/* ignore_attnames may be NULL, meaning the same as if it has size
   zero (i.e. ignore nothing) */
/* Loads a datset and forces all attributes to be symbolic with no missing
   values no matter what the original datset was like. Does this by
   bucketing real-values into n buckets where n == "arity". And then
   replacing all missing values with a new symbolic value: "missing".
*/
datset *mk_symbolic_datset_from_filename(char *filename,int arity,
					 string_array *ignore_attnames);

/* Sorry there's no docs for this -- I moved it here because both adtree and
	mrkd had it, and it was colliding with itself. It had no docs
	there either. FIXME */
ivec* mk_sorted_attnums_from_names_friendly(datset* ds, string_array* sa);
int ds_name_to_attnum_friendly(datset* ds, char* name);

/* Adds a new symbolic attribute to ds with given attname.

   The valnames of the new attribute are the sorted duplicateless
   version of row_to_valname

   The symbolic value stored in the i'th row of the new attribute
   has the same valname as the i'th element of row_to_valname

   Thus you can think of this operation as copying in the array of
   strings into the new column. 

   PRE: ds_num_rows(ds) == string_array_size(row_to_valname)
*/
void ds_add_column_given_valnames(datset *ds,char *attname,
				  string_array *row_to_valname);

/* result[i] = string representation of contents of cell at location
    row = rows[i] and col = attnum
   
   rows == NULL means "use all rows"
*/
string_array *mk_row_to_value(datset *ds,ivec *rows,int attnum);

#endif /* #ifndef DSUT_H */
