/* 
   File:        amdmex.h
   Author:      Andrew W. Moore
   Created:     Oct 11, 1996
   Updated:     8 Dec 86
   Description: Extensions and advanced amdm stuff

   Copyright 1996, Schenley Park Research

   This file contains advanced utility functions involving dyvs dyms and
   ivecs. It never accesses the data structures directly, so if the
   underlying representation of dyms and dyvs changes these won't need to.

   The prototypes of these functions used to be declared at the end of amdm.h.
   Now it's amdmex.h
*/

#ifndef AMDMEX_H
#define AMDMEX_H

#include "standard.h"
#include "ambs.h"
#include "amdyv.h"
#include "amdym.h"
#include "am_string_array.h"

/* Useful symbols for dealing with 2-d rectangles... */
#define XCOMP 3333
#define YCOMP 4444
#define LO    5555
#define HI    6666

typedef struct dyv_array
{
  int size;
  int array_size;
  dyv **array;
} dyv_array;

typedef struct dym_array
{
  int size;
  int array_size;
  dym **array;
} dym_array;

dyv *mk_reverse_dyv(dyv *x);
ivec *mk_reverse_ivec(ivec *x);

/* Inverts ivec f, writing magic to values of inverse that don't occur in f.
   Passed ivec must be non-negative -- this is an unnecessary restriction,
   but 1) it seems like the usual case for calls to this function, and
   2) it avoid making the caller pass a "known good" magic value to use
   if the f is into and not onto [0,max(f)] */
ivec *mk_invert_nonneg_ivec(ivec* f);

void assert_is_permutation_ivec(const ivec* iv);

/* Assumes that f is a permutation of the identity ivec. */
ivec *mk_invert_permutation_ivec(ivec* f);

/* x := x with y appended on the end */
void append_to_dyv(dyv *x,dyv *y);

/* Return x with y appended on the end */
dyv *mk_dyv_append(dyv *x,dyv *y);

/* Makes a dyv of the same size as lo and hi (which must
   both be the same size such that
   dyv_ref(result,i) is uniformly randomly distributed between
   dyv_ref(lo,i) and dyv_ref(hi,i)
*/
dyv *mk_dyv_range_random(dyv *lo, dyv *hi);

/* Forall (i,j) such that ilo <= i < ihi
                          jlo <= j < jhi (note the strict inequality on right)

   We do a[i][j] += delta
*/
void dym_increment_block(dym *a,int ilo,int jlo,int ihi,int jhi,double delta);

/* Forall i such that ilo <= i < ihi (note the strict inequality on right)

   We do a[i] += delta
*/
void dyv_increment_block(dyv *a,int ilo,int ihi,double delta);


/* After calling, ivec_ref(iv,k) points to the k'th smallest
   element in dv (ties handles arbitrarily) */
void indices_of_sorted_dyv(const dyv *dv,ivec *iv);

dyv *mk_sorted_dyv(dyv *x);

/* After calling, ivec_ref(result,k) points to the k'th smallest
   element in dv (ties handles arbitrarily) */
ivec *mk_indices_of_sorted_dyv(const dyv *dv);

ivec *mk_indices_of_sorted_ivec(ivec *v); /* Artur */
void indices_of_sorted_ivec(ivec *v,ivec *iv);

void sorted_eigens_of_spd_dym(dym *d,dym *evectors,dyv *evalues);
dyv *mk_sorted_eigens_of_spd_dym(dym *d);
void random_unit_dyv(dyv *dv);
bool dyv_weakly_dominates(const dyv *dx, const dyv *dy);
bool dyv_equal(const dyv *dx, const dyv *dy);
bool dym_weakly_dominates(const dym *dx, const dym *dy);
bool dym_equal(const dym *dx, const dym *dy);
double sum_of_dym_diff_squares( dym *dm1, dym *dm2);
dym *mk_subdym( const dym *dm, const ivec *goodrows, const ivec *goodcols);
dym *mk_subdym_complement( dym *dm, ivec *badrows, ivec *badcols);
double check_robust_cholesky_factorization( const dym *A, const
					    ivec *goodlist);
dyv *mk_normalize_dyv(const dyv *d);
void normalize_dyv(const dyv *src,dyv *dest);

/* Added by Jeremy... normalize a single dyv WITHOUT
   the overhead of memory allocation/deallocation */
void normalize_this_dyv(dyv* d);

#ifndef AMFAST
#define dyv_partial_sum(dv,indices) (dyv_partial_sum_slow( dv, indices))
#else
#define dyv_partial_sum(dv,indices) (dyv_partial_sum_fast( dv, indices))
#endif

double dyv_partial_sum( const dyv *dv, const ivec *indices);
double dyv_sparse_scalar_product( dyv *dv1, dyv *dv2, ivec *goodlist);

/* Makes a random dyv with magnitude 1.0 */
dyv *mk_random_unit_dyv(int size);
void dyv_clip_in_unit(dyv *dv);
dyv *mk_dyv_clip_in_unit(dyv *dv);
void random_rotation(dym *d);
/* Declared but not defined - sir 8/6/2000
void test_random_rotation(int argc,char *argv[]); */

/* save the dyv on one line of a file in a format easy to load.
   the comment is just to make the file a small bit human readable.
   It may be NULL.
*/
void fprintf_dyv_for_load(FILE *s, const dyv *d, char *comment);
/* Warning this function only works if the next line of fp contains a dyv
   stored using the above function
*/
dyv *mk_dyv_from_file(FILE *fp, char **r_errmess);

void fprintf_oneline_dyv(FILE *s,const char *m1, const dyv *d, const char *m2);
void fprintf_formatted_oneline_dyv(FILE *s,const char *m1, const dyv *d, 
				   const char *m2, int n);

/* Makes a random subset of iv of size "k" */
ivec *mk_random_ivec_subset(ivec *iv,int k);

/* Same as above, but takes O(k) time rather than O(ivec_size(iv)) */
ivec *mk_random_ivec_subset_fast(ivec* iv, int k);

/* Makes a random small subset (sivec) of the numbers in 0..N-1
   of size "k".
   If seed is non-NULL, then it is assumed a sivec and
   all numbers in it are guaranteed to be in the returned set.
   The other numbers are sampled uniformly from the remaining set of 0..N-1
   The returned set is "small" in the sense that this algorithm is inefficient
   unless k << N.
*/
ivec *mk_random_small_subset(int N, int k, const ivec *seed);

/*
  A generalization of mk_random_small_subset in which the random set
  is guaranteed not to include any elements in the sivec "exclude".
  If an element is BOTH in seed and exclude then it will be INCLUDED.
 */

ivec *mk_random_small_subset_exclusive(int N, int k, const ivec *seed, const ivec *exclude);

ivec *mk_ivec_from_dyv(dyv *d);
dyv *mk_dyv_from_ivec(const ivec *iv);
ivec *mk_ivec_from_args(char *key,int argc,char *argv[],ivec *def);
void copy_dym_row(dym *source,int source_row,dym *dest,int dest_row);
void copy_dym_col(dym *source,int source_col,dym *dest,int dest_col);
void swap_dym_rows(dym *dm,int i,int j);
void swap_dym_cols(dym *dm,int i,int j);

/* rows may be NULL dneoting "use all rows" */
double sum_of_dyv_rows(const dyv *dv,const ivec *rows);

/* In the following functions, rows may be NULL denoting USE ALL ROWS */
dyv *mk_sum_of_dym_rows(dym *dm,ivec *rows);
void sum_of_dym_rows(dym *dm,ivec *rows, dyv *r_sum) ;

dyv *mk_sumsq_of_dym_rows(dym *dm,ivec *rows);
dyv *mk_sumcube_of_dym_rows(dym *dm,ivec *rows);
dyv *mk_mean_of_dym_rows(dym *dm,ivec *rows);
dyv *mk_sdev_of_dym_rows(dym *dm,ivec *rows);

dyv *mk_sumsq_of_dym_cols(dym *dm,ivec *cols);

/* PRE: All the vectors mentioned in "rows" must have length
   equal to dyv_length.

   If "rows" is empty, returns a vector of length "dyv_length" 
   containing all zeroes. */
/* "rows" may be NULL, meaning "use all rows" */
dyv *mk_sum_of_dyv_array_rows(dyv_array *da,ivec *rows,
			      int dyv_length);

/* rows may be NULL denoting explain all */
void explain_dyv_array(dyv_array *da,ivec *rows);

void pdyv_array(dyv_array *da);

bool dyv_array_equal(dyv_array *da1,dyv_array *da2);

dyv *mk_dyv_array_sum(dyv_array *sps,ivec *rows);
      
dyv *mk_dyv_array_centroid(dyv_array *sps,ivec *rows);

void save_dym_array_to_file(FILE *s, dym_array *ma);

ivec *mk_dyv_lengths(dyv_array *da);

int dyv_array_num_dims(dyv_array *da);

ivec *mk_ivec_from_dym_col(dym *a,int col);
ivec *mk_ivec_from_dym_row(dym *a,int row);

void diag_times_dyv(dyv *a,dyv *b,dyv *x);
dyv *mk_diag_times_dyv(dyv *a,dyv *b);

dyv *mk_sum_of_dym_cols(dym *dm);
dyv *mk_mean_of_dym_cols(dym *dm);

/* PRE: mean must be mean of x.
        mean can be obtained with mk_mean_of_dym_rows(x,rows)
   rows may be NULL meaning "use all rows" */
dym *mk_covariance_of_dym_rows(dym *x,dyv *mean,ivec *rows);

/* Makes the naive model (i.e. the diagonal of a diagonal covariance
   matrix) */
/* PRE: mean must be mean of x.
        mean can be obtained with mk_mean_of_dym_rows(x,rows)
   rows may be NULL meaning "use all rows" */
dyv *mk_diag_covariance_of_dym_rows(dym *x,dyv *mean,ivec *rows);

dyv_array *mk_empty_dyv_array(void);
void add_to_dyv_array(dyv_array *da, const dyv *dv);

dyv_array *mk_const_dyv_array(dyv *base_vec, int size);
/* Creates an dyv array with size entries each composed of an dyv of size 0  [e.g. mk_dyv(0)] */
dyv_array *mk_zero_dyv_array(int size);

int dyv_array_size(const dyv_array *da);
dyv *safe_dyv_array_ref(const dyv_array *da,int idx);

dyv_array *mk_array_of_zero_length_dyvs(int size);
dyv_array *mk_array_of_null_dyvs(int size);

dyv_array *mk_dyv_array( int size);
dyv_array *mk_dyv_array_subset( dyv_array *da, ivec *indices);

#ifdef AMFAST
#define dyv_array_ref(dva,i) ((dva)->array[i])
#else
#define dyv_array_ref(dva,i) safe_dyv_array_ref(dva,i)
#endif

void dyv_array_set(dyv_array *dva, int idx, const dyv *dv);
void dyv_array_set_no_copy( dyv_array *dva, int idx, dyv *dv);
void fprintf_dyv_array(FILE *s, const char *m1, const dyv_array *da, const char *m2);
void fprintf_oneline_dyv_array( FILE *f, const char *pre, const dyv_array *da,
				const char *post);
void free_dyv_array(dyv_array *da);
dyv_array *mk_copy_dyv_array(const dyv_array *da);
void dyv_array_remove(dyv_array *dva,int idx);

bool is_in_dyv(dyv *dv,double value, double error);
int find_index_in_dyv(dyv *dv,double value,double error);

dyv *mk_sum_of_dyv_array(const dyv_array *da);

dym_array *mk_empty_dym_array(void);

/* Creates an dym array with size entries each filled with the given dym. */

dym_array *mk_const_dym_array(dym *base_vec, int size);


void add_to_dym_array(dym_array *da,dym *dm);
int dym_array_size(dym_array *da);
dym *dym_array_ref(dym_array *da,int idx);

void pdym_array(dym_array *x);
void fprintf_dym_array(FILE *s,char *m1,dym_array *da,char *m2);
void free_dym_array(dym_array *da);
dym_array *mk_copy_dym_array(dym_array *da);

dyv *mk_dyv_from_string_array_with_error_message(const string_array *sa,
                                                 const char *format,
                                                 char **err_mess);
dyv *mk_dyv_from_string(const char *string, const char *format);
dyv *mk_dyv_from_string_with_error_message(const char *string,
                                           const char *format,
                                           char **err_mess);
void mk_io_dyvs_from_string(char *string,char *format,dyv **r_in_dyv,dyv **r_out_dyv);
char *mk_string_from_dyv(dyv *d);
dyv *mk_midpoint_dyv(dyv *a,dyv *b);
int find_index_of_kth_smallest(const dyv *x,int k);
double dyv_kth_smallest(const dyv *d,int k);
void save_dym_to_file(FILE *s, dym *m);

/* save and load a dyv or ivec from/to a stream.
   The actual data is stored as a dym. This
   way you can read a dyv or an ivec from a file and not
   have it scan all the way through the end - just
   as much as the saver had saved
*/   
void save_dyv_as_dym(FILE *s, const dyv *dv);
dyv *load_dyv_as_dym(FILE *s, char **err_msg);

void save_ivec_as_dym(FILE *s, const ivec *iv);
ivec *load_ivec_as_dym(FILE *s, char **err_msg);

string_array *mk_string_array_from_stream(FILE *s,char **r_errmess);

void save_string_array_to_stream(FILE *s, const string_array *sa);

/* Saves dyv to the file in a format that can later be read back with
   mk_dyv_from_file. */
void save_dyv_to_stream(FILE *s, const dyv *v);

/* Reads a dyv from a file.  The dyv must have been saved in the
   format produced by save_dyv_to_file.  The file pointer must be
   at the start of the dyv on entry, and will be at the end of the
   dyv on exit.  

   If this succeeds, r_errmess is set to NULL.  Otherwise it is
   set to a message explaining the problem, and mk_dyv_from_file
   returns NULL. */
dyv *mk_dyv_from_stream(FILE *s, char **r_errmess);

/* Saves ivec to the file in a format that can later be read back with
   mk_ivec_from_file. */
void save_ivec_to_stream(FILE *s, const ivec *v);

/* Reads a ivec from a file.  The ivec must have been saved in the
   format produced by save_ivec_to_file.  The file pointer must be
   at the start of the ivec on entry, and will be at the end of the
   ivec on exit.  

   If this succeeds, r_errmess is set to NULL.  Otherwise it is
   set to a message explaining the problem, and mk_ivec_from_file
   returns NULL. */
ivec *mk_ivec_from_stream(FILE *s, char **r_errmess);

/*** Supplementary string_array load/save operations ***/

/* Loads a string_array from the given filename. 

    If succeeds, returns the string_array and sets *r_errmess = NULL.
    If fails, returns NULL and AM_MALLOCS a null-terminated error
      message string in *r_errmess
*/
string_array *mk_string_array_from_filename(const char *filename, char **r_errmess);

/* Loads string_array from filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
string_array *mk_string_array_from_filename_simple(const char *filename);

/* Saves string_array to filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
void save_string_array_to_filename(const char *filename, const string_array *x);

/*** End Supplementary string_array load/save operations ***/

/*** Supplementary dyv load/save operations ***/

/* Loads a dyv from the given filename. 

    If succeeds, returns the dyv and sets *r_errmess = NULL.
    If fails, returns NULL and AM_MALLOCS a null-terminated error
      message string in *r_errmess
*/
dyv *mk_dyv_from_filename(const char *filename, char **r_errmess);

/* Loads dyv from filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
dyv *mk_dyv_from_filename_simple(const char *filename);

/* Saves dyv to filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
void save_dyv_to_filename(const char *filename, const dyv *x);

/*** End Supplementary dyv load/save operations ***/

/*** Supplementary ivec load/save operations ***/

/* Loads a ivec from the given filename. 

    If succeeds, returns the ivec and sets *r_errmess = NULL.
    If fails, returns NULL and AM_MALLOCS a null-terminated error
      message string in *r_errmess
*/
ivec *mk_ivec_from_filename(const char *filename, char **r_errmess);

/* Loads ivec from filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
ivec *mk_ivec_from_filename_simple(const char *filename);

/* Saves ivec to filename. If there's a problem, prints an
   error to stdout and aborts the entire program. */
void save_ivec_to_filename(const char *filename, const ivec *x);

/*** End Supplementary ivec load/save operations ***/

/*** Supplementary ivec load/save operations ***/

ivec *mk_sivec_from_stream_tokens(FILE *s);

ivec *mk_sivec_from_file_tokens(const char *const filename);

/*** End Supplementary ivec load/save operations ***/

dyv_array *mk_dyv_array_from_dym_rows(dym *dm) ;
dyv_array *mk_dyv_array_from_dym_cols(dym *dm) ;

/* PRE: all dyvs in x must have the same number of elements.
      : x must contains at least one dyv

   Makes dym by treating the i'th element of x as the ith row of matrix

   dym_ref(result,i,j) == dyv_ref(dyv_array_ref(x,i),j)
*/
dym *mk_dym_from_dyv_array(dyv_array *x);

/* PRE: all dyvs in x must have the same number of elements.

   dyv_ref(dyv_array_ref(result,i),j) == dyv_ref(dyv_array_ref(x,j),i)
*/

dyv_array *mk_dyv_array_transpose(dyv_array *x);

dym_array *mk_dym_array_from_file(FILE *s, char **r_errmess);
dyv *mk_dyv_from_string_array(const string_array *sa, const char *format );

string_array *mk_string_array_from_dyv(const dyv *d);

/* Returns a dyv_array of size "num_dyvs" in which the i'th
   element (forall 0 <= i < num_dyvs) is a vector of
   size "dyv_size" in which each element is 0.0 */
dyv_array *mk_dyv_array_of_zeroed_dyvs(int num_dyvs,int dyv_size);

/* Returns a dym_array of size "num_dyms" in which the i'th
   element (forall 0 <= i < num_dyms) is a matrix of
   size "dym_size x dym_size" in which each element is 0.0 */
dym_array *mk_dym_array_of_zeroed_dyms(int num_dyms,int dym_size);

/* Returns a dym_array of size "num_dyms" in which the i'th
   element (forall 0 <= i < num_dyms) is a matrix of
   size "dym_size_r x dym_size_c" in which each element is 0.0 */
dym_array *mk_dym_array_of_zeroed_nonrect_dyms(int num_dyms,int dym_size_r,
					       int dym_size_c);

void zero_dym_array(dym_array *da);

string_array *mk_string_array_from_stream_tokens(FILE *s);

/* Returns NULL if can't open file */
string_array *mk_string_array_from_file_tokens(char *filename);

/* Returns NULL if can't open file */
char *mk_string_from_file_tokens(char *filename);

void make_argc_argv_from_string_array(string_array *sa,int *actual_argc,
                                      char ***actual_argv);

/* This function AM_MALLOCS and RETURNS (in actual_argc and
   actual_argv) a new argc and argv. These are usually the same
   as argc and argv. But if argfile <filename> appears on the
   commandline, reads in all the tokens in <filename> and strings them
   together to make a longer argc argv. These new elements are
   appended onto the end of copies of the original argc argv.

   When you are done with these you should call

     free_loaded_argc_argv(actual_argc,actual_argv)

   Example:

     If the contents of file plop are:
        # This is a comment
        nsamples 35
        name Andrew

   ...and if someone runs the program foo with

       foo height 35 argfile plop noisy t

    Then after load_actual_argc_argv is called it will be as though
    the following command line was used:

       foo height 35 noisy t nsamples 35 name Andrew
*/
void load_actual_argc_argv(int argc,char *argv[],
                           int *actual_argc,char **r_actual_argv[]);

void free_loaded_argc_argv(int argc,char **argv);

/* The following is for if you want to rename "argfile" to some other
   flag name */
void load_actual_argc_argv_with_keyword(int argc, char** argv, 
                                        int* actual_argc, char*** actual_argv,
                                        char* keyword);

/* Following is to make an argc/argv from what's provided with the given
   string removed */
void mk_filtered_argc_argv(int argc, char** argv, int* actual_argc,
                           char*** actual_argv, char* remove);

/* Functions to manipulate typical argv arrays for AUTON programs. */
char **mk_copy_argv( int argc, char **argv);
void free_argv_copy( int argc, char **argvcopy);
void fprintf_argv( FILE *f, char *pre, int argc, char **argv, char *post);
void mk_argv_copy_append_val( int argc, char **argv, char *val,
                              int *newargc, char ***newargv);
void mk_argv_copy_with_new_key_val( int argc, char **argv,
                                    char *key, char *val,
                                    char ***newargv);
void mk_argv_copy_remove_key_val( int argc, char **argv, char *key,
                                  int *newargc, char ***newargv);

/* rows may be NULL denoting "use all datapoints" */
void make_vector_limits(dym *x,ivec *rows,dyv **xlo,dyv **xhi);
void set_vector_limits_sensible(dyv *xlo,dyv *xhi);

void make_vector_limits_from_dyv_array(const dyv_array *da, dyv **xlo,dyv **xhi);

/* rows may be NULL denoting "use all datapoints" */
void make_sensible_vector_limits(dym *x,ivec *rows,dyv **xlo,dyv **xhi);

/* A line is interesting if its not all white space and
the leftmost non whitespace character isnt # */
bool line_string_is_interesting(char *line_string);

bool line_contains_only_character_c(char *line_string,char c);
bool line_contains_only_dashes(char *line_string);
bool exload_white_space(char c);

/* Searches the file for the next line that isn't all whitespace and
   that doesn't have # as its first non-whitespace character. 

   If no-such line before file-end, returns NULL */
char *mk_next_interesting_line_string(FILE *s,int *line_number);

/* As above excepts breaks resulting line into a string array of tokens... */
/* DON'T USE THIS!! It's old and inferior to...

string_array *mk_next_tokens(FILE *s,int *line_number,int line_format);

which is documented later in this file */
string_array *mk_next_interesting_line(FILE *s,int *line_number);

void save_dyv_to_file_plain(char *fname,dyv *x,char **r_errmess);
void execute_command(char *exec_string,char **r_errmess);
dyv *mk_dyv_from_file_plain(char *fname,int size,char **r_errmess);

/* Supplementary load operations for lines with a header.
   For example, suppose a file looked like:

   Num_patients: 20
   Interesting_indices: 1 2 3
   Probability: 0.95
   Names: Larry Curly Moe

   The "headers" here are "Num_patients:","Interesting_indices:", and
   "Probability".  Notice that no spaces are allowed in these headers.
   For the first line, we obtain an int from the line.
   The second line gives us an ivec.  The third line gives back a double.
   Finally, for the fourth line we want the list of names, which is a string
   array without the header.  There are utility functions to do these 
   operations and more.
 */

int read_bool_from_file_with_header(FILE* fp, char* filename, char* header_str,
				    int* linenum);
int read_int_from_file_with_header(FILE* fp, char* filename, char* header_str, 
				   int* linenum);
double read_double_from_file_with_header(FILE* fp, char* filename, 
					 char* header_str, int* linenum);
dyv* mk_dyv_from_file_with_header(FILE* fp, char* filename, char* header_str, 
				  int* linenum);
ivec* mk_ivec_from_file_with_header(FILE* fp, char* filename,
				    char* header_str, int* linenum);

/* The string returned has memory allocated for it so remember to free it.  
   This function Returns NULL if line is not valid */
char* mk_string_from_file_with_header(FILE* fp,
				      char* filename,
				      char* header_str,
				      int* linenum);
string_array* mk_string_array_from_file_with_header(FILE* fp,
						    char* filename,
						    char* header_str,
						    int* linenum);

/* Corresponding functions to write to a file using a header*/

/* 
   Suppose we have an ivec {0,1,2} and a header "Header:".  This will write
   out the ivec to a file as:
   Header: 0 1 2
*/
void ivec_write_with_header(FILE* fp, char* header, ivec* iv);

void dyv_write_with_header(FILE* fp, char* header, dyv* dv);

void string_array_write_with_header(FILE* fp, char* header, string_array* sa);

/********* DATFILE PARSING UTILITIES ************/

/* int linestyle formats (we recommend AUTO_FORMAT)....

   The following constants determine how lines are read from
   a datafile. COMMA_FORMAT expects commas between each line on
   the datafile. WHITESPACE expects one or more spaces between 
   each item. BAR_FORMAT expects vertical bars (|)

   And AUTO_FORMAT will accept commas, whitespace and bars
   as separators. */
#define COMMA_FORMAT      0
#define WHITESPACE_FORMAT 1
#define BAR_FORMAT        2
#define AUTO_FORMAT       3

/************* NEW LINE PARSING CODE *************/

/* If line_format is WHITESPACE then the line is read SPACE_STYLE
   if line_format is COMMA      then the line is read COMMA_STYLE
   if line_format is BAR        then the line is read BAR_STYLE
   if lineformat is  ANY        then

        count the number of unquoted commas on the line (n_comma)
        and count the number of unquoted bars on the line (n_bar)

        if ( n_comma == 0 && n_bar == 0 ) use SPACE_FORMAT
        if ( n_comma >= n_bar )           use COMMA_FORMAT
        if ( n_bar > n_comma )            use BAR_FORMAT

   The line parser runs through a finite state machine. On
   each character it looks at the character type:

     S Space       - The character is the ' ' char
     C Comma       - The character is the ',' char
     A SingleQuote - The character is the '\'' char
     Q DoubleQuote - The character is the '\"' char
     T Token       - The character is something other than the above
     
   The line parser is building up an array of tokens. It begins with
   an empty array of tokens. It has a current token being built. It begins
   with the current token empty. After each character is read, it performs
   one of the following actions:

     ADD   Add the curent token to the array. Set the current token to empty
     PUSH  Put the current character at the end of the current token
     NIL   Do nothing
     DASH  Put a dash character at the end of the current token
     DP    Put a dash, then the current character at end of token
     UNKN  Add the UNKNOWN_STRING to the array. Clear current token


  COMMA_STYLE parsing:

       All whitespace to immediate left and right of commas is removed.
       All other contiguous blocks of whitespace are replaced with - symbols
         (outside quotes, N contiguous spaces are replaced with one -.
          inside quotes, N contiguous spaces are replaced with N -'s)
       The resulting tokens between commas are used.
       Empty string between commas is turned into UNKNOWN STRING
  
  SPACE_STYLE parsing:

       All whitespace inside quotes are turned to dashes
       All other CONTIGUOUS blocks of whitespace are collapsed to one space
       Then the resulting tokens between whitespaces are used.
*/

string_array *mk_parse_data_line(char *string,int line_format);
string_array *mk_next_tokens(FILE *s,int *line_number,int line_format);
string_array *mk_default_attribute_names(int num_atts);

void make_attnames_and_dym_from_filename(char *filename,int argc,char *argv[],
					 string_array **r_attnames,
                                         dym **r_x,char **r_errmess);

dym *mk_dym_from_file(FILE *s, char **r_errmess);
dym *mk_dym_from_filename(char *filename,char **r_errmess);
dym *mk_dym_from_filename_simple(char *filename);
void save_attnames_and_dym(FILE *s,string_array *attnames,dym *x);
void save_dym_to_filename(const char *filename, const dym *x);
void add_to_x_from_string_array(dym *x,string_array *sa,char **r_errmess);
bool contains_a_number(string_array *sa);
bool line_has_unquoted_comma(char *string);
void line_count_unquoted(char *string, int *r_num_unquoted_commas, int *r_num_unquoted_bars);
bool all_numeric(string_array *sa);

/***************** SIVEC ***************/

/* A sivec is a regular old ivec, except it is treated as a set of
   integers.

   An ivec is a legal sivec if it is sorted in increasing order with no
   duplicates.

   The following set of functions consititute a reasonable simple
   package of set-theory operations.

   Note that everything is as efficient as possible for a set package 
   except for adding a single element and deleting a single element, 
   which (because of our representation by means of sorted ivecs) could
   take time linear in set size. */

bool is_sivec(const ivec *iv);

/* Makes { 0 , 1 , ... size-1 } */
ivec *mk_identity_sivec(int size);

/* Returns number of elements in sivec */
int sivec_size(const ivec *siv);

/* Returns the minimum value in sivec. 
   Time cost: Constant */
int sivec_min(const ivec *siv);

/* Returns the maximum value in sivec. 
   Time cost: Constant */
int sivec_max(const ivec *siv);

/* Adds the element while maintaining legal siveckiness.
   (If element already there, no change)
   Time cost: O(size) */
void add_to_sivec(ivec *siv,int value);

ivec *mk_add_to_sivec(ivec *siv,int value);

/* Returns -1 if the value does not exist in siv.
   Else returns index such that
      value == ivec_ref(siv,value) 
  Time cost: O(log(size)) */
int index_in_sivec(const ivec *siv,int value);

/* Returns true iff siv contains value
   Time cost: O(log(size)) */
bool is_in_sivec(const ivec *siv,int value);

/* Does nothing if value is not in siv.
   If value is in siv, the sivec is updated to
   represent siv \ { value } */
void sivec_remove_value(ivec *siv,int value);
  
/* Returns answer to A subset-of B?
   Returns true if and only if the set of integers in a is
   a subset of the set of integers in b */
bool sivec_subset(const ivec *siva,const ivec *sivb);

bool sivec_equal(const ivec *siva,const ivec *sivb);

/* Returns TRUE iff A is a subset of B and A != B */
bool sivec_strict_subset(const ivec *siva,const ivec *sivb);

ivec *mk_sivec_union(const ivec *siva,const ivec *sivb);

/* Returns A \ B.
   This is { x : x in A and x not in B } */
ivec *mk_sivec_difference(const ivec *siva,const ivec *sivb);

/* Returns {0,...,size_of_everything-1} \ siva. */
ivec *mk_sivec_complement( const ivec *siva, int size_of_everything);

ivec *mk_sivec_intersection(const ivec *siva,const ivec *sivb);

int size_of_sivec_intersection(const ivec *siva,const ivec *sivb);

/* Returns TRUE iff A intersect B is empty. O(size) time */
bool sivec_disjoint(ivec *siva,ivec *sivb);

ivec *mk_sivec_from_ivec(const ivec *iv);

ivec *mk_ivec_from_string(char *s);

/* Turns a space separated string into a sivec.
   Example: "3 1 4 1 5 9" ====> { 1 , 3 , 4 , 5 , 9 } */
ivec *mk_sivec_from_string(char *s);

ivec *mk_sivec_1(int v1);
ivec *mk_sivec_2(int v1,int v2);
ivec *mk_sivec_3(int v1,int v2,int v3);
ivec *mk_sivec_4(int v1,int v2,int v3,int v4);
ivec *mk_sivec_5(int v1,int v2,int v3,int v4,int v5);

ivec_array *mk_array_of_empty_ivecs(int size);

ivec_array *mk_sivec_array_from_ivec_array(ivec_array *iva);

int index_of_longest_ivec(ivec_array *iva);

ivec *mk_union_of_sivec_array(ivec_array *iva);

ivec *mk_union_of_sivec_array_subset(ivec_array *iva,ivec *indexes);

/*----------------------- returns a random permutation of v */
void shuffle_dyv(dyv *v);

/* Returns a dyv in which result[i] = q means
   "the i'th element of source is the q'th smallest element in source"

   result[i] = 1 means "the i'th element of source is its smallest element"
   result[i] = dyv_size(source)
               means "the i'th element of source is its biggest element"

   The big deal about this function is that it treats ties fairly, so
   that, for example, if exactly two elements (i and j) tie for smallest
   element we'll have
     result[i] = result[j] = 1.5

   Note that it is always the case that dym_sum(result) = N choose 2
   where N is the length of source.
*/
dyv *mk_ranks_fast(dyv *source);

/***************** sosarray ***************/

bool string_less(char *s1,char *s2);
bool string_greater(char *s1,char *s2);
bool string_leq(char *s1,char *s2);
bool string_geq(char *s1,char *s2);

/* A sosarray is a regular old string_array, except it is treated as a set of
   integers.

   An string_array is a legal sosarray if it is sorted in increasing order with no
   duplicates.

   The following set of functions consititute a reasonable simple
   package of set-theory operations.

   Note that everything is as efficient as possible for a set package 
   except for adding a single element and deleting a single element, 
   which (because of our representation by means of sorted string_arrays) could
   take time linear in set size. */

bool is_sosarray(string_array *sa);

/* Returns number of elements in sosarray */
int sosarray_size(string_array *sosarr);

/* If sosarr has 0 elements returns 0
   If value > string_array_max(sosarr) returns size
   If value <= string_array_min(sosarr) returns 0
   Else returns index such that
      value <= string_array_ref(sosarr,index) 
      string_array_ref(sosarr,index-1) < value
      
   It returns the value such that string_array_insert(sa,index,value)
   would represent the set with value added to sa (assuming value
   wasn't already in sa). */
int find_sosarray_insert_index(string_array *sosarr,char *string);

/* Adds the element while maintaining legal sosarraykiness.
   (If element already there, no change)
   Time cost: O(size) */
void add_to_sosarray(string_array *sosarr,char *string);

/* Returns -1 if the string does not exist in sosarr.
   Else returns index such that
      string == string_array_ref(sosarr,string) 
  Time cost: O(log(size)) */
int index_in_sosarray(string_array *sosarr,char *string);

/* Returns true iff sosarr contains string
   Time cost: O(log(size)) */
bool is_in_sosarray(string_array *sosarr,char *string);

void sosarray_remove_at_index(string_array *sosarr,int idx);

/* Does nothing if string is not in sosarr.
   If string is in sosarr, the sosarray is updated to
   represent sosarr \ { string } */
void sosarray_remove_string(string_array *sosarr,char *string);
  
/* Returns answer to A subset-of B?
   Returns true if and only if the set of integers in a is
   a subset of the set of integers in b */
bool sosarray_subset(string_array *sosarra,string_array *sosarrb);

bool equal_string_array(string_array *sa1,string_array *sa2);

bool sosarray_equal(string_array *sosarra,string_array *sosarrb);

/* Returns TRUE iff A is a subset of B and A != B */
bool sosarray_strict_subset(string_array *sosarra,string_array *sosarrb);

string_array *mk_sosarray_union(string_array *sosarra,string_array *sosarrb);

/* Returns A \ B.
   This is { x : x in A and x not in B } */
string_array *mk_sosarray_difference(string_array *sosarra,string_array *sosarrb);

string_array *mk_sosarray_intersection(string_array *sosarra,string_array *sosarrb);

/* Returns TRUE iff A intersect B is empty. O(size) time */
bool sosarray_disjoint(string_array *sosarra,string_array *sosarrb);

string_array *mk_sosarray_from_string_array(string_array *sa);

string_array *mk_string_array_from_string(char *s);

/* Turns a space separated string into a sosarray.
   Example: "3 1 4 1 5 9" ====> { 1 , 3 , 4 , 5 , 9 } */
string_array *mk_sosarray_from_string(char *s);
char *sosarray_first(string_array *sosarr);
char *sosarray_last(string_array *sosarr);

/* Makes an 'LS' style string matrix. That means it takes the
   string array and puts each element into cells of a string
   matrix. The string matrix has "cols" columns. The order in
   which string_array elements are placed is

      sa[0]      sa[r+0]    ....    sa[(cols-1)r+0]
      sa[1]      sa[r+1]    ....    sa[(cols-1)r+1]
        :                                  :
        :                                  :
        :                                  :
      sa[r-1]    sa[2r-1]   ....    sa[(cols-1)r-1]

    where r is the least r such that r*cols >= string_array_size

   ...and some of the rightmost column might be filled with
      empty cells.
*/
string_matrix *mk_ls_style_string_matrix_given_cols(string_array *name,int cols);

/* Returns the max string length in sa */
int string_array_max_length(string_array *sa);

/* Makes an 'LS' style string matrix cleverly designed so that when
   printed it uses less than "max_chars" characters per line. (it auto-chooses
   and sizes the columns) */
string_matrix *mk_ls_style_string_matrix(string_array *names,int max_chars);

int sm_rows(string_matrix *sm);

void sm_set_string(string_matrix *sm,int row,int col,char *string);

void sm_set_double(string_matrix *sm,int row,int col,double x);

void sm_set_int(string_matrix *sm,int row,int col,double n);

/* Prints the contents of string_array cleverly much in the same way
   that "short ls" in unix displays filenames. It's cleverly designed so that when
   printed it uses less than "max_chars" characters per line. (it auto-chooses
   and sizes the columns) */
void display_names(FILE *s,string_array *names);

void explain_named_dyv(string_array *names,dyv *values);

/* forall i, rx[i] := x[i] + a */
void dyv_add_constant(dyv *x,double a,dyv *rx);

/* Returns dyv of size ivec_size(rows) in which
    result[i] = x[rows[i]] */
dyv *mk_dyv_subset(dyv *x,ivec *rows);

/* Writes dv[a] to dv[b-1] into r_dyv.  If a or b is negative, then
   it is replaced by by len+a or len+b. Returns the number of elements
   written.
*/
int dyv_slice( const dyv *dv, int a, int b, dyv *r_dv);

/* If b >= a >= 0, creates "slice" of dv on [a,b-1].  If a or b < 0, then
   it is replaced by len+a or len+b.  Indices which exceed bounds are
   replaced by appropriate indices.  Therefore this function always succeeds,
   sometimes returning an empty dyv.
*/
dyv *mk_dyv_slice( const dyv *dv, int a, int b);

/* Returns ivec of size ivec_size(rows) in which
    result[i] = x[rows[i]] */
ivec *mk_ivec_subset(ivec *x,ivec *rows);

ivec *mk_ivec_intersection(ivec *a,ivec *b);

/*
  mk_dym_subset - builds a dym out of a subset of rows
  and columns from a matrix x such that if:
    r_given = rows(i)
    c_given = cols(j)
    val = x(r_given,c_given)
  then:
    new_dym(i,j) = val;
  Note: this can also be used to permute rows and columns.

  rows may be NULL meaning "use all rows"
  cols may be NULL meaning "use all columns"
*/
dym *mk_dym_subset( const dym *x, const ivec *rows, const ivec *cols);

dym *mk_dym_from_col_subset(dym *x, ivec *cols);

ivec* mk_find_all_indices_in_ivec(ivec* vect, int val);

ivec* mk_find_all_indices_in_dyv(dyv* vect, double val, double tol);

dyv *mk_dyv_extremes_from_center( dyv *dv, double center, int numextremes,
				  ivec **indices);
dyv *mk_dyv_extremes( dyv *dv, int numextremes, ivec **indices);
void fprintf_oneline_dyv_extremes_from_center( FILE *f, char *pre, dyv *dv,
					       char *post, double center,
					       int numvals);
void fprintf_oneline_dyv_extremes( FILE *f, char *pre, dyv *dv, char *post,
				   int numvals);

void fprintf_subdym( FILE *f, char *pre, dyv *dv, dym *d, char *post,
		     const ivec *goodrows, const ivec *goodcols);


double median_of_three(double x,double y,double z);
bool double_equal(double x,double y);

/*
  Given two orders, as permutations of the same length, generates a new
  order that is an interleaving of them. The new order is a permutation as
  well, and it alternates taking objects from order1 and order2.  Objects
  that were already put in the order are skipped.
 */
ivec *mk_interleave_order(const ivec *order1, const ivec *order2);

/* A generalization of mk_interleave_order to more than 2 orders */
ivec *mk_interleave_multiple_order(const ivec_array *order);

/*
  A generalization of mk_interleave_multiple_order where each order can
  have elements taken from it multiple times. This is according to the schedule
  ivec which has the same length as order and specifies how many elements
  to take from each order.

  For example, an all-ones schedule is the same as mk_interleave_multiple_order.

  A schedule of 2-3-1 means:
  - take the first two elements from order[0]
  - then take the next three elements from order[1] which were not seen yet
  - then take the next element from order[2] which was not seen yet
  - continue cycling from order[0], adding the unseen elements in order until 
    no more elements (equivalently, until a permutation is generated).

  If chosen_order is non-NULL, it will be filled with an ivec containing, for
  each slot in the returned ivec, the index of the permuation in order
  which caused it to be inserted.

 */
ivec *mk_interleave_multiple_order_with_schedule(const ivec_array *order, const ivec *schedule,
                                                 ivec **chosen_order);

#endif /* #ifndef AMDMEX_H */
