/*
   File:        kmeans.c
   Author:      Andrew W. Moore, Dan Pelleg
   Created:     Sun Apr  5 11:49:13 EDT 1998
   Description: Kd-tree for simple k-means clustering
   $Id: kmeans.c,v 1.68 2004/01/21 02:22:00 dpelleg Exp $

   Copyright 1998, Carnegie Mellon University

*/

#include "simple_kdtree.h"
#include "kmeans.h"
#include "amgr.h"
#include "debug.h"
#include "incremental.h"
#include "traversal.h"
#include "split.h"
#include "dyvp_array.h"
#include <math.h>
#ifdef UNIX_PLATFORM 
#include <sys/resource.h>
#include <unistd.h>
#endif /* UNIX_PLATFORM */

static const char *const CVSID = "$Id: kmeans.c,v 1.68 2004/01/21 02:22:00 dpelleg Exp $";

#define remove_this_to_enableMAKE_PSFILES
#define LINELEN 256

/*   dym *data is used for debugging/visualization */
static void kmeans_slow_find_sums(const dym *x, vctr *centers, ivec *nums, dyv_array *sums, dyv *distortion);

static void hplane_update_sums(const dym *data, simple_kdtree *node, vctr *centers, 
							   ivec *nums, dyv_array *sums);

/* return dyv c (as index into centers), "owner" of all points in hrect in
   the sense each p in hrect has c has its closest center. Return -1 if no
   such c exists */
static int owner_of_node(hrect *hr, const vctr *centers);

/* heuristic suggestion of start centers. <ncenters> of them will be placed
in <centers>. */
static void gen_start_centers(simple_kdtree *kd, unsigned ncenters,
							  vctr *centers);

/* scan through centers, return index of center closest to point x */
static int find_closest_center(const dyv *x, const vctr *centers);

static void kmeans_hplane_find_sums(const dym *x, simple_kdtree *kd, vctr *centers,
									ivec *nums, dyv_array *sums);

static double kmeans_pass(kmeans_state *kmstate, const kmeans_parm *parm);

/* random selection of ncenters points out of x to be start centers */
static void random_start_centers(dym *x, unsigned ncenters, vctr *centers);

/*
  Given a confidence level alpha, return the critical value
  for it under the corrected Anderson-Darling statistic.
  This is done by looking up a small table of values, and
  the closest value is chosen.
  alpha should be 0.99, 0.999 or other close-to-one value.
 */
static double AD_critical_value(double alpha);

#ifdef WANT_MK_RANDSTART_CENTERS_SLOW
/* random selection of ncenters points out of x to be start centers,
   trying each point num_resamples times to get best one.
   The new centers are returned.
   allowed_subset is the set of indices into x from which points are
   allowed to be sampled. NULL means "all points allowed".
   Slow (ncenters^2) but supposedly good. */
static vctr *mk_randstart_centers_slow(dym *x, int ncenters, int num_resamples,
									 ivec *allowed_subset);

#endif /* WANT_MK_RANDSTART_CENTERS_SLOW */

static void mk_dym_from_points_in_kdtree_recursive(simple_kdtree *kd, dym *x);

static void collect_points_in_kdtree(ivec **ret, const simple_kdtree *kd);

void kdtree_main(int argc,char *argv[])
{

#ifdef MAKE_PSFILES
  int picnum = 0;
  char tmp[128];
#endif /* MAKE_PSFILES */

  const char *filename = string_from_args("in",argc,argv,"data/default.ds");
  dym *x = mk_simple_dym_from_ds_file(filename,argc,argv);
  double min_box_width_rel = double_from_args("min_box_width", argc, argv, 1e-3);
  int max_leaf_size = int_from_args("max_leaf_size", argc, argv, 40);
  simple_kdtree *kd;
  const int num_dims = dym_cols(x);
  const int num_rows = dym_rows(x);
  int i;
  dyv_array *x_da = mk_dyv_array_of_zeroed_dyvs(num_rows, num_dims);


  /* make a copy of x in x_da */
  {
	int i;
	dyv *row = mk_dyv(num_dims);
	for(i=0; i<num_rows; i++) {
	  copy_dym_row_to_dyv(x, row, i);
	  dyv_array_set(x_da, i, row);
	}
	free_dyv(row);
  }

  kd = mk_simple_kdtree(x_da, min_box_width_rel, max_leaf_size);

  if(DEBUG_FLAG(D_DRAWPOINTS))  {
	hrect_set_ag_frame_with_border(kd->hr,0,1);

    for ( i = 0 ; i < 10 ; i++ )
    {
#ifdef MAKE_PSFILES
      snprintf(tmp, 128, "kmeans-%d.ps", picnum++);
      ag_on(tmp);
#else /*  MAKE_PSFILES */
      ag_on("");
#endif /*  MAKE_PSFILES */
      draw_simple_x(x);
      draw_simple_kdtree(kd,i,i);
      printf("Depth %d\n",i);
	  if(DEBUG_FLAG(D_INTERACTIVE))
		wait_for_key();
      ag_off();
    }

#ifdef MAKE_PSFILES
      snprintf(tmp, 128, "kmeans-%d.ps", picnum++);
      ag_on(tmp);
#else /*  MAKE_PSFILES */
      ag_on("");
#endif /*  MAKE_PSFILES */
    draw_simple_x(x);
    draw_simple_kdtree(kd,0,7);
    printf("First seven levels\n");
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
    ag_off();
  }
  free_dym(x);
  free_simple_kdtree(kd);
}

void gen_simple_gaussian_datafile(const char *filename,int num_rows,int num_cols,
								  int num_classes,double sigma,int seed)
{
  FILE *s = safe_fopen(filename,"w");
  dym *class_centers = mk_dym(num_classes,num_cols);
  int i,j;
  char universe_name[200];
  hrect *universe = mk_zero_hrect(num_cols); /* dimensions of universe */
  sprintf(universe_name, "%s.universe", filename);

  for (j=0 ; j<num_cols ; j++) {
	hrect_lo_set(universe, j, HUGE_VAL);
	hrect_hi_set(universe, j, -HUGE_VAL);
  }

  fprintf(s,"# num_rows = %d\n",num_rows);
  fprintf(s,"# num_cols = %d\n",num_cols);
  fprintf(s,"# num_classes = %d\n",num_classes);
  fprintf(s,"# seed = %d\n",seed);
  fprintf(s,"# sigma = %g\n",sigma);
  fprintf(s,"# Generated by gen_gaussian_datafile()\n\n");
  am_srand(seed);

  for ( j = 0 ; j < num_cols ; j++ )
    fprintf(s,"%s%d ","x",j);

  fprintf(s,"\n");

  for ( i = 0 ; i < num_classes ; i++ )
    for ( j = 0 ; j < num_cols ; j++ )
      dym_set(class_centers,i,j,range_random(0.0,1.0));

  if(INSTR_FLAG(D_SHOW_START_CENTERS)) {
	fprintf_dym(stderr, "class centers", class_centers, "\n");
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
  }

  for ( i = 0 ; i < num_rows ; i++ )
  {
    int classnum = int_random(num_classes);
    for ( j = 0 ; j < num_cols ; j++ )
    {
      double x = dym_ref(class_centers,classnum,j) +
                 sigma * gen_gauss();
	  if(x < hrect_lo_ref(universe, j))
		hrect_lo_set(universe, j, x);
	  if(x > hrect_hi_ref(universe, j))
		hrect_hi_set(universe, j, x);

      fprintf(s,"%g ",x);
    }
    fprintf(s,"\n");
  }

  expand_hrect(universe, 0.001);
  save_hrect(universe_name, universe);
  free_hrect(universe);

  /* save centers to file */
  {
	char tmp[128];				/* BUGBUG: length (no snprintf on DUX) */
	sprintf(tmp, "%s.ctrs", filename);
	save_dym_to_filename(tmp, class_centers);
  }

  free_dym(class_centers);
  fclose(s);
}

void gen_simple_main(int argc,char *argv[])
{
  int num_rows = int_from_args("num_rows",argc,argv,500);
  int num_cols = int_from_args("num_cols",argc,argv,2);
  int num_classes = int_from_args("num_classes",argc,argv,8);
  int seed = int_from_args("seed",argc,argv,1);
  double sigma = double_from_args("sigma",argc,argv,0.2);
  const char *fname = string_from_args("out",argc,argv,"data/out.ds");

  gen_simple_gaussian_datafile(fname,num_rows,num_cols,num_classes,sigma,seed);

  printf("Saved %d datapoints to %s\n",num_rows,fname);
}

static int find_closest_center(const dyv *x, const vctr *centers)
{
  int j;
  double best_dsqd = HUGE_VAL;
  int best_j = -1;

  for(j=get_first_live_vctr(centers); is_valid_vctr(centers, j); j=get_next_live_vctr(centers, j)) {
    double dsqd = dyv_dsqd(x, vctr_ref(centers, j));
    if (dsqd < best_dsqd ) {
	  best_dsqd = dsqd;
	  best_j = j;
    }
  }

  return best_j;
}

int find_closest_candidate(const dyv *x, const ivec *candidates, vctr *centers)
{
  int j;
  double best_dsqd = -1.0;
  int best_j = -1;
  const int num_centers = ivec_size(candidates);

  if(num_centers == 1)
	return ivec_ref(candidates, 0);

  for (j=0; j<num_centers; j++) {
    double dsqd = dyv_dsqd(x,vctr_ref(centers,ivec_ref(candidates, j)));
	INSTR_COUNTER_INC(6);
    if (j == 0 || dsqd < best_dsqd ) {
	  best_dsqd = dsqd;
	  best_j = ivec_ref(candidates, j);
    }
  }

  return best_j;
}

int center_color(int j)
{
  static const int cols[] = {AG_BLACK, AG_RED, AG_GREEN, AG_BLUE, AG_MAGENTA, AG_GRAY};
  /*  static const int cols[] = {AG_BLACK, AG_RED, AG_GREEN, AG_BLUE, AG_RED, AG_BLUE, AG_RED, AG_BLUE, AG_RED}; BUGBUG for demo purposes only */
  static const int num_cols = sizeof(cols) / sizeof(cols[0]);
  return cols[j % num_cols];
}

static void kmeans_slow_find_sums(const dym *x, vctr *centers,
                                  ivec *nums, dyv_array *sums, dyv *distortion)
{
  int i;
  int num_rows = dym_rows(x);

  for ( i = 0 ; i < num_rows ; i++ )
  {
    dyv *x_i = mk_dyv_from_dym_row(x, i);
    int j = find_closest_center(x_i, centers);
	int pidx_j = vctr_pidx(centers, j);
    dyv *sum_j = dyv_array_ref(sums, pidx_j);
	double dist = dyv_dsqd(x_i, vctr_ref(centers, j));
    dyv_plus(x_i, sum_j, sum_j);
    ivec_increment(nums, pidx_j, 1);
	dyv_increment(distortion, pidx_j, dist);
    if(DEBUG_FLAG(D_DRAWPOINTS))    {
      ag_set_pen_color(center_color(pidx_j));
      ag_dot(dyv_ref(x_i,0),dyv_ref(x_i,1));
    }
#ifdef DEBUG
    fprintf_dyv(stderr,"x_i",x_i,"\n");
    printf("j = %d(%d)\n",j,pidx_j);
    fprintf_dyv(stderr,"sum_j",sum_j,"\n");
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
#endif

    free_dyv(x_i);
  }
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##cost: %d\n",
								num_rows*(dym_cols(x)*(1+vctr_live_size(centers)))));
}

static void kmeans_hplane_find_sums(const dym *x, simple_kdtree *kd, vctr *centers,
                                    ivec *nums, dyv_array *sums)
{
  /* for the cost calculation */
  int K = vctr_live_size(centers);
  int M = dym_cols(x);
  /* reset counters */
  INSTR_RESET_COUNTER(1);			/* no. of points seen at leaves */
  INSTR_RESET_COUNTER(2);			/* nodes lucky enough to have a single owner */
  INSTR_RESET_COUNTER(3);			/* unlucky, non-leaf, nodes */
  INSTR_RESET_COUNTER(4);			/* #times candidate_is_full_owner called */
  INSTR_RESET_COUNTER(5);			/* #times hplane_update_sums called */

  hplane_update_sums(x, kd, centers, nums, sums);
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#leaves: %d\n"
							  "#nodes w/single owner:%d\n"
							  "#nodes w/o unique owner:%d\n"
							  "#candidate_is_full_owner tests: %d\n"
							  "#nodes visited: %d\n",
							  INSTR_COUNTER_VAL(1),
							  INSTR_COUNTER_VAL(2),
							  INSTR_COUNTER_VAL(3),
							  INSTR_COUNTER_VAL(4),
							  INSTR_COUNTER_VAL(5)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##cost: %d\n",
							  INSTR_COUNTER_VAL(1)*M +
							  INSTR_COUNTER_VAL(4)*M +
							  (INSTR_COUNTER_VAL(2) + INSTR_COUNTER_VAL(3))*K*M));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##avg. len total: %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(4) - (K-1)*INSTR_COUNTER_VAL(2))/(INSTR_COUNTER_VAL(5) - INSTR_COUNTER_VAL(1))));
					  
}

static void hplane_update_sums(const dym *data, simple_kdtree *node, vctr *centers,
                               ivec *nums, dyv_array *sums)
{
  INSTR_COUNTER_INC(5);
  if(is_kd_leaf(node)) {   /* if node is a leaf, do the obvious */
	int i;
	int num_points = dyv_array_size(node->points);

	for(i=0; i<num_points; i++) {
	  const dyv *x_i = dyv_array_ref(node->points, i);
	  int j = find_closest_center(x_i, centers);
	  dyv *sum_j = dyv_array_ref(sums, j);
	  INSTR_COUNTER_INC(1);
	  dyv_plus(x_i, sum_j, sum_j);
	  ivec_increment(nums, j, 1);
	  if(DEBUG_FLAG(D_DRAWPOINTS)) {
		ag_set_pen_color(center_color(j));
		ag_pixel(dyv_ref(x_i,0),dyv_ref(x_i,1));
	  }

	}
  } else {
    int owner = owner_of_node(node->hr, centers);
	if(owner != -1) {			/* found an owner of all points */
								/* do comulative updates    */
	  dyv *sum_owner = dyv_array_ref(sums, owner);
	  int points_in_hrect;
	  INSTR_COUNTER_INC(2);
	  dyv_plus(node->sum,sum_owner,sum_owner);
	  ivec_increment(nums,owner,node->num_points);
	  if(DEBUG_FLAG(D_DRAWPOINTS)) {
		ag_set_pen_color(center_color(vctr_pidx(centers, owner)));
		ag_hrect(node->hr);
		points_in_hrect = paint_points_in_hrect(data, node->hr);
		if(points_in_hrect != node->num_points) {
		  my_error("Not same number of points, you cheat!");
		}
	  }
	} else {					/* no single owner */
	  /* go down a level */
	  INSTR_COUNTER_INC(3);
	  hplane_update_sums(data, node->left,  centers, nums, sums);
	  hplane_update_sums(data, node->right, centers, nums, sums);
	}
  }
}

static int owner_of_node(hrect *hr, const vctr *centers)
{
  double closest_dsqd;
  int num_closest, prospective_owner, i;
  dyv *temp = mk_zero_dyv(hrect_size(hr));
  
  /* first, see which center has minimal distance to hrect */
  for(num_closest = 0, closest_dsqd = HUGE_VAL, prospective_owner = -1, 
		i=get_first_live_vctr(centers);
	  is_valid_vctr(centers, i);
	  i=get_next_live_vctr(centers, i)) {
	double dist_ci_to_hrect = dsqd_hrect_point(hr, vctr_ref(centers, i), temp);
	if(dist_ci_to_hrect < closest_dsqd) {
	  num_closest = 1;
	  prospective_owner = i;
	  closest_dsqd = dist_ci_to_hrect;
	} else if(dist_ci_to_hrect == closest_dsqd) {
	  num_closest++;
	}
  }
  free_dyv(temp); temp = 0;

  if(num_closest > 1)
	return -1;					/* no hope here -- two or more minimal, equidistant */

  /* for each center other than closest, does it compete with our
	 candidate over points? */
  for(i=get_first_live_vctr(centers);
	  is_valid_vctr(centers, i);
	  i=get_next_live_vctr(centers, i)) {
	if((i != prospective_owner) && 
	   !candidate_is_full_owner(hr, centers, prospective_owner, i))
	  return -1;				/* we have a competitor */
  }
  return prospective_owner;		/* we have a winner */
}

bool candidate_is_full_owner(const hrect *hr, const vctr *centers, 
                             int candidate, int competitor)
{
  int num_dims = hrect_size(hr);
  dyv *extreme = mk_dyv(num_dims);
  /* extreme will be the corner of the hyper-rectangle that is most
     extreme in the direction (competitor - candidate) */
  int i;
  dyv *candidate_dv = vctr_ref(centers, candidate);
  dyv *competitor_dv = vctr_ref(centers, competitor);
  bool result;
  INSTR_COUNTER_INC(4);
  DBG_PRINTF(D_TRACE, ("Entering candidate_is_full_owner: cand=%d comp=%d\n",
					   candidate, competitor));

  for(i = 0; i<num_dims; i++) {
    if(dyv_ref(competitor_dv,i) - dyv_ref(candidate_dv,i) > 0)
      dyv_set(extreme,i,hrect_hi_ref(hr,i));
    else
      dyv_set(extreme,i,hrect_lo_ref(hr,i));
  }

  result = dyv_dsqd(extreme, candidate_dv) < dyv_dsqd(extreme, competitor_dv);
  
  free_dyv(extreme);

  DBG_PRINTF(D_TRACE, ("Exiting candidate_is_full_owner\n"));
  return result;
}

static double kmeans_pass(kmeans_state *kmstate, const kmeans_parm *parm)
{
  int j;
  double sum_diffs = 0.0;
  vctr *centers = kmstate->current_centers; /* shorthand */
  nodestats stats;
  ivec *active_ctrs = 0;
  int nactive_ctrs;
  const int total_npoints = kmstate->kdtree->num_points;
#ifndef NDEBUG
  int total_npoints_encountered = 0;
#endif /* NDEBUG */
  const int ndims = centers->ndims;

  if(DEBUG_FLAG(D_DRAWPOINTS))  {
#ifdef MAKE_PSFILES
	char tmp[128];
	static picnum;
	snprintf(tmp, 128, "kmeans-%d.ps", picnum++);
	ag_on(tmp);
#else /*  MAKE_PSFILES */
	ag_on("");
#endif /*  MAKE_PSFILES */

  }

  /* find sums and nums */
  switch(parm->mtd) {
  case MTD_BLACKLIST_NOCACHE:
	stats = kmeans_blacklist_mk_sums(kmstate, parm);
  break;
  case MTD_BLACKLIST:
  case MTD_BLACKLIST_RANDSTART:
	stats =  kmeans_blacklist_mk_sums(kmstate, parm);
  break;
  case MTD_BLACKLIST_APPROX: {
	means my_means;
	kmeans_blacklist_aprx_find_sums(kmstate, &my_means);
	stats.sums = my_means.sums;
	stats.nums = my_means.nums;
	stats.distortion = mk_zero_dyv(1+vctr_max_pidx(centers));
	break;
  }
  case MTD_HPLANE: {
	const int stats_size = 1+vctr_max_pidx(centers);
	dym *x = mk_dym_from_points_in_kdtree(kmstate->kdtree, ndims); /* slow as hell */
	stats.nums = mk_zero_ivec(stats_size);
	stats.sums = mk_dyv_array_of_zeroes(stats_size, dym_cols(x));
	kmeans_hplane_find_sums(x, kmstate->kdtree,
							kmstate->current_centers, 
							stats.nums,
							stats.sums);
	stats.distortion = mk_zero_dyv(stats_size);
	free_dym(x);
	break;
  }
  case MTD_SLOW: {
	const int stats_size = 1+vctr_max_pidx(centers);
	dym *x = mk_dym_from_points_in_kdtree(kmstate->kdtree, ndims); /* slow as hell */
	stats.nums = mk_zero_ivec(stats_size);
	stats.sums = mk_dyv_array_of_zeroes(stats_size, dym_cols(x));
	stats.distortion = mk_zero_dyv(stats_size);
    kmeans_slow_find_sums(x, kmstate->current_centers,
						  stats.nums, stats.sums, stats.distortion);
	free_dym(x);
	break;
  }
  default:
	my_error("Bad value in kmeans_pass\n");
  }

  /* optionally kill starved centers */
  /* (we're not killing starved centers in phased searches) */
  if(kmstate->phase == PHS_UNPHASED) {
	int i;

	/* starved centers should be killed */
	for(i=get_first_live_vctr(centers); is_valid_vctr(centers, i); i=get_next_live_vctr(centers, i)) {
	  int pidx = vctr_pidx(centers, i);
	  if(ivec_ref(stats.nums, pidx) == 0)
		remove_from_vctr(centers, i, TRUE);
	}
  }

  /* figure out which centers to update: parents or children */
  switch(kmstate->phase) {
  case PHS_UNPHASED:			/* all participate */
	active_ctrs = mk_live_ctrs(centers);
	break;
  case PHS_PARENTS:
  case PHS_CHILDREN: {			/* just children */
	ivec *all = mk_live_ctrs(centers);
	active_ctrs = mk_sivec_difference(all, kmstate->phase1_ctrs);
	free_ivec(all);
  }
	break;
  default:
	my_error("Unmatched value for phase in kmeans_pass");
  }
  nactive_ctrs = ivec_size(active_ctrs);
  for (j=0; j<nactive_ctrs; j++) {
	int cj = ivec_ref(active_ctrs, j);
	int pidx_j = vctr_pidx(centers, cj);
    dyv *sum_j = dyv_array_ref(stats.sums, pidx_j);
    int n_j = ivec_ref(stats.nums, pidx_j);
	dyv *new_cen_j = 0;
	double abs_diff = 0.0;
	dyv *center_j = vctr_ref(centers, cj);
#ifndef NDEBUG
	dyv *old_cen_j = mk_copy_dyv(vctr_ref(centers, cj));
	total_npoints_encountered += n_j;
#endif /* NDEBUG */
	if(kmstate->phase == PHS_UNPHASED) {
	  /* make a new ctr in new position, remove the old */
	  dyv *diff_j;
	  int new_cj = cj;
	  new_cen_j = mk_dyv_scalar_mult(sum_j, 1/(double) n_j);
	  diff_j = mk_dyv_subtract(center_j, new_cen_j);
	  /* if no movement, do not allocate a new center */
	  if(dyv_max(diff_j) != 0 || dyv_min(diff_j) != 0) {
		abs_diff = dyv_magnitude(diff_j);
		remove_from_vctr(centers, cj, TRUE);
		new_cj = add_to_vctr(centers, new_cen_j);
	  }
	  free_dyv(diff_j);
	} else {
	  /* we do not move, nor kill, starved children centers */
	  if(n_j != 0) {
		dyv *diff_j;
		const double sigsqd = (n_j > 1 ? dyv_ref(stats.distortion, pidx_j)/(n_j - 1) : 0);
		new_cen_j = mk_dyv_scalar_mult(sum_j, 1/(double) n_j);
		diff_j = mk_dyv_subtract(center_j, new_cen_j);
		/* if no movement, do not allocate a new center */
		if(dyv_max(diff_j) > 0 || dyv_min(diff_j) < 0)
		  abs_diff = move_vctr(centers, /* obj */
							   cj, /* index */
							   new_cen_j, /* new location */
							   log(n_j/ (double) total_npoints), /* logprob */
							   sigsqd);	/* sigsqd */
		free_dyv(diff_j);
	  }
	}

#ifndef NDEBUG
	/* if no new_cen_j was allocated, pretend it was the old one for visualization */
	if(!new_cen_j)
	  new_cen_j = old_cen_j;

	if(DEBUG_FLAG(D_DRAWPOINTS))    {
	  ag_set_pen_color(center_color(pidx_j));
	  ag_disc(dyv_ref(old_cen_j,0), dyv_ref(old_cen_j,1), 10.0*sqrt(n_j/ (double) total_npoints));
	  ag_set_pen_color(AG_BLACK);
	  ag_line(dyv_ref(old_cen_j,0), dyv_ref(old_cen_j,1),
			  dyv_ref(new_cen_j,0), dyv_ref(new_cen_j,1));
	}

	if(DEBUG_FLAG(D_SHOW_RUNNING_CENTERS)) {
	  fprintf(stderr, "j = %d, n_j = %d\n", cj, n_j);
	  fprintf_dyv(stderr, "old_cen_j", old_cen_j, "\n");
	  fprintf_dyv(stderr, "new_cen_j", new_cen_j, "\n");
	  fprintf(stderr, "new logprob %6.4f\n", log(n_j/ (double) total_npoints));
	  fprintf(stderr, "abs_diff = %g\n", abs_diff);
	}
	if(new_cen_j != old_cen_j)
	  free_dyv(old_cen_j);
#endif /* NDEBUG */
	if(new_cen_j)
	  free_dyv(new_cen_j);

	sum_diffs += abs_diff;
  }

#ifndef NDEBUG
	if(total_npoints_encountered != total_npoints)
	  my_error("Number of points doesn't sum up");
#endif /* NDEBUG */

  if(DEBUG_FLAG(D_DRAWPOINTS)) {
    ag_off();
  }
  if(DEBUG_FLAG(D_INTERACTIVE))
	wait_for_key();

  free_ivec(active_ctrs); active_ctrs = 0;
  nodestats_free_stats(&stats);
  return sum_diffs;
}

void kmeans(kmeans_state *kmstate, kmeans_parm *parm)
{
  bool finished;
#ifndef NDEBUG
  static int scount = 0;
#endif
  int count = 0;
  int time_out = global_time() + (int) parm->max_secs;
  kmstate->b_cutoff = 1.0;

  for(finished = FALSE; count < (int) parm->max_iter && !finished; count++) {
	double sum_diffs = kmeans_pass(kmstate, parm);
	kmstate->b_cutoff *= parm->cutoff_factor;
	finished = 	(sum_diffs < 1e-10);
	if(global_time() >= time_out) {
	  DBG_PRINTF(D_SHOW_END_CENTERS, ("#INFO Ran out of time per iteration\n"));
	  break;
	}
	DBG_PRINTF(D_PASSES, ("pass %3d, diff=%6.2e, time left=%d\n", ++scount, sum_diffs, time_out - global_time()));
  }
  DBG_PRINTF(D_SHOW_END_CENTERS, ("#ITERS: %d\n", count));
}

#ifdef FOUND_A_WAY_TO_HAVE_THE_FULL_DATA_SET_IN_X
void random_start_centers(dym *x, unsigned ncenters, vctr *centers)
{
  int i;
  ivec *random_pts;
  ncenters = int_min(dym_rows(x), ncenters);
  random_pts = mk_random_subset_of_ints(dym_rows(x), ncenters);
  for(i=0; i<ncenters; i++) {
	dyv *center = mk_dyv_from_dym_row(x, ivec_ref(random_pts, i));
	add_to_vctr(centers, center);
	free_dyv(center);
  }
  free_ivec(random_pts);
}
#endif /* FOUND_A_WAY_TO_HAVE_THE_FULL_DATA_SET_IN_X */

static void random_start_centers(dym *x, unsigned ncenters, vctr *centers)
{
  my_error("Sorry, random_start_centers not supported anymore");
}

static void gen_start_centers(simple_kdtree *kd, unsigned ncenters,
                             vctr *centers) 
{
  if(!kd)
	my_error("Empty KD tree in gen_start_centers!");
  if(ncenters == 0)
	return;
  if(is_kd_leaf(kd)) {
	/* randomly sample from points in leaf, without replacement */
	int i;
	int nleaf_points = kd->num_points;
	ivec *random_pts;
	if(ncenters > nleaf_points)	/* not enough pts for us here */
	  ncenters = nleaf_points;
	random_pts = mk_random_subset_of_ints(nleaf_points, ncenters);

	for(i=0; i<ncenters; i++) {
	  const dyv *center = dyv_array_ref(kd->points, ivec_ref(random_pts, i));
	  add_to_vctr(centers, center);
	}
	free_ivec(random_pts);
  } else {						/* get points from children, recursively */
	int nchild_left = kd->left->num_points;
	int ncenters_left = 0.5 + (1.0 * ncenters * nchild_left / kd->num_points);
	int ncenters_right = ncenters - ncenters_left;
	
	gen_start_centers(kd->left, ncenters_left, centers);
	gen_start_centers(kd->right, ncenters_right, centers);
  }
  return;
}

vctr *test_kmeans(simple_kdtree *kd, int k, kmeans_parm *parm, vctr *centers, const int num_dims)
{
  struct timeval start_time, end_time;
#ifdef UNIX_PLATFORM
  struct rusage rusage;
#endif
  kmeans_state kmstate;
  kmeans_parm caching_parm;
  kmstate.kdtree = kd;
  kmstate.phase = PHS_UNPHASED;
  memcpy(&caching_parm, parm, sizeof(caching_parm));
  caching_parm.usecache = CACHE_RDWR;

  if(!centers) {				/* have to create our own centers? */
	centers = mk_empty_vctr(num_dims);
	if(parm->mtd == MTD_BLACKLIST_RANDSTART)
	  random_start_centers(NULL, k, centers);
	else
	  gen_start_centers(kd, k, centers);
	/*
	  random_start_centers_slow(x, k, int_max(100, dym_rows(x)/(k*k*20)), centers, NULL);
	*/

	if(vctr_live_size(centers) < k) {
	  k = vctr_live_size(centers);
	  DBG_PRINTF(D_SHOW_END_CENTERS, ("Could only initialize %d centers.\n", k));
	}
  }

  kmstate.current_centers = centers;

  if(DEBUG_FLAG(D_SHOWDATA)) {
	dym *x = mk_dym_from_points_in_kdtree(kd, num_dims); /* slow as hell */
	fprintf_dym(stderr, "x", x, "\n");
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
	free_dym(x);
  }
  if(DEBUG_FLAG(D_SHOW_START_CENTERS)) {
	fprintf_vctr(stderr, "start_centers", centers, "\n");
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
  }
#ifdef UNIX_PLATFORM
  getrusage(RUSAGE_SELF, &rusage);
  memcpy(&start_time, &rusage.ru_utime, sizeof(rusage.ru_utime));
#endif
  kmeans(&kmstate, parm);
#ifdef UNIX_PLATFORM
  getrusage(RUSAGE_SELF, &rusage);
  memcpy(&end_time, &rusage.ru_utime, sizeof(rusage.ru_utime));
#endif
  if(INSTR_FLAG(D_SHOW_END_CENTERS)) {
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##NPOINTS %d\n", kd->num_points));
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##NCOLS %d\n", num_dims));
#ifdef UNIX_PLATFORM
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##TIME %f\n",
									  1.0*end_time.tv_sec - start_time.tv_sec
									  + 1e-6*(end_time.tv_usec - start_time.tv_usec)));
#endif
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##DISTORTION %f\n",
									  kmeans_distortion(kd, centers, &caching_parm)));
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##BICSCORE %f\n",
									  bic_score(kd, centers, &caching_parm, NULL)));
	INSTR_PRINTF(D_SHOW_END_CENTERS, ("##SIZE TREE: %lu\n", kd ? simple_kdtree_size(kd) : 0));
	if(DEBUG_FLAG(D_PASSES)) {
	  DBG_PRINTF(D_PASSES, ("##EXTRA END CENTERS:\n"));
	  fprintf_vctr(stderr, "centers", centers, "\n");
	  DBG_PRINTF(D_PASSES, ("##EXTRA END CENTERS END.\n"));
	}
 	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
  }
  if(DEBUG_FLAG(D_DRAW_END_CENTERS)) {
	int j;
	int num_centers = vctr_live_size(centers);

	ag_on("");
	ag_set_pen_color(AG_BLACK);

	for (j=0; j<num_centers; j++) {
	  double x=dyv_ref(vctr_ref(centers, j),0);
	  double y=dyv_ref(vctr_ref(centers, j),1); 
	  ag_pixel(x,y);
	}
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
	ag_off();
  }
  return centers;
}

void kmeans_main(int argc,char *argv[])
{
  const char *filename = string_from_args("in",argc,argv,"default.ds");
  double k = (double) int_from_args("k",argc,argv,0);
  double k_step = 0.0;          /* how much to increase k by if searching for k */
  const char *method_name = string_from_args("method", argc, argv, "blacklist");
  double min_box_width = double_from_args("min_box_width", argc, argv, 1e-3);
  int max_leaf_size = int_from_args("max_leaf_size", argc, argv, 40);
  int num_splits = int_from_args("num_splits", argc, argv, 0);
  int del_steps_ratio = int_from_args("del_steps_ratio", argc, argv, 0);
  simple_kdtree *kd = 0;
  const char *init_ctrs = string_from_args("init_ctrs", argc, argv, "");
  const char *save_ctrs = string_from_args("save_ctrs", argc, argv, "");
  const char *printclusters = string_from_args("printclusters", argc, argv, "");
  const char *split_statistic = string_from_args("split_stat", argc, argv, "BIC");
  const double user_split_fraction_if_all_bad = double_from_args("forced_split_fraction", argc, argv, -1);
  const double split_conf = double_from_args("split_conf_level", argc, argv, 0.9999);
  kmeans_parm parm, caching_parm;
  parm.mtd = MTD_BLACKLIST;
  parm.max_iter = abs(int_from_args("max_iter", argc, argv, 200));
  parm.max_secs = (int) parm.max_iter * 30;	/* BUGBUG: should be 30 */
  parm.max_ctrs = int_from_args("max_ctrs", argc, argv, ((int) k)*(1<<num_splits));
  parm.cutoff_factor =  double_from_args("cutoff_factor", argc, argv, 0.5);
  parm.maintain_memb = FALSE;
  parm.data_list = NULL;
  parm.split_statistic = SCORE_BIC; /* default */
  parm.split_cutoff = 0;
  parm.split_fraction_if_all_bad = 0.5;

  am_srand(int_from_args("seed", argc, argv, 1));

  parm.loop_over_k = FALSE;
  parm.usecache = CACHE_RDWR;
  parm.delete_ctrs = FALSE;
  if(caseless_eq_string(method_name, "slow")) {
	parm.mtd = MTD_SLOW;
  } else if(caseless_eq_string(method_name, "hplane"))
	parm.mtd = MTD_HPLANE;
  else if(caseless_eq_string(method_name, "blacklist")) {
	parm.loop_over_k = FALSE;
	if(k==0) {
	  if(num_splits <= 0)
		my_error("Must supply num_splits\n");
	  k = 1.0*(int)parm.max_ctrs/num_splits;
	  if(k<1.0)
		k=1.0;
	}
	parm.mtd = MTD_BLACKLIST;
  } else if(caseless_eq_string(method_name, "BL-delete")) {
	parm.loop_over_k = FALSE;
	parm.delete_ctrs = TRUE;
	if(k==0) {
	  if(num_splits <= 0)
		my_error("Must supply num_splits\n");
	  k = 1.0*(int)parm.max_ctrs/num_splits;
	  if(k<1.0)
		k=1.0;
	}
    if(del_steps_ratio <= 0)
      my_error("del_steps_ratio must be positive\n");
	parm.mtd = MTD_BLACKLIST;
  } else if(caseless_eq_string(method_name, "BL-search")) {
	parm.loop_over_k = TRUE;
	parm.mtd = MTD_BLACKLIST;
  }
  else if(caseless_eq_string(method_name, "BL-RS"))
	parm.mtd = MTD_BLACKLIST_RANDSTART;
  else if(caseless_eq_string(method_name, "BL-aprx"))
	parm.mtd = MTD_BLACKLIST_APPROX;
  else if(caseless_eq_string(method_name, "BL-nocache")) {
	parm.usecache = CACHE_NONE;
	parm.mtd = MTD_BLACKLIST_NOCACHE;
  }
  else
	my_error(mk_printf("Don't know anything about the %s method.\n"
					   "Legal values are: slow, hplane, blacklist, BL-RS, BL-aprx\n"
					   "BL-nocache, BL-search\n",
					   method_name));

  if(parm.loop_over_k) {
	if(num_splits <= 0) {
	  num_splits = 1;
	  k_step = 0;
	} else {
	  if(k==0) {
		k_step = 1.0*(int)parm.max_ctrs/(num_splits+1); /* how much to increase k each time */
		k = 1.0*(int)parm.max_ctrs/num_splits;
	  } else
		k_step = ((int)parm.max_ctrs - k)/num_splits; /* how much to increase k each time */
	}
	if(k_step < 1.0)
	  k_step = 1.0;
  }

  if(k<=0)
	my_error("Must supply k\n");

  if(caseless_eq_string(split_statistic, "BIC")) {
	parm.split_statistic = SCORE_BIC;
    parm.maintain_memb = FALSE;
  } else if(caseless_eq_string(split_statistic, "AD")) {
	parm.split_statistic = SCORE_AD;
    parm.maintain_memb = TRUE;
    parm.split_cutoff = -AD_critical_value(split_conf); /* note: we take the negative since this is the way split_centers works */
    parm.split_fraction_if_all_bad = 0;
  } else {
    my_errorf("Don't know anything about the %s split statistic.\n"
              "Legal values are: BIC, AD\n",
              split_statistic);
  }

  /* the forced_split_fraction commandline argument overrides the split fraction value */
  if(user_split_fraction_if_all_bad != -1) {
    if(user_split_fraction_if_all_bad < 0 || user_split_fraction_if_all_bad > 1) {
      my_errorf("The value for forced_split_fraction must be in [0,1] but you specified %g",
                user_split_fraction_if_all_bad);
    }
    parm.split_fraction_if_all_bad = user_split_fraction_if_all_bad;
  }

  /* make a parm version that always uses cache */
  memcpy(&caching_parm, &parm, sizeof(caching_parm));
  caching_parm.usecache = CACHE_RDWR;
  /* caching_parm.split_statistic = SCORE_BIC; */

  /*  if(parm.mtd != MTD_SLOW) 
  kd = mk_simple_kdtree(x, min_box_width, max_leaf_size); */
  make_simple_kdtree_from_file(filename, min_box_width,
                               max_leaf_size, &kd,
                               parm.maintain_memb ? &parm.data_list : NULL);

  if(DEBUG_FLAG(D_DRAWPOINTS))
	hrect_set_ag_frame_with_border(kd->hr,0,1);

  /* repeat running kmeans, splitting centers until done */
  {
	vctr *ctrs = 0;
	vctr *best_ctrs = 0;		/* best estimate for ctrs so far */
	double best_score = -HUGE_VAL;
	const int ndims = kd_num_dims(kd);
    int splitnum = 0;           /* counter for how many splits we've done already */
    bool maybe_ncenters_didnt_change; /* used to mark iteration where #centers may not change
                                       and that's OK with us*/

	if(strcmp(init_ctrs, "")) /* initialize ctrs from file */
	  ctrs = mk_vctr_from_filename(init_ctrs);

	do {
	  int num_old_ctrs;
	  double score;
      maybe_ncenters_didnt_change = FALSE;

	  /* do a single grand-iteration */
	  ctrs = test_kmeans(kd, (int) (k + 0.5), &parm, ctrs, ndims);

	  num_old_ctrs = vctr_live_size(ctrs);
	  DBG_PRINTF(D_SHOW_END_CENTERS, ("#NCENTERS: %d\n", num_old_ctrs));

	  /* get the score of the current estimate */
	  score = bic_score(kd, ctrs, &caching_parm, NULL);	/* always use cache for this */

	  /* update best-scoring estimate */
	  if(score > best_score) {
		if(best_ctrs)
		  free_vctr(best_ctrs);
		best_ctrs = mk_copy_vctr(ctrs);
		best_score = score;
	  }

	  /* quit if landed straight on max #ctrs */
	  if(num_old_ctrs == (int)parm.max_ctrs)
		break;

      /* If we're just looping over k, increase k.
         Otherwise, do a splitting/deletion step */
	  if(parm.loop_over_k) {
		k += k_step;
		free_vctr(ctrs);		/* discard current centers */
		ctrs = 0;
	  } else {
		if(num_splits - splitnum >= 1) {		/* no point splitting in last iteration */
#ifdef UNIX_PLATFORM
		  struct rusage rusage;
		  struct timeval start_time, end_time;
		  getrusage(RUSAGE_SELF, &rusage);
		  memcpy(&start_time, &rusage.ru_utime, sizeof(rusage.ru_utime));
#endif

          /* we either split or delete here, depending on the number of the
             iteration we're in */
          /* we don't want the very first step to be a deletion, so we're adding
             1 to splitnum */
          if(parm.delete_ctrs && ((splitnum + 1) % del_steps_ratio == 0)) { /* it's a deletion step */
            maybe_ncenters_didnt_change = TRUE;
            delete_ctr(kd, ctrs, &parm, maybe_ncenters_didnt_change);
          } else {              /* it's a splitting step */
            split_centers(kd, ctrs, &parm);
          }
#ifdef UNIX_PLATFORM
		  getrusage(RUSAGE_SELF, &rusage);
		  memcpy(&end_time, &rusage.ru_utime, sizeof(rusage.ru_utime));
		  INSTR_PRINTF(D_SHOW_END_CENTERS, ("##STIME %f\n",
											1.0*end_time.tv_sec - start_time.tv_sec
											+ 1e-6*(end_time.tv_usec - start_time.tv_usec)));
#endif
		  /* quit if #centers doesn't change */
		  if(!maybe_ncenters_didnt_change && vctr_live_size(ctrs) == num_old_ctrs)
			break;
		}
	  }
	} while( ++splitnum < num_splits);

	if(DEBUG_FLAG(D_DRAWPOINTS)) {
	  int j;
	  dym *x = mk_dym_from_points_in_kdtree(kd, ndims);
	  ag_on("");
	  ag_set_pen_color(AG_BLACK);
	  
	  draw_simple_x(x);
	  for(j=get_first_live_vctr(best_ctrs);
		  is_valid_vctr(best_ctrs, j);
		  j=get_next_live_vctr(best_ctrs, j)) {
		dyv *center_j = vctr_ref(best_ctrs, j);
		ag_set_pen_color(center_color(vctr_pidx(best_ctrs, j)));
		ag_disc(dyv_ref(center_j,0),dyv_ref(center_j,1),5.0);
	  }
	  if(DEBUG_FLAG(D_INTERACTIVE)) {
		DBG_PRINTF(D_INTERACTIVE, ("These are the resulting centers...\n"));
		wait_for_key();
	  }
	  free_dym(x);
	}

	if(INSTR_FLAG(D_SHOW_END_CENTERS)) {
	  INSTR_PRINTF(D_SHOW_END_CENTERS, ("##END CENTERS:\n"));
	  fprintf_vctr(stderr, "end_centers", best_ctrs, "\n");
	  INSTR_PRINTF(D_SHOW_END_CENTERS, ("##END CENTERS END.\n"));
	  INSTR_PRINTF(D_SHOW_END_CENTERS, ("##DISTORTION %f\n",
										kmeans_distortion(kd, best_ctrs, &caching_parm)));
	  INSTR_PRINTF(D_SHOW_END_CENTERS, ("##BICSCORE %f\n",
										bic_score(kd, best_ctrs, &caching_parm, NULL)));
	}

	if(strcmp(printclusters, "")) {
      kmeans_printclusters(kd, best_ctrs, &parm, printclusters);
	}

	if(strcmp(save_ctrs, "")) /* save ctrs to file */
	  save_vctr_to_filename(best_ctrs, save_ctrs);

	free_vctr(best_ctrs);
	if(ctrs)
	  free_vctr(ctrs);
  }

  if(kd) {
	free_cache_entries(kd);
	free_simple_kdtree(kd);
  }
  if(parm.data_list) {
    free_dyvp_array(parm.data_list);
  }
}

void paint_points_in_kdnode(const simple_kdtree *kd)
{
  if(is_kd_leaf(kd)) {
	int i;
	for(i=0; i<kd->num_points; i++) {
	  const dyv *point = dyv_array_ref(kd->points, i);
	  ag_pixel(dyv_ref(point,0),dyv_ref(point,1));
	}
  } else {
	paint_points_in_kdnode(kd->left);
	paint_points_in_kdnode(kd->right);
  }
}

int paint_points_in_hrect(const dym *data, const hrect *hr)
{
  int i;
  int num_points = dym_rows(data);
  int in_hrect;
  for(i=in_hrect=0; i<num_points; i++) {
    dyv *point = mk_dyv_from_dym_row(data,i);
	if(is_in_hrect(hr, point)) {
	  ag_pixel(dyv_ref(point,0),dyv_ref(point,1));
	  in_hrect++;
	}
	free_dyv(point);
  }
  return in_hrect;
}

int print_points_in_kdtree(FILE *fp, const simple_kdtree *kd, bool just_ids)
{
  if(is_kd_leaf(kd)) {
	int i;
	for(i=0; i<kd->num_points; i++) {
      if(just_ids)
        fprintf(fp, "+%d\n", ivec_ref(kd->point_ids, i));
      else {
        const dyv *point = dyv_array_ref(kd->points, i);
        fprintf_oneline_dyv(fp, "+", point, "\n");
      }
	}
	return kd->num_points;
  } else
	return print_points_in_kdtree(fp, kd->left, just_ids) + print_points_in_kdtree(fp, kd->right, just_ids);
}

ivec *mk_sivec_points_in_kdtree(const simple_kdtree *kd)
{
  ivec *ret = mk_ivec(0);
  collect_points_in_kdtree(&ret, kd);
  return ret;
}

void collect_points_in_kdtree(ivec **ret, const simple_kdtree *kd)
{
  if(is_kd_leaf(kd)) {
    /* remember the point_ids ivec is padded after num_points, and in
     general not guaranteed to be a sivec
    */
    ivec *pts_in_leaf = mk_copy_ivec_subset(kd->point_ids, 0, kd->num_points);
    ivec *punion;
    ivec_sort(pts_in_leaf, pts_in_leaf);
    punion  = mk_sivec_union(*ret, pts_in_leaf);
    free_ivec(*ret);
    free_ivec(pts_in_leaf);
    *ret = punion;
    return;
  } else {
    collect_points_in_kdtree(ret, kd->left);
    collect_points_in_kdtree(ret, kd->right);
    return;
  }
}

#if WANT_MK_RANDSTART_CENTERS_SLOW
vctr *mk_randstart_centers_slow(dym *x, int k, int sample_size,
								ivec *allowed_subset)
{
  vctr *centers = mk_empty_vctr();
  int curr_center;
  int num_points = allowed_subset ? ivec_size(allowed_subset) : dym_rows(x);
  int num_dims = dym_cols(x);
  dyv *zerov = mk_constant_dyv(num_dims, 0.0); /* makeshift value */
  /* select random point as first center */
  {
	int random_idx = int_random(num_points);
	dyv *center0;
	if(allowed_subset)
	  random_idx = ivec_ref(allowed_subset, random_idx);
	center0 = mk_dyv_from_dym_row(x, random_idx);
    add_to_vctr(centers, center0);
    free_dyv(center0);
  }

  /* each other center is taken to be furthest, among randomly chosen
	 set of points, from existing centers */
  for(curr_center = 1; curr_center < k; curr_center++) {
	int nsamples;
	double maximin_dsqd = -HUGE_VAL;
    add_to_vctr(centers, zerov);
	for(nsamples = 0; nsamples < sample_size; nsamples++) {
	  int commited_center;
	  double min_dsqd = HUGE_VAL;
	  int random_idx = int_random(num_points);
	  dyv *random_point;
	  if(allowed_subset)
		random_idx = ivec_ref(allowed_subset, random_idx);
	  random_point = mk_dyv_from_dym_row(x, random_idx);
	  /* calc minimal distance of random_point to any commited center */
	  for(commited_center = 0; commited_center < curr_center; commited_center++) {
		double dsqd = dyv_dsqd(vctr_ref(centers, commited_center), random_point);
		if(dsqd < min_dsqd) {
		  min_dsqd = dsqd;
		}
	  }
	  /* this minimal distance greater then current? use random_point as center */
	  if(min_dsqd > maximin_dsqd) {
		 maximin_dsqd = min_dsqd;
		 move_vctr(centers, curr_center, random_point, 0.0);
	  }
	free_dyv(random_point);
	}
	/* commit to center curr_center at this point */
  }

  free_dyv(zerov);
  return centers;
}

#endif  /* WANT_MK_RANDSTART_CENTERS_SLOW */

dym *mk_dym_from_points_in_kdtree(simple_kdtree *kd, const int ndims)
{
  dym *ret = mk_zero_dym(0, ndims);
  mk_dym_from_points_in_kdtree_recursive(kd, ret);
  return ret;
}

static void mk_dym_from_points_in_kdtree_recursive(simple_kdtree *kd, dym *x)
{
  if(is_kd_leaf(kd)) {
	int i;
	for(i=0; i<kd->num_points; i++) {
	  add_row(x);
	  copy_dyv_to_dym_row(dyv_array_ref(kd->points, i),x,dym_rows(x)-1);
	}
  } else {
	mk_dym_from_points_in_kdtree_recursive(kd->left, x);
	mk_dym_from_points_in_kdtree_recursive(kd->right, x);
  }
}

void membership_main(int argc, char *argv[])
{
  if ( argc < 4 )
    printf("Usage: kmeans membership -in <datafilename>\n");
  else {
	/* The input contains "random" center ids,
	   in ctrnums we maintain an ivec where in index i it has the id
	   of ctr number i.
	   In ctrs[i] we keep the ivec with the points belonging to that center */
	ivec *ctrnums = mk_ivec(0);
	ivec **ctrs;
    const char *filename = string_from_args("in", argc, argv, "default.csv");
	FILE *fin;
	char buf[LINELEN];
	int numctrs, numpoints, currctr, i, j;
	int lastctr = 0;

	/* first pass --- build ctrnum and determine how many distinct centers are there */
	if(NULL == (fin = fopen(filename, "r")))
	  my_errorf("Can't open file %s", filename);
	while(fgets(buf, LINELEN, fin)) {
	  int ctrnum;
	  if(buf[0] != '+')			/* ignore lines w/no membership info */
		continue;
	  if(buf[1] != '#')			/* ignore lines w/ points */
		continue;
	  if(1 != sscanf(buf, "+#%d", &ctrnum)) {
		fprintf(stderr, "Incomprehensible line %s", buf);
		continue;
	  }
	  /* add it to ctrnums if not already there */
	  if(-1 == (lastctr = find_index_in_ivec_hint(ctrnums, ctrnum, lastctr))) {
		add_to_ivec(ctrnums, ctrnum);
		lastctr = ivec_size(ctrnums)-1;
	  }
	}
	fclose(fin);
	numctrs = ivec_size(ctrnums);
	fprintf(stderr, "%d centers...", numctrs);
	/* second pass --- build the dyv_arrays */
	ctrs = AM_MALLOC_ARRAY(ivec *, numctrs);
	memset(ctrs, 0, numctrs * sizeof(*ctrs));
	if(NULL == (fin = fopen(filename, "r")))
	  my_errorf("Can't open file %s", filename);
	currctr = -1;
	lastctr = 0;
	numpoints = 0;
	while(fgets(buf, LINELEN, fin)) {
	  if(buf[0] != '+')			/* ignore lines w/no membership info */
		continue;
	  if(buf[1] == '#') {		/* a line with a center */
		if(1 != sscanf(buf, "+#%d", &currctr)) {
		  fprintf(stderr, "Incomprehensible line %s", buf);
		  continue;
		}
	  } else {
		/* a line with a point */
		int ctridx;
		int point_id;
		/* want to check here if currctr points to right place already */
		if(-1 == (ctridx = find_index_in_ivec_hint(ctrnums, currctr, lastctr))) {
		  my_errorf("Oops, can't find center #%d in line %s", currctr, buf);
		}
		if(!ctrs[ctridx]) {
		  ctrs[ctridx] = mk_ivec(0);
        }
        if(1 != sscanf(&buf[1], "%d", &point_id)) {
		  fprintf(stderr, "Incomprehensible line %s", buf);
        }
		add_to_ivec(ctrs[ctridx], point_id);
		numpoints++;
		lastctr = ctridx;
		continue;
	  }
	}
	fclose(fin);

	/* output */
	fprintf(stderr, "%d points...\n", numpoints);
	for(i=0; i<numctrs; i++) {
	  for(j=0; j<ivec_size(ctrs[i]); j++)
		fprintf(stdout, "%d\n", ivec_ref(ctrs[i], j));
	  putchar('\n');
	}

	/* clean up */
	for(i=0; i<numctrs; i++)
	  if(ctrs[i])
		free_ivec(ctrs[i]);
	AM_FREE_ARRAY(ctrs, ivec *, numctrs);
	free_ivec(ctrnums);
  }
}

double AD_critical_value(double alpha)
{

  /* Table for critical values for the corrected Anderson-Darling statistic.
     Taken from:
     Gregory Hamerly:
     Learning structure and concepts in data through data clustering.
     PhD thesis, UCSD, 2003.
  */
  static const double crit[][2] = {
    { 0.5, 0.3416 },
    { 0.6, 0.3843 },
    { 0.7, 0.4373 },
    { 0.75, 0.4706 },
    { 0.8, 0.5110 },
    { 0.85, 0.5632 },
    { 0.9, 0.6329 },
    { 0.95, 0.7544 },
    { 0.975, 0.8746 },
    { 0.98, 0.9163 },
    { 0.99, 1.0412 },
    { 0.995, 1.1625 },
    { 0.999, 1.4607 },
    { 0.9995, 1.5943 },
    { 0.9999, 1.8195 },
  };

  static const int crit_len = sizeof(crit)/sizeof(crit[0]);

  int best_match_idx, i;
  double best_match_dist = -1;       /* difference from alpha to the best value we have */

  /* I don't want to perform hard comparisons on doubles - they
     might not work as expected, depending on machine representation.
     Instead I'm searching for the closest match. This also has a nice
     "soft failure" in case the user did not specify a value that
     exists in the table (we just pick another close value).
  */

  /* we do a linear search - the table is quite small */
  for(best_match_idx = -1, i = 0; i<crit_len; i++) {
    if(best_match_idx == -1 || (fabs(crit[i][0] - alpha) < best_match_dist)) {
      best_match_idx = i;
      best_match_dist = fabs(crit[i][0] - alpha);
    }
  }

  if(best_match_dist > 1E-10) {
    fprintf(stderr, "Warning: using confidence value for alpha = %g\n",
            crit[best_match_idx][0]);
  }

  return crit[best_match_idx][1];
}
