/*
   File:        traversal.c
   Author:      Andrew W. Moore, Dan Pelleg
   Created:     Sun Apr  5 11:49:13 EDT 1998
   Description: Smart "blacklisting" traversal of kd-tree
   $Id: traversal.c,v 1.26 2004/01/18 16:46:54 dpelleg Exp $

   Copyright 1998, Carnegie Mellon University
*/

#include <math.h>
#include <assert.h>
#include "kmeans.h"
#include "traversal.h"
#include "split.h"
#include "debug.h"
#include "amgr.h"

static const char *const CVSID = "$Id: traversal.c,v 1.26 2004/01/18 16:46:54 dpelleg Exp $";

static void blacklist_aprx_update_sums(kmeans_state *kmstate, ivec *candidates,
									   means *means);

static bool exists_ctr_with_zombie_children(const vctr *vcenters, const ivec *parents);

/* the "means" funcs for blacklist_traversal */
/* update nums/sums to reflect the
   centroids of points node and all descendants */
static bool means_prune(const simple_kdtree *node, int closest_ctr, const traversal_stats *stats);

cached_stats *mk_cached_stats(const ivec *candidates)
{
  int i;
  int nowners = ivec_size(candidates);
  cached_stats *ret = AM_MALLOC(cached_stats);
  if(nowners > KDCACHE_MAXOWNERS)
	my_error("Too many owners in cached_stats");
  for(i=0; i<KDCACHE_MAXOWNERS; i++)
	ret->owners[i] = i<nowners ? ivec_ref(candidates, i) : KDCACHE_NO_OWNER;
  ret->nhits = 0;
  return ret;
}

cached_stats *mk_cp_cached_stats(const cached_stats *other)
{
  cached_stats *ret = AM_MALLOC(cached_stats);
  memcpy(ret->owners, other->owners, sizeof(ret->owners));
  ret->nhits = -1;
  return ret;
}

void free_cached_stats(cached_stats *obj)
{
  DBG_PRINTF(D_KDCACHE, ("Hit count %d\n", obj->nhits));
  AM_FREE(obj, cached_stats);
}

bool cached_owners_match(cached_stats *stats, const ivec *current_owners)
{
  int i;
  const int co_sz = ivec_size(current_owners);
#ifndef AMFAST
  if(!is_sivec(current_owners)) {
	my_error("current_owners should be sorted");
  }
#endif /* AMFAST */
  if (co_sz > KDCACHE_MAXOWNERS)
	return FALSE;
  /* so now co_sz <= KDCACHE_MAXOWNERS */
  for(i=0; i<co_sz; i++) {
	if(stats->owners[i] == KDCACHE_NO_OWNER) /* last in owners reached */
	  return FALSE;
	if(stats->owners[i] != ivec_ref(current_owners, i))
	  return FALSE;
  }
  /* here, for all i <co_sz, state_owners[i] == current_owners[i] */
  return co_sz == KDCACHE_MAXOWNERS;

}

ivec *mk_refine_owners(const hrect *hr, const ivec *considerables, const vctr *centers)
{
  ivec *refined = mk_ivec(0);
  int num_centers = ivec_size(considerables);
  dyv *distances = mk_constant_dyv(num_centers, -1.0);

  /* first, calculate distances of centers to hrect and pick
	 a center which has minimal distance to hrect */
  double closest_dsqd;
  int closest, closest_idx_in_considerables, i;
  dyv *temp = mk_zero_dyv(hrect_size(hr));
  DBG_PRINTF(D_TRACE, ("Entering mk_refine_owners\n"));
  INSTR_COUNTER_INC(7);
  for(i = 0, closest_dsqd = HUGE_VAL, closest = closest_idx_in_considerables = -1;
	  i<num_centers;
	  i++) {
	int ci = ivec_ref(considerables, i);
	double dist_ci_to_hrect = dsqd_hrect_point(hr, vctr_ref(centers, ci), temp);
	INSTR_COUNTER_INC(8);
	dyv_set(distances, i, dist_ci_to_hrect);
	if(dist_ci_to_hrect < closest_dsqd) {
	  closest_idx_in_considerables = i;
	  closest = ci;
	  closest_dsqd = dist_ci_to_hrect;
	}
  }
  free_dyv(temp); temp = 0;

  DBG_PRINTF(D_TRACE, ("Closest is %d\n", closest));

  /* for each center, does it compete with our candidate over points? */
  /* note this loop will surely add closest to refined
	 at some point (but we're sure it'll be placed so that the result is
	 sorted ) */
  for(i = 0; i<num_centers; i++) {
	const int ci = ivec_ref(considerables, i);
	DBG_PRINTF(D_TRACE, ("looking at center #%d\n", ci));
	if((dyv_ref(distances, i) == 
		dyv_ref(distances, closest_idx_in_considerables))
	  || !candidate_is_full_owner(hr, centers, closest, ci))
	  add_to_ivec(refined, ci); /* we have a competitor */
  }

  free_dyv(distances);
  DBG_PRINTF(D_TRACE, ("Exiting mk_refine_owners\n"));
  return refined;
}

static bool exists_ctr_with_zombie_children(const vctr *vcenters, const ivec *parents)
{
  int i;
  for(i=0; i<ivec_size(parents); i++) {
	int ci = ivec_ref(parents, i);
	if(has_dead_children(vcenters, ci))
	  return TRUE;
  }
  return FALSE;
}

#ifdef WANT_ALL_CTRS_WITH_ZOMBIE_CHILDREN
static bool all_ctrs_with_zombie_children(const vctr *vcenters, const ivec *parents);

bool all_ctrs_with_zombie_children(const vctr *vcenters, const ivec *parents)
{
  int i;
  for(i=0; i<ivec_size(parents); i++) {
	int ci = ivec_ref(parents, i);
	if(!has_dead_children(vcenters, ci))
	  return FALSE;
  }
  return TRUE;
}

#endif /* WANT_ALL_CTRS_WITH_ZOMBIE_CHILDREN */

void cached_bl_traversal(kmeans_state *kmstate,
						 const ivec *candidates, traversal_stats *stats,
						 const kmeans_parm *parm)
{
  simple_kdtree *node = kmstate->kdtree; /* shorthand */

  /* non-caching mode --- just use blacklist_traversal */
  if((parm->usecache == CACHE_NONE) || kmstate->phase == PHS_PARENTS) {
	blacklist_traversal(kmstate, candidates, stats, parm);
	return;
  }

  /* if have cached results, use them */
  if(node->cached) {
	if(cached_owners_match(node->cached, candidates)) {
	  node->cached->nhits++;	/* update hit counter */
	  DBG_PRINTF(D_KDCACHE, ("Cache hit: node %x, %d cands:\n", node, ivec_size(candidates)));
	  if(DEBUG_FLAG(D_DRAWPOINTS)) {
		ag_set_pen_color(AG_BLUEGREEN);
		ag_hrect(node->hr);
	  }

	  /* update stats from cache entry */
	  stats->apply_cache(node->cached, stats);

	  return;
	} else {					/* if cache miss, can get rid of this cache entry */
	  DBG_PRINTF(D_KDCACHE, ("Cache miss: node %x, %d cands:\n", node, ivec_size(candidates)));

	  /* if one of the candidates has zombie children, we keep the entry */
	  /* hoping it will be used next time they're resurrected */
	  if(!exists_ctr_with_zombie_children(kmstate->current_centers, candidates)
         && (parm->usecache == CACHE_RDWR)) {
		stats->free_cached_stats(node->cached);
		node->cached = 0;		/* BUGBUG? will never help?  */
	  }
	}
  }
  /* no cached results: if node profile doesn't fit caching, just call 
	 blacklist_traversal */
  if(ivec_size(candidates) > KDCACHE_MAXOWNERS
     || node->cached
     || (parm->usecache != CACHE_RDWR)) {
	blacklist_traversal(kmstate, candidates, stats, parm);
	return;
  }
  /* so node profile fits caching */
  /* remember what the stats looked like before doing this node ... */

  {
	cached_stats *prestats;
	kmeans_parm parm2;

	assert(node->cached == NULL);
	prestats = stats->mk_stat_snapshot(stats, candidates);
	/* scan this node, not caching anything... */
	memcpy(&parm2, parm, sizeof(parm2));
	parm2.usecache = CACHE_NONE;
	blacklist_traversal(kmstate, candidates, stats, &parm2);
	/* place the cache entry... */
	node->cached = stats->mk_cache_entry(prestats, stats);
	DBG_PRINTF(D_KDCACHE, ("Cache write: node %x, %d cands\n", node, ivec_size(candidates)));
	/* cleanup */
	stats->free_cached_stats(prestats);
  }
}

void blacklist_traversal(kmeans_state *kmstate,
						 const ivec *candidates, traversal_stats *stats,
						 const kmeans_parm *parm)
{
  vctr *centers = kmstate->current_centers; /* shorthand */
  simple_kdtree *node = kmstate->kdtree; /* shorthand */
  INSTR_COUNTER_INC(9);
  DBG_PRINTF(D_TRACE, ("Entered blacklist_traversal, node %p\n", node));
  if(is_kd_leaf(node)) {   /* if node is a leaf, do the obvious */
	int i;
	int num_points = dyv_array_size(node->points);

	for(i=0; i<num_points; i++) {
	  const int point_id = ivec_ref(node->point_ids, i);
	  const dyv *x_i = dyv_array_ref(node->points, i);
	  int j;
	  j = find_closest_candidate(x_i, candidates, centers);
	  if(kmstate->phase == PHS_PARENTS) {
		/* let children fight over this point */
		ivec *children = mk_vctr_children(centers, j);
		j = find_closest_candidate(x_i, children, centers);
		free_ivec(children);
	  }
	  stats->update_point(point_id, x_i, j, stats);
	}
  } else {
	ivec *updated_owners = mk_refine_owners(node->hr, candidates, centers);

	/* found an owner of all points */
	/* check whether to prune or to do cumulative updates */
	if((ivec_size(updated_owners) == 1)
	   && stats->prune(node, ivec_ref(updated_owners, 0), stats)) {
	  int owner_idx = ivec_ref(updated_owners, 0);

	  if(kmstate->phase != PHS_PARENTS) {
		INSTR_COUNTER_INC(2);
		stats->update_bulk(node, owner_idx, stats);
	  } else {					/* let the children fight for this node */
		ivec *children = mk_vctr_children(centers, owner_idx);
		kmeans_state state;
		memcpy(&state, kmstate, sizeof(state));/* new state almost identical */
		state.phase = PHS_CHILDREN;
		cached_bl_traversal(&state, children, stats, parm);
		free_ivec(children);
	  }
	} else if(ivec_size(updated_owners) > 0) { /* no single owner  */
	  /* go down a level */
	  kmeans_state state;
	  memcpy(&state, kmstate, sizeof(state)); /* new state almost identical */
	  INSTR_COUNTER_INC(3);
	  state.kdtree = kmstate->kdtree->left;
	  cached_bl_traversal(&state, updated_owners, stats, parm);
	  state.kdtree = kmstate->kdtree->right;
	  cached_bl_traversal(&state, updated_owners, stats, parm);
	}
	/* notice the third case is zero owners, in which we do nothing (and prune) */
	free_ivec(updated_owners);
  }
  DBG_PRINTF(D_TRACE, ("Exiting blacklist_traversal, node %p\n", node));
}

nodestats kmeans_blacklist_mk_sums(kmeans_state *kmstate, const kmeans_parm *parm)
{
  /* for the cost calculation */
  const int M = kmstate->current_centers->ndims;
  int num_centers = 1+vctr_max_pidx(kmstate->current_centers);

  traversal_stats stats = {
	nodestats_update_point,			/* func pointers */
	nodestats_update_bulk,
	means_prune,
	nodestats_mk_snapshot,
	nodestats_mk_cache_entry,
	nodestats_apply_cache,
	nodestats_free_cache,
	NULL,						/* placeholder for centers */
	{0, 0, 0}					/* stats placeholder */
  };
  stats.centers = kmstate->current_centers;
  stats.stat = nodestats_mk_stats(num_centers, M, parm->maintain_memb);

  /* reset counters */
  INSTR_RESET_COUNTER(1);			/* no. of leaves */
  INSTR_RESET_COUNTER(2);			/* nodes lucky enough to have a single owner */
  INSTR_RESET_COUNTER(3);			/* unlucky, non-leaf, nodes */
  INSTR_RESET_COUNTER(4);			/* #times candidate_is_full_owner called */
  INSTR_RESET_COUNTER(9);			/* #times blacklist_update_sums called */
  INSTR_RESET_COUNTER(6);			/* #closest centers considered for leaves */
  INSTR_RESET_COUNTER(7);			/* #times mk_refine_owners called */
  INSTR_RESET_COUNTER(8);			/* #total centers considered by mk_refine_owners */

  if(kmstate->phase == PHS_UNPHASED) {
	ivec *all_ctrs = mk_live_ctrs(kmstate->current_centers);
	cached_bl_traversal(kmstate, all_ctrs, &stats, parm);
	free_ivec(all_ctrs);
  } else {
	cached_bl_traversal(kmstate, kmstate->phase1_ctrs, &stats, parm);
  }
  DBG_PRINTF(D_SHOWCOUNTERS, ("#leaves: %d\n"
							  "#candidate_is_full_owner tests: %d\n"
							  "#nodes visited: %d\n",
							  INSTR_COUNTER_VAL(1),
							  INSTR_COUNTER_VAL(4),
							  INSTR_COUNTER_VAL(9)));
  DBG_PRINTF(D_SHOWCOUNTERS, ("#centers scanned by leaves: %d\n"
							  "#mk_refine_owners calls: %d\n"
							  "#centers scanned by mk_refine_owners:%d\n",
							  INSTR_COUNTER_VAL(6),
							  INSTR_COUNTER_VAL(7),
							  INSTR_COUNTER_VAL(8)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##cost: %d\n",
								INSTR_COUNTER_VAL(4)*M +
								INSTR_COUNTER_VAL(8)*M +
								INSTR_COUNTER_VAL(6)*M));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#avg. len candidate list (nodes): %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(8))/
								(INSTR_COUNTER_VAL(7) || 1)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#avg. len candidate list (leaves): %4.2f\n",
								1.0*(INSTR_COUNTER_VAL(6))/(INSTR_COUNTER_VAL(1) || 1)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##avg. len total: %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(6)+INSTR_COUNTER_VAL(8))
							  /((INSTR_COUNTER_VAL(1)+INSTR_COUNTER_VAL(7)) || 1)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##DISTORTION %f\n",
							  kmeans_distortion(kmstate->kdtree, kmstate->current_centers, parm)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##BICSCORE %f\n",
								bic_score(kmstate->kdtree, 
										  kmstate->current_centers, parm, NULL)));
  return stats.stat;
}

void kmeans_blacklist_aprx_find_sums(kmeans_state *kmstate, means *means)
{
  /* for the cost calculation */
  const int M = kmstate->current_centers->ndims;
  ivec *all_centers = mk_live_ctrs(kmstate->current_centers);
  /* reset counters */
  INSTR_RESET_COUNTER(1);			/* no. of leaves */
  INSTR_RESET_COUNTER(2);			/* nodes lucky enough to have a single owner */
  INSTR_RESET_COUNTER(3);			/* unlucky, non-leaf, nodes */
  INSTR_RESET_COUNTER(4);			/* #times candidate_is_full_owner called */
  INSTR_RESET_COUNTER(9);			/* #times blacklist_update_sums called */
  INSTR_RESET_COUNTER(6);			/* #closest centers considered for leaves */
  INSTR_RESET_COUNTER(7);			/* #times mk_refine_owners called */
  INSTR_RESET_COUNTER(8);			/* #total centers considered by mk_refine_owners */

  blacklist_aprx_update_sums(kmstate, all_centers, means);
  free_ivec(all_centers);
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#leaves: %d\n"
							  "#candidate_is_full_owner tests: %d\n"
							  "#nodes visited: %d\n",
							  INSTR_COUNTER_VAL(1),
							  INSTR_COUNTER_VAL(4),
							  INSTR_COUNTER_VAL(9)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#centers scanned by leaves: %d\n"
							  "#mk_refine_owners calls: %d\n"
							  "#centers scanned by mk_refine_owners:%d\n",
							  INSTR_COUNTER_VAL(6),
							  INSTR_COUNTER_VAL(7),
							  INSTR_COUNTER_VAL(8)));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##cost: %d\n",
								INSTR_COUNTER_VAL(4)*M +
								INSTR_COUNTER_VAL(8)*M +
								INSTR_COUNTER_VAL(6)*M));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#avg. len candidate list (nodes): %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(8))/(INSTR_COUNTER_VAL(7))));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("#avg. len candidate list (leaves): %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(6))/(INSTR_COUNTER_VAL(1))));
  INSTR_PRINTF(D_SHOWCOUNTERS, ("##avg. len total: %4.2f\n",
							  1.0*(INSTR_COUNTER_VAL(6)+INSTR_COUNTER_VAL(8))
							  /(INSTR_COUNTER_VAL(1)+INSTR_COUNTER_VAL(7))));
}

static void blacklist_aprx_update_sums(kmeans_state *kmstate, ivec *candidates,
                                       means *means)
{
  vctr *centers = kmstate->current_centers; /* shorthand */
  simple_kdtree *node = kmstate->kdtree; /* shorthand */
  INSTR_COUNTER_INC(9);
  /* if thin enough, do approximately */
  if(node->bias <= kmstate->b_cutoff) {
	/* every center gets a fraction of the pts, positioned at closest(c, hr) */
	int j;
	ivec *updated_owners = mk_refine_owners(node->hr, candidates, centers);
	int num_centers = ivec_size(updated_owners);
	int pts_used;
	
	for (pts_used = 0, j=0; j<num_centers; j++) {
	  int c_j = ivec_ref(updated_owners, j);
	  dyv *sum_j = dyv_array_ref(means->sums, c_j);
	  /* contrib_j is the contribution of node to the j'th center */
	  dyv *contrib_j = mk_copy_dyv(vctr_ref(centers, c_j));
	  /* we make rougly equal #points, last center gets remainder */
	  int npoints = (j == (num_centers - 1)) ?
		(node->num_points - pts_used) :
		node->num_points/num_centers;
	  pts_used += npoints;

	  /* find closest(center, hr) */
	  clip_to_inside_hrect(node->hr, contrib_j);

	  /* account for as many points as needed */
	  dyv_scalar_mult(contrib_j, npoints, contrib_j);
	  /* add to sum for this center */
	  dyv_plus(contrib_j, sum_j, sum_j);
	  ivec_increment(means->nums, c_j, npoints);
	  free_dyv(contrib_j);
	}
	if(pts_used != node->num_points)
	  my_error("#pts mismatch");
	free_ivec(updated_owners);
	return;
  }

  if(is_kd_leaf(node)) {   /* if node is a leaf, do the obvious */
	int i;
	int num_points = dyv_array_size(node->points);

	for(i=0; i<num_points; i++) {
	  const dyv *x_i = dyv_array_ref(node->points, i);
	  int j = find_closest_candidate(x_i, candidates, centers);
	  dyv *sum_j = dyv_array_ref(means->sums, j);
	  INSTR_COUNTER_INC(1);
	  dyv_plus(x_i, sum_j, sum_j);
	  ivec_increment(means->nums, j, 1);
	  if(DEBUG_FLAG(D_DRAWPOINTS)) {
		ag_set_pen_color(center_color(j));
		ag_pixel(dyv_ref(x_i,0),dyv_ref(x_i,1));
	  }
	}
  } else {
	ivec *updated_owners = mk_refine_owners(node->hr, candidates, centers);

	if(ivec_size(updated_owners) <= 1) { /* found an owner of all points */
								/* do comulative updates    */
	  int owner = ivec_ref(updated_owners, 0);
	  dyv *sum_owner = dyv_array_ref(means->sums, owner);
	  INSTR_COUNTER_INC(2);
	  dyv_plus(node->sum, sum_owner, sum_owner);
	  ivec_increment(means->nums, owner, node->num_points);
	  if(DEBUG_FLAG(D_DRAWPOINTS)) {
		ag_set_pen_color(center_color(owner));
		ag_hrect(node->hr);
		paint_points_in_kdnode(node);
	  }
	} else {					/* no single owner */
	  /* go down a level */
	  kmeans_state state;
	  INSTR_COUNTER_INC(3);
	  memcpy(&state, kmstate, sizeof(state)); /* new state almost identical */
	  state.kdtree = kmstate->kdtree->left;
	  blacklist_aprx_update_sums(&state, updated_owners, means);
	  state.kdtree = kmstate->kdtree->right;
	  blacklist_aprx_update_sums(&state, updated_owners, means);
	}
	free_ivec(updated_owners);
  }
}

double kmeans_distortion(simple_kdtree *kd,
						 vctr *centers, const kmeans_parm *parm)
{
  /* sum entries in dist vector to get answer */
  dist_info distinfo = mk_kmeans_distinfo(kd, centers, parm);
  /* we rely on unused entries being zero */
  double ret = dyv_sum(distinfo.distortion);
  distortion_free_stats(&distinfo);
  return ret;
}

dist_info mk_kmeans_distinfo(simple_kdtree *kd,
							 vctr *centers, const kmeans_parm *parm)
{
  const int ndims = centers->ndims;
  int num_centers = 1+vctr_max_pidx(centers);
  ivec *all_centers = mk_live_ctrs(centers);
  dist_info ret;

  kmeans_state kmstate;
  traversal_stats stats;

  kmstate.kdtree = kd;
  kmstate.current_centers = centers;
  kmstate.b_cutoff = 0.0;		/* unused */
  kmstate.phase = PHS_UNPHASED;
  kmstate.phase1_ctrs = 0;		/* unused */

  stats.update_point = nodestats_update_point;
  stats.update_bulk = nodestats_update_bulk;
  stats.prune = distortion_prune;
  stats.mk_stat_snapshot = nodestats_mk_snapshot;
  stats.mk_cache_entry = nodestats_mk_cache_entry;
  stats.apply_cache = nodestats_apply_cache;
  stats.free_cached_stats = nodestats_free_cache;
  stats.centers = centers;
  stats.stat = nodestats_mk_stats(num_centers, ndims, parm->maintain_memb);

  cached_bl_traversal(&kmstate, all_centers, &stats, parm);

  ret.distortion = stats.stat.distortion;
  ret.nums = stats.stat.nums;
  if(stats.stat.memb) {
    free_ivec_array(stats.stat.memb);
  }
  free_dyv_array(stats.stat.sums);
  free_ivec(all_centers);
  return ret;
}

void kmeans_printclusters(simple_kdtree *kd,
                          vctr *centers, const kmeans_parm *orig_parm,
                          const char *out_filename)
{
  kmeans_parm noncaching_parm;
  ivec *all_centers = mk_live_ctrs(centers);

  kmeans_state kmstate;
  traversal_stats stats;

  /* make a parm version that never uses cache */
  memcpy(&noncaching_parm, orig_parm, sizeof(noncaching_parm));
  noncaching_parm.usecache = FALSE;

  kmstate.kdtree = kd;
  kmstate.current_centers = centers;
  kmstate.b_cutoff = 0.0;		/* unused */
  kmstate.phase = PHS_UNPHASED;
  kmstate.phase1_ctrs = 0;		/* unused */

  stats.update_point = nodestats_print_point;
  stats.update_bulk = nodestats_print_bulk;
  stats.prune = distortion_prune;

  /* those value are probably not needed for the print funcs */
  stats.mk_stat_snapshot = nodestats_mk_snapshot;
  stats.mk_cache_entry = nodestats_mk_cache_entry;
  stats.apply_cache = nodestats_apply_cache;
  stats.free_cached_stats = nodestats_free_cache;
  stats.centers = centers;
  /* no need to keep storage for stats */
  stats.stat.distortion = NULL;
  stats.stat.nums = NULL;
  stats.stat.sums = NULL;
  stats.aux = safe_fopen(out_filename, "w");

  cached_bl_traversal(&kmstate, all_centers, &stats, &noncaching_parm);

  fclose(stats.aux);
  free_ivec(all_centers);
}

void distortion_update_point(int point_id, dyv *point,
							 int closest_ctr, traversal_stats *stats)
{
  dyv *distortion = stats->stat.distortion;
  vctr *centers = stats->centers;
  const int pidx = vctr_pidx(centers, closest_ctr);

  dyv_increment(distortion, pidx,
				dyv_dsqd(vctr_ref(centers, closest_ctr), point));
  ivec_increment(stats->stat.nums, pidx, 1);
}


void distortion_update_bulk(simple_kdtree *node, int closest_ctr,
							traversal_stats *stats)
{
  dyv *owner = vctr_ref(stats->centers, closest_ctr);
  double mag_sqd_owner = dyv_scalar_product(owner, owner);
  double dist_contrib = 0.0;
  const int pidx = vctr_pidx(stats->centers, closest_ctr);
  /* add contributions of owner squared */
  dist_contrib += node->num_points*mag_sqd_owner;

  /* add contributions of dot products */
  dist_contrib -= 2*dyv_scalar_product(owner, node->sum);

  /* add contributions of magnitudes moment */
  dist_contrib += node->sum_mags;
  dyv_increment(stats->stat.distortion, pidx, dist_contrib);
  ivec_increment(stats->stat.nums, pidx, node->num_points);
}

bool distortion_prune(const simple_kdtree *node, int closest_ctr, 
					  const struct traversal_stats *stats)
{
  return TRUE;
}

nodestats nodestats_mk_stats(int ncenters, int ndims, bool with_memb)
{
  nodestats ret;
  ret.distortion = mk_zero_dyv(ncenters);
  ret.nums = mk_zero_ivec(ncenters);
  ret.sums = mk_dyv_array_of_zeroes(ncenters, ndims);
  ret.memb = with_memb ? mk_array_of_empty_ivecs(ncenters) : NULL;
  return ret;
}

void distortion_free_stats(dist_info *dinfo)
{
  free_dyv(dinfo->distortion); dinfo->distortion = 0;
  free_ivec(dinfo->nums); dinfo->nums = 0;
}

void nodestats_free_stats(nodestats *ninfo)
{
  free_dyv_array(ninfo->sums); ninfo->sums = 0;
  free_dyv(ninfo->distortion); ninfo->distortion = 0;
  free_ivec(ninfo->nums); ninfo->nums = 0;
  if(ninfo->memb) {
    free_ivec_array(ninfo->memb);
    ninfo->memb = 0;
  }
}

cached_stats *nodestats_mk_snapshot(const traversal_stats *stats, const ivec *candidates)
{
  int i;
  const int ndims = stats->centers->ndims;
  const int ncands = ivec_size(candidates);
  cached_stats *ret = mk_cached_stats(candidates);
  const bool with_memb = (stats->stat.memb != NULL);
  ret->stat = nodestats_mk_stats(KDCACHE_MAXOWNERS, ndims, with_memb);
#ifndef AMFAST
  if(!is_sivec(candidates))
	my_error("candidates should be sorted");
#endif /* AMFAST */
  for(i=0; i<ncands; i++) {
	  const int ci = ivec_ref(candidates, i);
	  const int pidx = vctr_pidx(stats->centers, ci);
	  const double curr_distortion = dyv_ref(stats->stat.distortion, pidx);
	  const int curr_nums = ivec_ref(stats->stat.nums, pidx);
	  dyv *curr_sum = dyv_array_ref(stats->stat.sums, pidx);
	  dyv_set(ret->stat.distortion, i, curr_distortion);
	  ivec_set(ret->stat.nums, i, curr_nums);
	  dyv_array_set(ret->stat.sums, i, curr_sum);
      if(with_memb) {
        const ivec *curr_memb = ivec_array_ref(stats->stat.memb, pidx);
        ivec_array_set(ret->stat.memb, i, curr_memb);
        /* we don't need to free curr_memb. In fact it would be wrong
           to do so since it's held by stats->stat.memb and will be
           freed when it is.
        */
      }
  }
  ret->nhits = -1;
  return ret;
}

cached_stats *nodestats_mk_cache_entry(const cached_stats *prestats, const traversal_stats *stats)
{
  int i;
  const int ndims = stats->centers->ndims;
  cached_stats *ret = mk_cp_cached_stats(prestats);
  const bool with_memb = (prestats->stat.memb != NULL);
  ret->stat = nodestats_mk_stats(KDCACHE_MAXOWNERS, ndims, with_memb);
  /* compute diffs for all candidates */
  for(i=0; i<KDCACHE_MAXOWNERS; i++) {
	const int ci = ret->owners[i];
	int pidx;
	double diff_dist;
	int diff_nums;
	dyv *diff_sum;
	if(ci == KDCACHE_NO_OWNER)
	  break;
	pidx = vctr_pidx(stats->centers, ci);
	diff_dist = dyv_ref(stats->stat.distortion, pidx) - dyv_ref(prestats->stat.distortion, i);
	diff_nums = ivec_ref(stats->stat.nums, pidx) - ivec_ref(prestats->stat.nums, i);
	diff_sum = mk_dyv_subtract(dyv_array_ref(stats->stat.sums, pidx),
							   dyv_array_ref(prestats->stat.sums, i));
	dyv_set(ret->stat.distortion, i, diff_dist);
	ivec_set(ret->stat.nums, i, diff_nums);
	dyv_array_set(ret->stat.sums, i, diff_sum);
	free_dyv(diff_sum);
    if(with_memb) {
      /* if you get segfaults on the following line, try
         to turn any FALSEs as last argument in calls to nodestats_mk_stats(),
         into something like parm->maintain_memb, as applicable.
      */
      ivec *diff_memb = mk_sivec_difference(ivec_array_ref(stats->stat.memb, pidx),
                                            ivec_array_ref(prestats->stat.memb, i));
      ivec_array_set(ret->stat.memb, i, diff_memb);
      free_ivec(diff_memb);
    }
  }
  return ret;
}

void distortion_apply_cache(const cached_stats *cached, traversal_stats *stats)
{
  int i;
  my_error("not supposed to arrive here");
  /* add stats for all candidates */
  for(i=0; i<KDCACHE_MAXOWNERS; i++) {
	const int ci = cached->owners[i];
	int pidx;
	if(ci == KDCACHE_NO_OWNER)
	  break;
	pidx = vctr_pidx(stats->centers, ci);
	dyv_increment(stats->stat.distortion, pidx, 
				  dyv_ref(cached->stat.distortion, i));
	ivec_increment(stats->stat.nums, pidx, 
				   ivec_ref(cached->stat.nums, i));
  }
}

void nodestats_apply_cache(const cached_stats *cached, traversal_stats *stats)
{
  int i;
  const bool with_memb = (stats->stat.memb != NULL);
  /* add stats for all candidates */
  for(i=0; i<KDCACHE_MAXOWNERS; i++) {
	const int ci = cached->owners[i];
	int pidx;
	dyv *sum;
	if(ci == KDCACHE_NO_OWNER)
	  break;
	pidx = vctr_pidx(stats->centers, ci);
	dyv_increment(stats->stat.distortion, pidx, 
				  dyv_ref(cached->stat.distortion, i));
	sum = dyv_array_ref(stats->stat.sums, pidx);
	dyv_plus(sum, dyv_array_ref(cached->stat.sums, i), sum);
	ivec_increment(stats->stat.nums, pidx, 
				   ivec_ref(cached->stat.nums, i));
    if(with_memb) {
      ivec *punion = mk_sivec_union(ivec_array_ref(stats->stat.memb, pidx),
                                    ivec_array_ref(cached->stat.memb, i));
      ivec_array_set(stats->stat.memb, pidx, punion);
      free_ivec(punion);
    }
  }
}

void nodestats_free_cache(cached_stats *cached)
{
  nodestats_free_stats(&cached->stat);
  free_cached_stats(cached);
}

static bool means_prune(const simple_kdtree *node, int closest_ctr,
                        const struct traversal_stats *stats)
{
#if 0
  if(DEBUG_FLAG(D_DRAWPOINTS)) {
	static int d;
	int k;
	char tmp[32];
	sprintf(tmp, "pic%03d.ps", d++);
	/* ag_on(tmp);*/
	ag_set_pen_color(AG_RED);
	for(k=0; k<ivec_size(candidates); k++) {
	  ag_circle(dyv_ref(dyv_array_ref(centers,ivec_ref(candidates, k)), 0), 
				dyv_ref(dyv_array_ref(centers,ivec_ref(candidates, k)), 1), 5);
	}
	ag_set_pen_color(AG_BLACK);
	ag_hrect(node->hr);
	for(k=0; k<ivec_size(updated_owners); k++) {
	  ag_circle(dyv_ref(dyv_array_ref(centers,ivec_ref(updated_owners, k)), 0), 
				dyv_ref(dyv_array_ref(centers,ivec_ref(updated_owners, k)), 1), 3);
	}
	if(DEBUG_FLAG(D_INTERACTIVE))
	  wait_for_key();
	/* ag_off(); */
  }
#endif /* 0 */
  return TRUE;
}

void free_cache_entries(simple_kdtree *kd)
{
  if (kd->left != NULL)
    free_cache_entries(kd->left);
  if (kd->right != NULL)
    free_cache_entries(kd->right);
  if(kd->cached != NULL) {
	nodestats_free_cache(kd->cached);
	kd->cached = NULL;
  }
}

void nodestats_update_point(int point_id, const dyv *point,
							int closest_ctr, traversal_stats *stats)
{
  vctr *centers = stats->centers;
  const int pidx = vctr_pidx(centers, closest_ctr);
  dyv *distortion = stats->stat.distortion;
  dyv *sum_ctr = dyv_array_ref(stats->stat.sums, pidx);

  dyv_increment(distortion, pidx,
				dyv_dsqd(vctr_ref(centers, closest_ctr), point));
  ivec_increment(stats->stat.nums, pidx, 1);
  dyv_plus(point, sum_ctr, sum_ctr);
  if(stats->stat.memb) {
    add_to_sivec(ivec_array_ref(stats->stat.memb, pidx), point_id);
  }
  INSTR_COUNTER_INC(1);
  if(DEBUG_FLAG(D_DRAWPOINTS)) {
	ag_set_pen_color(center_color(pidx));
	ag_pixel(dyv_ref(point,0),dyv_ref(point,1));
  }
}

void nodestats_update_bulk(const simple_kdtree *node, int closest_ctr, traversal_stats *stats)
{
  dyv *owner = vctr_ref(stats->centers, closest_ctr);
  double mag_sqd_owner = dyv_scalar_product(owner, owner);
  double dist_contrib = 0.0;

  const int pidx = vctr_pidx(stats->centers, closest_ctr);
  dyv *sum_owner = dyv_array_ref(stats->stat.sums, pidx);
  /* add contributions of owner squared */
  dist_contrib += node->num_points*mag_sqd_owner;

  /* add contributions of dot products */
  dist_contrib -= 2*dyv_scalar_product(owner, node->sum);

  /* add contributions of magnitudes moment */
  dist_contrib += node->sum_mags;
  dyv_increment(stats->stat.distortion, pidx, dist_contrib);
  dyv_plus(node->sum, sum_owner, sum_owner);
  ivec_increment(stats->stat.nums, pidx, node->num_points);
  if(stats->stat.memb) {
    /* OPTIM: we might want to implement sivec_union (in place) */
    /* OPTIM: we might want to add probabilistic sampling to limit the size of the memb list */
    ivec *pts = mk_sivec_points_in_kdtree(node);
    ivec *all_pts = mk_sivec_union(ivec_array_ref(stats->stat.memb, pidx),
                                   pts);
    ivec_array_set(stats->stat.memb, pidx, all_pts);
    free_ivec(all_pts);
    free_ivec(pts);
  }

  DBG_PRINTF(D_EXTRA, ("Updating bulk node %p closest %d(%d)\n", node, closest_ctr, pidx));
  DBG_PRINTF(D_EXTRA, ("Nums+=%d dist+=%g memb=%d\n", node->num_points, dist_contrib,
                       ivec_size(ivec_array_ref(stats->stat.memb, pidx))));
  if(DEBUG_FLAG(D_DRAWPOINTS)) {
	ag_set_pen_color(center_color(pidx));
	ag_hrect(node->hr);
	paint_points_in_kdnode(node);
  }
}

void nodestats_print_bulk(const simple_kdtree *node, int closest_ctr, traversal_stats *stats)
{
  DBG_PRINTF(D_EXTRA, ("Printing bulk node %p closest %d\n", node, closest_ctr));
  fprintf(stats->aux, "+# %d\n", closest_ctr);
  print_points_in_kdtree(stats->aux, node, TRUE);
}

void nodestats_print_point(int point_id, const dyv *point,
                           int closest_ctr, traversal_stats *stats)
{
  static int prev = -1;
  if(prev != closest_ctr)       /* avoid printing redundant centers */
    fprintf(stats->aux, "+# %d\n", closest_ctr);
  fprintf(stats->aux, "+%d\n", point_id);
  prev = closest_ctr;
}
