;;;; -*-  Mode: LISP; Syntax: Common-Lisp; Base: 10                          -*-
;;;; ---------------------------------------------------------------------------
;;;; File name: stats.lsp
;;;;    System: CARVE
;;;;   Version: 1.0
;;;;    Author: Praveen Paritosh
;;;;   Created: October 17, 2002 22:23:11
;;;;   Purpose: general statistics for empirical poking
;;;; ---------------------------------------------------------------------------
;;;;  Modified: Thursday, October 17, 2002 at 22:52:51 by paritosh
;;;; ---------------------------------------------------------------------------
(in-package :fire)


(defvar data::*all-assertions* nil)

(defvar *assertions-path* "d:\\qrg\\fire\\v1\\data\\cyrano") 

(defun load-assertions ()
  (load-file *assertions-path* "assertions-2" :action :load-source)
  (load-file *assertions-path* "assertions-3" :action :load-source)
  (load-file *assertions-path* "assertions-4" :action :load-source)
  (load-file *assertions-path* "assertions-5" :action :load-source)
  (setq data::*all-assertions* 
        (append data::*assertions-2* 
                data::*assertions-3* 
                data::*assertions-4* 
                data::*assertions-5*)))

(defun get-pred-count (&optional (all-assertions data::*all-assertions*))
  (let ((pred-count-table (make-hash-table :test 'equal)))
    (dolist (assertion all-assertions)
      (let ((pred (car assertion)))
        (multiple-value-bind (count present-p) 
            (gethash pred pred-count-table)
          (if present-p
            (setf (gethash pred pred-count-table) (1+ count))
            (setf (gethash pred pred-count-table) 1)))))
    (values pred-count-table)))

(defun humble-predicate-type (pred-count-table)
  (let ((functions nil)
        (relations nil)
        (logicals nil)
        (attributes nil)
        (arityunknowns nil))
    (maphash #'(lambda (pred count)
                 (case (predicate-type pred)
                   (:function (push (cons pred count) functions))
                   (:relation (push (cons pred count) relations))
                   (:logical (push (cons pred count) logicals))
                   (:attribute (push (cons pred count) attributes))
                   (otherwise (push (cons pred count) arityunknowns)))) pred-count-table)
    (values (sort-alist functions) (sort-alist relations) (sort-alist logicals) (sort-alist attributes) (sort-alist arityunknowns))))

(defun get-arity-unknowns (pred-count-table)
  (let ((arityunknowns nil))
    (maphash #'(lambda (pred count)
                 (if (eql (arity pred) :unknown)
                   (push (cons pred count) arityunknowns))) pred-count-table)
    (values arityunknowns)))
  
(defun sort-alist (alist)
  (sort alist '> :key 'cdr))

(defun get-all-assertions (&optional kb *kb*)
  (let ((ass-2 (retrieve-all '(?x ?y) :kb kb))
        (ass-3 (retrieve-all '(?x ?y ?z) :kb kb))
        (ass-4 (retrieve-all '(?x ?y ?z ?w) :kb kb))
        (ass-5 (retrieve-all '(?x ?y ?z ?w ?u) :kb kb))
        (ass-6 (retrieve-all '(?x ?y ?z ?w ?u ?v) :kb kb)))
    (nconc ass-1 ass-2 ass-3 ass-4 ass-5 ass-6)))


   

(defun ninety-percentile-symbols (facts)
  (let* ((count-table (frequency-count-of-all-syms facts))
         (count-alist (sort (hash-table->alist count-table) '> :key #'cdr))
         (max (cdar count-alist))
         (min (cdr (nth (- (length count-alist) 1) count-alist)))
         (ninety-percentile (- max (* 0.9 (- max min))))
         
         
    
    
(defun frequency-count-of-all-syms (facts)
  (let ((count-table (make-hash-table)))
    (dolist (fact facts)
      (dolist (sym (flatten fact))
        (if (not (ltre:variable? sym))
          (multiple-value-bind (count present-p) 
              (gethash sym count-table)
            (if present-p
              (setf (gethash sym count-table) (1+ count))
              (setf (gethash sym count-table) 1))))))
    (values count-table)))

(defun hash-table->alist (table)
  (let ((alist nil))
    (maphash #'(lambda (k v)
                 (push (cons k v) alist)) table)
    (values alist)))


(defun dump-alist-as-spreadsheet (alist fname)
  "dumps an alist as a two column table, to be browsed as a spreadsheet"
  (let ((cl-user::*print-pretty* nil)) ;; Saves file space -- nil for debugging
    (with-open-file (fout fname
                          :direction :output
                          :if-does-not-exist :create
                          :if-exists :supersede)
      (dolist (pair alist)
        (format fout "~S ~S~%" (car pair) (cdr pair))))
    fname))

;; sums up the lengths of the values entries of a hash table. For example, 
;; one can use this to compute the total number of assertions in *retrieve-refs-cache*
(defun num-facts (table)
  (let ((num 0))
    (maphash (lambda (k v)
               (setq num (+ num (length v))))
             table)
    num))

(defun temp-fact-count (table)
  (maphash (lambda (k v)
             (format t "~A ~A~%" k (length v)))
           table))

(defun temp-count-valid-numbers (allcountrydata dimensions)
  (let ((count (make-array (length dimensions) :initial-element 0)))
    (dolist (countrydata allcountrydata)
      (dotimes (i (length dimensions))
        (if (numberp (cdr (nth i (cdr countrydata))))
          (incf (aref count i)))))
    count))