;;;; -*-  Mode: LISP; Syntax: Common-Lisp; Base: 10                          -*-
;;;; ---------------------------------------------------------------------------
;;;; File name: quant.lsp
;;;;    System: CARVE
;;;;   Version: 1.0
;;;;    Author: Praveen Paritosh
;;;;   Created: August 18, 2002 00:23:11
;;;;   Purpose: Testing minimal case construction and SME and SEQL for QUANT
;;;; ---------------------------------------------------------------------------
;;;;  Modified: Tuesday, October 1, 2002 at 04:04:51 by paritosh
;;;; ---------------------------------------------------------------------------

(in-package :fire)

;; CAUTION: EXPERIMENTAL WORK, IN PROGRESS. 
;; UNDO the TASK/RELEVANT-PREDICATE? distinction

(setq *guess-predicate-info-if-needed* t)

;; The key idea here is to see if we can build the relational limit points 
;; that make up the quantity space of a quantitative dimension that is an 
;; aligned dimension across multiple cases. Here is where I explore the 
;; feasibility of this idea from cases extracted dynamically from the
;; CYC KB. Right now, the hypothesis of how to do this, the conjectured
;; theory, seems plausible to me. But that is not sufficient evidence! 
;; Besides the fact that the theory itself might take more work to be 
;; useful, or might not actually work out, the other problems that I am 
;; prepared to run into -- 
;; 1. The cases extracted from CYC might not be that structurally rich 
;;    for structural similarity to provide as meaningful results. 
;; 2. In any clustering problem, the naive approach is to compute the 
;;    similarity between all pairs of exemplars, giving rise to n^2 
;;    comparisons. SEQL does a linear sequence, but that might have a
;;    stronger sequence effect than we might want. I am looking for 
;;    clever strategies to address this issue. 

;; All data associated with this system will be in *fire-path*\\QUANT\\data\\countries
;; In order to get a grip on the issues involved in doing this, I start 
;; experimenting with a set of examples, 50 cases chosen from the countries
;; in the CIAWorldfactBookMt in CYC. Things to do -- 
;; 1. Constructing the cases - Start with minimalQuantFn, which is just the
;;    minimal dynamic case creation. Dump them to disk in a manner such that
;;    I can read them all back simply. 
;;      - Can I do dump all stuff in file? 
;;      - I will also have to save references for the dgroups, a list of all 
;;        the dgroups that I have? 
;;      - What can I do to make the cases better for QUANT?
;; 2. Doing the matches -- 
;;      - do all pairs matches, just to see whats going on. 
;;      - how do we dump all this data? 
;;      - how to look at this? RBrowse/Analogy Summarisation? 
;; Everything should be incremental so that I can stop the program at any point. 

;; Batch experiments to see how the cases look like. 
;; Cache the facts chosen to be in the case for all exemplars, and then maphash
;; over this table to make all the dgroups. 

(defvar *dgroup-facts-cache* (make-hash-table :test 'equal))
(defvar *dgroups-cache* (make-hash-table :test 'equal))

(defun batch-cache-dgroup-facts (concepts &optional (retrieve-refs-cache *retrieve-refs-cache*)
                                          (dgroup-facts-cache *dgroup-facts-cache*)
                                          (reasoner *reasoner*)
                                          (casefn 'data::minimalCaseFn)
                                          (task 'data::QuantCountries))
  (declare (ignore casefn))
  (let ((source (analogy-source-of reasoner)))
    (dolist (concept concepts)
      (get-and-cache-dgroup-facts concept retrieve-refs-cache dgroup-facts-cache
                                         source task))))
  
(defun batch-cache-dgroups (concepts &optional (dgroup-facts-cache *dgroup-facts-cache*)
                                     (dgroups-cache *dgroups-cache*)
                                     (reasoner *reasoner*))
  (let ((source (analogy-source-of reasoner)))
    (dolist (concept concepts)
      (make-and-cache-dgroup concept dgroup-facts-cache dgroups-cache source))))

(defun get-and-cache-dgroup-facts (concept retrieve-refs-cache  dgroup-facts-cache source task)
  (setf (gethash concept dgroup-facts-cache) 
    (gather-minimal-quant-case-dgroup concept source task retrieve-refs-cache)))

(defun make-and-cache-dgroup (concept dgroup-facts-cache dgroups-cache source)
  (setf (gethash concept dgroups-cache) 
    (make-dgroup-from-facts (make-dgroup-name concept) 
                            (gethash concept dgroup-facts-cache)
                            source)))

;;;;; Where should the symbol that corresponds to the dgroup be? The data package makes sense to me. 
(defun make-dgroup-name (concept)
  (intern (concatenate 'string (symbol-name concept) "Dgrp") :data))

;;;(defun test-dgroup-construction (&key (concept-list *test-dgroup-cases*) (reasoner *reasoner*))
;;;  (let ((dgroups nil))
;;;    (dolist (concept concept-list)
;;;      (format t "Building dgroup for ~A~%" concept)
;;;      (make-minimal-dgroup concept (make-dgroup-name concept) 'data::minimalQuantCaseFn reasoner)
;;;      ;; might as well push the dgroup object itself in this list that we return rather than the 
;;;      ;; name of the dgroup, later. 
;;;      (push (make-dgroup-name concept) dgroups))
;;;    (values dgroups)))
;;;
;;;

;;;
;;;(defun make-minimal-dgroup (concept dgroup-name &optional 
;;;                                    (casefn 'data::minimalQuantCaseFn) 
;;;                                    (task 'data::QuantCountries) 
;;;                                    (reasoner *reasoner*))
;;;  (let* ((analogy-source (analogy-source-of reasoner))
;;;         (facts (gather-dgroup-facts casefn (list concept) task analogy-source))
;;;         (dgroup (make-dgroup-from-facts dgroup-name facts analogy-source)))
;;;    (values dgroup)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Minimal Quant Case Function
;;; This is not fundamentally new type of case function. I am planning
;;; to use dynamic case construction to build quantity spaces of different 
;;; parameters in the CIAWorldFactbookMt. En route, I am adding specific
;;; filters that make sense for my task. I did not want to interfere with
;;; the minimalCaseFn as someone else might expect to behave in a certain
;;; way. PKP 09.26.02

;;; minimalQuantCaseFn -- an experimental caseFn, will be removed/merged - PKP 9.26.02
(defmethod gather-dgroup-facts ((type (eql 'data::minimal-quant-case-fn))
                                (specs list)
                                (source analogy-source))
  (gather-minimal-quant-case-dgroup (car specs) source))

(defmethod gather-dgroup-facts ((type (eql 'data::minimalQuantCaseFn))
                                (specs list)
                                (source analogy-source))
  (gather-minimal-quant-case-dgroup (car specs) source))

;; RELEVANT-PREDICATE? is based on the TASK and FILTER-FACT? on the case function. 

(defmethod relevant-predicate? ((pred symbol) (task (eql 'data::QuantCountries)))
  (or (member pred *quant-countries-relevant-predicates*)
      (not (member pred *quant-countries-irrelevant-predicates*))))

(defmethod filter-fact? (fact source (filter-style (eql :minimalQuantCaseFn)))
  ;; For minimal case fun, you simply take every expression except
  ;;  for the ones with bookkeeping predicates
  (or (contains-bad-cyc-predicate? fact)
      (member (fact-predicate fact) *special-filter-predicates*)
      (book-keeping-fact? fact)
      ;; Heuristic: most facts that contain strings dont seem to be relevant
      (contains-string? fact)))

(defun contains-string? (fact)
  (cond ((null fact) nil)
        ((stringp fact) t)
        ((not (consp fact)) nil)
        (t (or (contains-string? (car fact))
               (contains-string? (cdr fact))))))

(defun gather-minimal-quant-case-dgroup (concept source task refscache)
  (let ((mentioning-facts (retrieve-references-using-cache concept :retrieve-refs-cache refscache)))
    (get-basic-relevant-facts concept mentioning-facts source :minimalQuantCaseFn task)))

(defparameter *quant-countries-relevant-predicates* nil) 

(defparameter *quant-countries-irrelevant-predicates* 
  '(countryOfCity geopoliticalSubdivision geopoliticalSubdivisionsOf geographicalSubRegions chiefPorts objectFoundInLocation
                  adjSemTrans testAnswersCycL))
  
;;;Doing a query reifies a lot of stuff in the reasoner's WM, which is bad, because my cases
;;;are huge and we are doing lots of matches right now. 

(defun do-match-query (dgroup1 dgroup2 analogy-source)
  (let* ((match (caar (ask-it `(data::matchBetween ,dgroup1 ,dgroup2 nil ?match) :response :bindings)))
         (sme-id (third match))
         (sme-pointer (car (member sme-id (smes analogy-source) :key #'sme::id))))
    (values sme-pointer)))

(defun do-match (base target source)
;;;  (update-ubiquitous-preds! source)
;;;  (update-internal-preds! source)
  (sme:with-vocabulary source
    (let* ((base-dgroup (get-dgroup base source))
           (target-dgroup (get-dgroup target source))
           (sme (sme::define-sme base-dgroup target-dgroup)))
      (sme:match-with-appropriate-filters sme nil)
      (values sme))))


(defun do-all-pair-matches (dgroups source)
  (let* ((num-dgroups (length dgroups))
         (sme-matrix (make-array (list num-dgroups num-dgroups) :initial-element nil)))
    (dotimes (i num-dgroups)
      (dotimes (j num-dgroups)
        (setf (aref sme-matrix i j) 
          (do-match (nth i dgroups) (nth j dgroups) source))))
    (values sme-matrix)))

;;;; Experimental junk

;; Redundant functions, remove. 
(defun save-dgroups (dgroups filename &key (reasoner *reasoner*))
  (let ((analogy-source (analogy-source-of reasoner)))
    (dolist (dgroup dgroups)
      (save-dgroup (get-dgroup dgroup analogy-source) filename))))

(defun save-dgroup (dgroup filename)
  (with-open-file (fout filename :direction :output :if-exists :append :if-does-not-exist :create)
    (sme:sme-dumper dgroup fout)
    (format fout "~%;;------------------------------------~%~%")))


(defparameter *test-dgroup-cases* '(data::Nepal data::Bhutan data::China-PeoplesRepublic data::Japan data::France data::Germany))

;; These are not all countries, I randomly removed some of the ones that were obscure. 
(defparameter *countries-partial* 
  '(data::Niger 
     data::Zimbabwe  data::Yemen
    data::Iran data::Jordan-TheNation  data::Mauritius data::Mexico
     data::Fiji data::Ecuador data::Lithuania data::Guinea
    data::Japan  data::Nauru data::Norway  data::Bosnia-Herzegovina
    data::Tanzania data::EquatorialGuinea data::Haiti  data::China-PeoplesRepublic
    data::Cyprus-TheCountry data::AntiguaAndBarbuda data::Barbados data::Switzerland
    data::SierraLeone  data::SriLanka  data::Kazakhstan
    data::Nepal data::Finland data::Belgium data::Israel data::Italy data::SanMarino
    data::Yugoslavia data::UnitedArabEmirates data::UnitedKingdomOfGreatBritainAndNorthernIreland
    data::Suriname data::NorthKorea data::Turkey data::Luxembourg data::Namibia data::Maldives
    data::Uzbekistan  data::Mozambique data::UnionOfSovietSocialistRepublics
    data::Slovakia data::Latvia data::Mali data::Ukraine data::Ethiopia data::Libya
    data::Mongolia data::SouthAfrica data::Portugal data::CzechRepublic data::Afghanistan
    data::Algeria data::Guyana  data::Botswana
     data::Vietnam data::Ghana data::Peru data::Angola 
    data::Senegal data::Somalia data::Seychelles data::Gabon data::Russia data::Greece
    data::Belize data::Ireland-TheNation data::Canada data::Nigeria data::Grenada data::Poland
    data::EastGermany data::PrincipalityOfSealand data::Bhutan data::Gambia data::France
    data::Honduras data::ElSalvador data::SolomonIslands-TheNation data::Egypt data::Sweden
    data::Malaysia  data::UnitedStatesOfAmerica data::Tunisia data::Kyrgyzstan
    data::Slovenia data::Macedonia data::Romania data::Bangladesh data::Indonesia-TheNation
    data::Venezuela data::Cuba  data::Burma data::Armenia data::TrinidadAndTobago
     data::Hungary data::Bahamas 
     data::Panama data::Uruguay data::Zambia data::Azerbaijan
    data::Chile data::Kuwait data::SouthKorea  data::Argentina
     data::WestGermany data::Singapore data::Morocco data::Syria
     data::Croatia data::India data::Brazil  data::SaudiArabia
    data::Bahrain data::Guatemala  data::Oman data::VaticanCity data::Bolivia
    data::Germany  data::Paraguay data::Brunei data::Albania
    data::Tajikistan data::Spain data::Netherlands 
    data::CentralAfricanRepublic data::Qatar data::Kenya data::Thailand data::Denmark
     data::Philippines data::Congo-Brazzaville  data::Chad
    data::Lebanon data::Uganda data::Madagascar  data::Pakistan
    data::Jamaica data::Sudan data::Iraq data::Laos  data::Iceland
    data::Czechoslovakia data::Swaziland data::Cambodia data::Andorra data::Liberia
    data::WesternSamoa data::PapuaNewGuinea data::CostaRica  data::Rwanda
     data::Bulgaria data::Dominica data::Colombia data::NewZealand
    data::DominicanRepublic data::Australia data::Cameroon data::BurkinaFaso data::CapeVerde
    data::Nicaragua  data::Taiwan-RepublicOfChina data::Monaco data::Austria
    )) 

;; Testing
(defun cache-facts (concepts)
  (dolist (concept concepts)
    (if (not (gethash concept *retrieve-refs-cache*))
      (retrieve-references-using-cache concept))))


;; What is the first cut for relevant predicates? 
;;;Relevant information for countries. The problem, of course is that recursively
;;;looking for statements involving these predicates and ground terms found in the
;;;facts in the minimal case might lead to sucking out facts about all other 
;;;countries -- bad idea. Is there another distinction that we need to make between
;;;relevant and higherorder predicates? The key motivation is to make the cases 
;;;more interesting from a systematicity point of view, right? 
;;;
;;;Area 
;;;
;;;Population
;;; Annual rate of increase
;;; Birth rate
;;; Infant mortality rate
;;; Density
;;;
;;;Capital 
;;; Size
;;; Largest cities and size
;;;
;;;Ethnicity
;;;
;;;Religion
;;;
;;;Languages spoken
;;;
;;;Economy 
;;;GDP
;;;PPP
;;;Growth rate
;;;Inflation
;;;Arable land fraction
;;;Products
;;;Labour force and distribution
;;; in Industry
;;; in Agriculture/Animal Husbandry
;;; in construction
;;; in services
;;; other
;;;Unemployment 
;;;Industry
;;;Natural resources
;;;Exports (amount and distribution)
;;;Imports
;;;Major trading partners