;;;; -*-  Mode: LISP; Syntax: Common-Lisp; Base: 10                          -*-
;;;; ---------------------------------------------------------------------------
;;;; File name: ksl-to-cyc-v2.lsp
;;;;    System: Carve
;;;;    Author: Praveen Paritosh
;;;;   Created: January 26, 2004 15:45:00
;;;;   Purpose: To translate from Stanford CIA Factbook KB to Cyc
;;;; ---------------------------------------------------------------------------
;;;;  Modified: Monday, February 9, 2004 at 13:50:31 by paritosh
;;;; ---------------------------------------------------------------------------

(in-package :fire)

;; Goal
;; Extract facts from the Stanford WFB, rewrite in CycL and add to the KB
;; For starters we deal only with African countries. Which is what will be
;; reported in the CogSci 2004 paper. 

;; Get selected facts to add for all the african countries. 

;; Still store that as a list of (<country> <list of facts>)

;; This is the subset of KSL [numeric values along the ten dimensions as collected by 
;; gather-selected-facts] that we are currently planning to add to CYC. 

(defparameter *wfb-africa-selected-cache* "D:\\qrg\\fire\\v1\\carve\\data\\countrydata\\africa_wfb_selected.lsp")
(defparameter *wfb-africa-selected-data-file* "D:\\qrg\\fire\\v1\\carve\\data\\countrydata\\africa_wfb_selected.txt")

(defun gather-selected-facts-for-all-of-africa (&optional (file *wfb-africa-selected-cache*)
                                                          (mappings data::*wfb-cyc-name-mappings*))
  (dolist (country-pair mappings)
    (data::save-facts (cdr country-pair) (gather-selected-facts (car country-pair)) *wfb-africa-selected-cache*)))

;; Convert the selected facts in the datafile above into a matrix form so that 
;; we can do the analysis in MATLAB. 

(defun convert-data-to-array (&optional (selected-facts data::*wfb-africa-selected-facts*)
                                        (data-file *wfb-africa-selected-data-file*))
  (let* ((num-countries (length selected-facts))
         (sample-data (second (car selected-facts)))
         (num-fields (length sample-data))
         (fields (mapcar #'car sample-data))
         (countries (mapcar #'car selected-facts))
         (wfb-num-data-matrix (make-array (list num-countries num-fields) :initial-element 0.0)))
    (dotimes (i num-countries)
      (dotimes (j num-fields)
        (setf (aref wfb-num-data-matrix i j) (get-number (nth j (second (nth i selected-facts)))))))
    (with-open-file (fout data-file 
                          :direction :output
                          :if-exists :overwrite
                          :if-does-not-exist :create)   
      (cl-user::print-2d-array-with-labels wfb-num-data-matrix countries fields fout))
    (values wfb-num-data-matrix fields countries)))

(defun get-number (fact)
  ;; This is a kludge. Ignoring unit handling right now. So the fact either contains
  ;; pure number or a list like (UnitFn <num>)
  (let ((num (car (last fact))))
    (if (numberp num)
      num
      (second num))))  
    
;; The output of this program is a text file that is space delimited that contains 
;; the 10 fields for all the 54 countries and row col labels, too. 

;; This goes through the cached version of the WFB facts, for each country
;; applies filters for specific facts that we are interested in, translates
;; them into Cyc notation. You have to write for each such kind of fact, two
;; functions -- e.g., birth-rate-fact? that looks at a WFB formula and returns
;; t if it is the kind of fact you are looking for, and birth-rate-translate
;; which takes this fact, and produces a formula that represents this knowledge
;; which is compatible with Cyc (cyc's predicates, arity, etc.). 

(defun gather-selected-facts (country &optional (wfb-cache data::*africa-wfb-facts*))
  ;; looking at only facts from that country will speed it up. 
  (let ((country-facts (cadar (member country wfb-cache :key #'car)))
        (gathered-facts nil))
    (dolist (fact country-facts)
      (cond ((birth-rate-fact? fact)
             (push (birth-rate-translate fact) gathered-facts))
            ((death-rate-fact? fact)
             (push (death-rate-translate fact) gathered-facts))
            ((exports-fact? fact)
             (push (exports-translate fact) gathered-facts))
            ((imports-fact? fact)
             (push (imports-translate fact) gathered-facts))
            ((population-fact? fact)
             (push (population-translate fact) gathered-facts))
            ((population-growth-rate-fact? fact)
             (push (population-growth-rate-translate fact) gathered-facts))
            ((literacy-total-fact? fact)
             (push (literacy-total-translate fact) gathered-facts))
            ((life-expectancy-total-fact? fact)
             (push (life-expectancy-total-translate fact) gathered-facts))
            ((area-total-fact? fact)
             (push (area-total-translate fact) gathered-facts))
            ((infant-mortality-fact? fact)
             (push (infant-mortality-translate fact) gathered-facts))
            (t nil)))            
    (values gathered-facts)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; S: (wfb::birth_rate albania (* 21.7 |births/1,000 population|) );; line 1149
;; C: (birthRate Azerbaijan 22.05)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun birth-rate-fact? (ksl-fact)
  (and (eql (length ksl-fact) 3)
       (eql (car ksl-fact)
            'wfb::birth_rate)
       (numberp (strip-units (third ksl-fact)))))

(defun birth-rate-translate (ksl-fact)
  `(data::birthRate ,(cyclify-country-name (second ksl-fact))
                    ,(strip-units (third ksl-fact))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Death Rate
;; C: (deathRate China-PeoplesRepublic 7.36)
;; S: (wfb::death_rate china (* 7.36 |deaths/1,000 population|) );; line 23431
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun death-rate-fact? (ksl-fact)
  (and (eql (length ksl-fact) 3)
       (eql (car ksl-fact)
            'wfb::death_rate)
       (numberp (strip-units (third ksl-fact)))))

(defun death-rate-translate (ksl-fact)
  `(data::deathRate ,(cyclify-country-name (second ksl-fact))
                    ,(strip-units (third ksl-fact))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; S: (wfb::exports albania (* 112e6 $) );; line 1395
;; C: (exportValue Albania (cl-user::Dollar-UnitedStates 112000000))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun exports-fact? (fact)
  (and (eql (length fact) 3)
       (eql (car fact) 'wfb::exports)
       (numberp (strip-units (third fact)))))

(defun exports-translate (fact)
  `(data::exportValue ,(cyclify-country-name (second fact))
                      (data::Dollar-UnitedStates ,(strip-units (third fact)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; S: (wfb::imports china (* 115.7e9 $) );; line 23717
;; C: (importValue Bahrain (BillionDollars 3.5))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun imports-fact? (fact)
  (and (eql (length fact) 3)
       (eql (car fact) 'wfb::imports)
       (numberp (strip-units (third fact)))))

(defun imports-translate (fact)
  `(data::importValue ,(cyclify-country-name (second fact))
                      (data::Dollar-UnitedStates ,(strip-units (third fact)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; C: (populationDuring UnitedStatesOfAmerica (YearFn 1997) 267954764)
;; S: (wfb::population united_states 263814032 );; line 117370
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun population-fact? (fact)
  (and (eql (length fact) 3)
       (eql (car fact) 'wfb::population)
       (numberp (third fact))))

(defun population-translate (fact)
  `(data::populationDuring ,(cyclify-country-name (second fact))
                           (data::YearFn 1995)
                           ,(third fact)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; C: (populationGrowthRate Afghanistan (Percent 14.47)) ;; Funny that year is dropped here?
;; S: (wfb::population_growth_rate united_states (* 1.02 %) );; line 117383
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun population-growth-rate-fact? (fact)
  (and (eql (length fact) 3)
       (eql (car fact) 'wfb::population_growth_rate)
       (numberp (strip-units (third fact)))))

(defun population-growth-rate-translate (fact)
  `(data::populationGrowthRate ,(cyclify-country-name (second fact))
                               (data::Percent ,(strip-units (third fact)))))
    
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Area
;; C: (areaOfRegion China-PeoplesRepublic (SquareKilometer 9596960))
;;    (landAreaOfRegion China-PeoplesRepublic (SquareKilometer 9326410))
;; S: (wfb::area china total_area (* 9596960 sq_km) );; line 23314
;;    (wfb::area china land_area (* 9326410 sq_km) );; line 23316
;;    (wfb::land_use comoros arable_land (* 35 %) );; line 25469
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun area-total-fact? (fact)
  (and (eql (length fact) 4)
       (eql (car fact) 'wfb::area)
       (eql (third fact) 'data::total_area)
       (numberp (strip-units (fourth fact)))))

(defun area-total-translate (fact)
  `(data::areaOfRegion ,(cyclify-country-name (second fact))
                       (data::SquareKilometer ,(strip-units (fourth fact)))))


;;;(wfb::national_product_per_capita china (* 2500 $) );; line 23692

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Literacy : Total Literacy rates
;; Cyc has two concepts literacyRate and literacyRateForGroupInRegion
;; (literacyRateForGroupInRegion Greece Person (Percent 95))
;; (literacyRateForGroupInRegion Greece FemalePerson (Percent 93))
;; (literacyRateForGroupInRegion Greece MalePerson (Percent 98))
;; In Stanford 
;; (wfb::literacy armenia total_population (* 99 %) );; line 5573
;; (wfb::literacy armenia male (* 99 %) );; line 5575
;; (wfb::literacy armenia female (* 98 %) );; line 5577
;; also useful -- 
;; (wfb::literacy-condition armenia "age 15 and over can read and write" );; line 5571
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun literacy-total-fact? (fact)
  (and (eql (length fact) 4)
       (eql (car fact) 'wfb::literacy)
       (eql (third fact) 'data::total_population)
       (numberp (strip-units (fourth fact)))))

(defun literacy-total-translate (fact)
  `(data::literacyRateForGroupInRegion ,(cyclify-country-name (second fact))
                                       data::Person
                                       (data::Percent ,(strip-units (fourth fact)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Life Expectancy
;;;(wfb::life_expectancy_at_birth china total_population (* 68.08 years) );; line 23444
;;;(wfb::life_expectancy_at_birth china male (* 67.09 years) );; line 23446
;;;(wfb::life_expectancy_at_birth china female (* 69.18 years) );; line 23448
;;; CYC: (lifeExpectancyForGroupInRegion Australia Person (YearsDuration 77.78))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun life-expectancy-total-fact? (fact)
  (and (eql (length fact) 4)
       (eql (car fact) 'wfb::life_expectancy_at_birth)
       (eql (third fact) 'data::total_population)
       (numberp (strip-units (fourth fact)))))

(defun life-expectancy-total-translate (fact)
  `(data::lifeExpectancyForGroupInRegion ,(cyclify-country-name (second fact))
                                         data::Person
                                         (data::YearsDuration ,(strip-units (fourth fact)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; (wfb::infant_mortality_rate china (* 52.1 |deaths/1,000 live births|) );; line 23439
;; CYC: (infantMortalityRate Azerbaijan 33.9)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun infant-mortality-fact? (fact)
  (and (eql (length fact) 3)
       (eql (car fact) 'wfb::infant_mortality_rate)
       (numberp (strip-units (third fact)))))

(defun infant-mortality-translate (fact)
  `(data::infantMortalityRate ,(cyclify-country-name (second fact))
                              ,(strip-units (third fact))))


;; Migration 
;; S: (wfb::net_migration_rate china (* 0 |migrant(s)/1,000 population|) );; line 23435

;;;(wfb::infant_mortality_rate china (* 52.1 |deaths/1,000 live births|) );; line 23439
;;;(wfb::life_expectancy_at_birth china total_population (* 68.08 years) );; line 23444
;;;(wfb::life_expectancy_at_birth china male (* 67.09 years) );; line 23446
;;;(wfb::life_expectancy_at_birth china female (* 69.18 years) );; line 23448
;;;
;;;(wfb::total_labor_force china 583.6e6 );; line 23491

;; Economy 

;;;(wfb::national_product_real_growth_rate china (* 11.8 %) );; line 23688
;;;(wfb::|Inflation rate (consumer prices)| china "25.5% (December 1994 over December 1993)" );; line 23696
;;;
;;;
;;;(wfb::external_debt china (* 100e9 $) );; line 23726
;;;(wfb::unemployment_rate colombia (* 7.9 %) );; line 25223
;;;
;;;
;;;(wfb::electricity china capacity (* 162000000 kw) );; line 23735
;;;(wfb::electricity china production (* 746e9 kwh) );; line 23737
;;;(wfb::electricity china consumption_per_capita (* 593 kwh) );; line 23739
;;;
;;;(wfb::land_use comoros arable_land (* 35 %) );; line 25469
;;;
;;;(wfb::railroads china total "65,780 km" );; line 23791
;;;
;;;(wfb::highways china total (* 1.029e6 km) );; line 23802
;;;
;;;(wfb::airports china total 204 );; line 23843
;;;
;;;(wfb::radio china radios 215e6 );; line 23884
;;;
;;;(wfb::television china televisions 75e6 );; line 23891


(defun strip-units (expr)
  ;; The usual form is (* val unit)
  (second expr))

(defun cyclify-country-name (country &optional (mappings cl-user::*wfb-cyc-name-mappings*))
  (if (member country mappings :key #'car)
    (cdar (member country mappings :key #'car))
    (let ((capitalized-name (intern (remove #\_ (string-capitalize country)) :data)))
      (case capitalized-name
        (data::UnitedStates 'data::UnitedStatesOfAmerica)
        (data::BosniaAndHerzegovina  'data::Bosnia-Herzegovina)
        (data::China 'data::China-PeoplesRepublic)
        (data::Cyprus 'data::Cyprus-TheCountry)
        (data::Georgia 'data::Georgia-TheNation)
        (data::Indonesia 'data::Indonesia-TheNation)
        (data::Ireland 'data::Ireland-TheNation)
        (data::Jordan 'data::Jordan-TheNation)
        (data::|Macedonia, The Former Yugoslav Republic Of| 'data::Macedonia)
        (data::Malta 'data::Malta-TheCountry)
        (data::|Micronesia, Federated States of| 'data::Micronesia-FederatedStatesOf)
        (data::SaintKittsAndNevis 'data::SaintKitts-Nevis)
        (data::SaintVincentAndTheGrenadines 'data::StVincentAndTheGrenadines)
        (data::SolomonIslands 'data::SolomonIslands-TheNation)    
        (t capitalized-name)))))
  
(defun gather-all-countries (&optional (facts data::*wfb-facts*))
  (let (countries)
    (dolist (fact facts)
      (if (population-fact? fact)
        (push (second fact) countries)))
    (reverse countries)))

;; what is this for? remove if useless, please. 
(defvar *countries* 
    '((Afghanistan Albania Algeria AmericanSamoa Andorra Angola Anguilla
                   AntiguaAndBarbuda Argentina Armenia Aruba Australia Austria Azerbaijan TheBahamas
                   Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia
                   BosniaAndHerzegovina Botswana Brazil BritishVirginIslands Brunei Bulgaria Burkina
                   Burma Burundi Cambodia Cameroon Canada CapeVerde CaymanIsland
                   CentralAfricanRepublic Chad Chile China |Cocos (Keeling) Islands| Colombia
                   Comoros Congo CookIslands CostaRica |Cote D'Ivoire| Croatia Cuba Cyprus
                   CzechRepublic Denmark Djibouti Dominica DominicanRepublic Ecuador Egypt
                   ElSalvador EquatorialGuinea Eritrea Estonia Ethiopia FaroeIslands Fiji Finland
                   France FrenchGuiana FrenchPolynesia Gabon TheGambia GazaStrip Georgia Germany
                   Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey
                   Guinea Guinea-Bissau Guyana Haiti Honduras HongKong Hungary Iceland India
                   Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan
                   Kenya Kiribati |Korea, North| |Korea, South| Kuwait Kyrgyzstan Laos Latvia
                   Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macau
                   |Macedonia, The Former Yugoslav Republic Of| Madagascar Malawi Malaysia Maldives
                   Mali Malta |Man, Isle Of| MarshallIslands Martinique Mauritania Mauritius Mayotte
                   Mexico |Micronesia, Federated States Of| Moldova Monaco Mongolia Montserrat
                   Morocco Mozambique Namibia Nauru Nepal Netherlands NetherlandsAntilles
                   NewCaledonia NewZealand Nicaragua Niger Nigeria NorthernMarianaIslands Norway
                   Oman Pakistan Palau Panama PapuaNewGuinea Paraguay Peru Philippines
                   PitcairnIslands Poland Portugal PuertoRico Qatar Reunion Romania Russia Rwanda
                   SaintHelena SaintKittsAndNevis SaintLucia SaintPierreAndMiquelon
                   SaintVincentAndTheGrenadines SanMarino SaoTomeAndPrincipe SaudiArabia Senegal
                   SerbiaAndMontenegroMontenegro SerbiaAndMontenegroSerbia Seychelles SierraLeone
                   Singapore Slovakia Slovenia SolomonIslands Somalia SouthAfrica Spain SriLanka
                   Sudan Suriname Svalbard Swaziland Sweden Switzerland Syria Taiwan Tajikistan
                   Tanzania Thailand Togo Tokelau Tonga TrinidadAndTobago Tunisia Turkey
                   Turkmenistan TurksAndCaicosIslands Tuvalu Uganda Ukraine UnitedArabEmirates
                   UnitedKingdom UnitedStates Uruguay Uzbekistan Vanuatu Venezuela Vietnam
                   VirginIslands WakeIsland WallisAndFutuna WestBank WesternSahara WesternSamoa
                   Yemen Zaire Zambia Zimbabwe)))

    
    
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; End of Code