;;;; -*-  Mode: LISP; Syntax: Common-Lisp; Base: 10                          -*-
;;;; ---------------------------------------------------------------------------
;;;; File name: bootstrap.lsp
;;;;    System: CARVE
;;;;   Version: 1.0
;;;;    Author: Praveen Paritosh
;;;;   Created: October 17, 2002 22:23:11
;;;;   Purpose: Bootstrapping CARVE
;;;; ---------------------------------------------------------------------------
;;;;  Modified: Friday, January 23, 2004 at 17:42:40 by paritosh
;;;; ---------------------------------------------------------------------------

;; The CIA factbook cases are quite flat with lots of numbers. To get CARVE
;; started, we need to bootstrap the process. Here we test some simple and
;; general heuristics that produce some first cut symbolic discretisations. 

;; Need to be able to handle units before anything else

;; For a list of facts, for the facts for which filterfun returns true, 
;; returns a list of args.

;; List of parameters to extract

(defparameter *countries-dimensions-examples* 
  '(
    (total_area  (wfb::area canada total_area (* 9976140 sq_km) ))
    (land_area  (wfb::area canada land_area (* 9220970 sq_km) ))
    (total_borders (wfb::borders canada total (* 8893 km) ))
    (arable_land  (wfb::land_use canada arable_land (* 5 %) ))
    (population (wfb::population canada 28434545 ))
    (population_growth_rate (wfb::population_growth_rate canada (* 1.09 %) ))
    (birth_rate  (wfb::birth_rate canada (* 13.74 |births/1,000 population|) ))
    (death_rate  (wfb::death_rate canada (* 7.43 |deaths/1,000 population|) ))
    (net_migration_rate  (wfb::net_migration_rate canada (* 4.55 |migrant(s)/1,000 population|) ))
    (infant_mortality_rate (wfb::infant_mortality_rate canada (* 6.8 |deaths/1,000 live births|) ))
    (life_expectancy_at_birth_total (wfb::life_expectancy_at_birth canada total_population (* 78.29 years) ))
    (literacy_total_population  (wfb::literacy canada total_population (* 97 %) ))
    (total_fertility_rate  (wfb::total_fertility_rate canada (* 1.83 children_born/woman) ))
    (total_labor_force  (wfb::total_labor_force canada 13.38e6 ))
    (national_product  (wfb::national_product canada (* 639.8e9 $) ))
    (national_product_real_growth_rate  (wfb::national_product_real_growth_rate canada (* 4.5 %) ))
    (national_product_per_capita  (wfb::national_product_per_capita canada (* 22760 $) ))
    (unemployment_rate  (wfb::unemployment_rate canada (* 9.6 %) ))
    (budget_expenditures  (wfb::budget canada expenditures (* 115.3e9 $) ))
    (exports_total  (wfb::exports canada (* 164.3e9 $) ))
    (imports_total  (wfb::imports canada (* 151.5e9 $) ))
    (external_debt  (wfb::external_debt canada (* 243e9 $) ))
    (electricity_capacity  (wfb::electricity canada capacity (* 108090000 kw) ))
    (electricity_production  (wfb::electricity canada production (* 511e9 kwh) ))
    (electricity_consumption_per_capita  (wfb::electricity canada consumption_per_capita (* 16133 kwh) ))
    (highways_total  (wfb::highways canada total (* 849404 km) ))
    (airports_total  (wfb::airports canada total 1386 ))
    (television_broadcast_stations  (wfb::television canada broadcast_stations 53 ))
    (defense_expenditures_number (wfb::defense_expenditures canada (* 9.0e9 $) ))
    ;;(defense_expenditures_fraction_of_gdp  (wfb::defense_expenditures canada (* 1.6 %_of_gdp) ))
    
    ))

(defparameter *countries-dimensions* 
  '(total_area land_area total_borders arable_land population population_growth_rate
               birth_rate death_rate net_migration_rate infant_mortality_rate
               life_expectancy_at_birth_total literacy_total_population total_fertility_rate
               total_labor_force national_product national_product_real_growth_rate
               national_product_per_capita unemployment_rate budget_expenditures exports_total
               imports_total external_debt electricity_capacity electricity_production
               electricity_consumption_per_capita highways_total airports_total
               television_broadcast_stations defense_expenditures_number
               ;defense_expenditures_fraction_of_gdp
               ))

(defparameter *countries*
  '(afghanistan albania algeria american_samoa andorra angola anguilla
 antigua_and_barbuda argentina armenia aruba australia austria azerbaijan the_bahamas
 bahrain bangladesh barbados belarus belgium belize benin bermuda bhutan bolivia
 bosnia_and_herzegovina botswana brazil british_virgin_islands brunei bulgaria
 burkina burma burundi cambodia cameroon canada cape_verde cayman_island
 central_african_republic chad chile china christmas_island |Cocos (Keeling) Islands|
 colombia comoros congo cook_islands costa_rica |Cote d'Ivoire| croatia cuba
 czech_republic denmark djibouti dominica dominican_republic ecuador egypt
 el_salvador equatorial_guinea eritrea estonia ethiopia
 |Falkland Islands (Islas Malvinas)| faroe_islands fiji finland france french_guiana
 french_polynesia gabon the_gambia gaza_strip georgia germany ghana gibraltar greece
 greenland grenada guadeloupe guam guatemala guernsey guinea guinea-bissau guyana
 haiti |Holy See (Vatican City)| honduras hong_kong hungary iceland india indonesia
 iran iraq ireland israel italy jamaica japan jersey johnston_atoll jordan kazakhstan
 kenya kiribati |Korea, North| |Korea, South| kuwait kyrgyzstan laos latvia lebanon
 lesotho liberia libya liechtenstein lithuania luxembourg macau
 |Macedonia, The Former Yugoslav Republic of| madagascar malawi malaysia maldives
 mali malta |Man, Isle of| marshall_islands martinique mauritania mauritius mayotte
 mexico |Micronesia, Federated States of| moldova monaco mongolia montserrat morocco
 mozambique namibia nauru nepal netherlands netherlands_antilles new_caledonia
 new_zealand nicaragua niger nigeria niue norfolk_island northern_mariana_islands
 norway oman pakistan palau panama papua_new_guinea paraguay peru philippines
 pitcairn_islands poland portugal puerto_rico qatar reunion romania russia rwanda
 saint_helena saint_kitts_and_nevis saint_lucia saint_pierre_and_miquelon
 saint_vincent_and_the_grenadines san_marino sao_tome_and_principe saudi_arabia
 senegal serbia_and_montenegro_montenegro serbia_and_montenegro_serbia seychelles
 sierra_leone singapore slovakia slovenia solomon_islands somalia spain sri_lanka
 sudan suriname svalbard swaziland sweden switzerland syria taiwan tajikistan
 tanzania thailand togo tokelau tonga trinidad_and_tobago tunisia turkey turkmenistan
 turks_and_caicos_islands tuvalu uganda ukraine united_arab_emirates united_kingdom
 united_states uruguay uzbekistan vanuatu venezuela vietnam virgin_islands
 wake_island wallis_and_futuna west_bank western_sahara western_samoa world yemen
                zaire zambia zimbabwe))

(defun make-fh-table (&optional (dimension-examples *countries-dimensions-examples*))
  (let ((fh-table (make-hash-table :test 'equal)))
    (dolist (dimension-example dimension-examples)
      (setf (gethash (car dimension-example) fh-table) 
        (make-fh-entry (cadr dimension-example))))
    fh-table))

;;fh-entry = length dimn qual atom/list
;; get-fact -- length=3 => dimn, country, quant
;;                   =4 => dimn, country, qualifer, quant
;; get-number-from-quant -- atom => atom
;;                          list => (second list)
(defun make-fh-entry (example)
  (let* ((length (length example))
         (dimn (first example))
         (qual (if (equal length 4)
                 (third example)
                 nil))
         (quant (if (listp (car (last example)))
                  'list
                  'atom))
         (fh-entry (list length dimn qual quant)))
    (values fh-entry)))

(defun satisfies-fh? (fact fh country)
  (and (eql (first fact) (second fh))
       (eql (second fact) country)
       (eql (length fact) (first fh))
       (or (and (eql (first fh) 4)
                (eql (third fact) (third fh)))
           (eql (first fh) 3))
       (cond ((and (listp (car (last fact)))
                   (equal (nth 3 fh) 'list)) t)
             ((and (atom (car (last fact)))
                   (equal (nth 3 fh) 'atom)) t)
             (t nil))))
             
(defun get-dimension-fact (country fh wfb-facts)
  (car (member country wfb-facts 
               :test #'(lambda (country fact)
                         (satisfies-fh? fact fh country)))))

(defun get-value (dimension country wfb-facts fh-table)
  (let* ((fh (gethash dimension fh-table))
         (fact (get-dimension-fact country fh wfb-facts))) 
    (if (eql (nth 3 fh) 'atom) ;; is the quantity a number or a unit expression, a list 
      (car (last fact))
      (second (car (last fact))))))
      
(defun tabulate-dimensions (wfb-facts countries dimensions fh-table)
  "for each of the countries, extract the numbers for all the dimensions"
  (let ((all-countries-data nil))
    (dolist (country countries)
      (let ((country-data (list country)))
        (dolist (dimension dimensions)
          (push (cons dimension (get-value dimension country wfb-facts fh-table)) country-data))
        (push (reverse country-data) all-countries-data)))
    (reverse all-countries-data)))

(defun dump-tabulated-data (data fname separator)
  (with-open-file (stream fname
                        :direction :output
                        :if-does-not-exist :create
                        :if-exists :supersede)
    ;; top column 
    (dolist (dimn (cdar data))
      (format stream "~A~A" separator (car dimn)))
    (format stream "~%")
    ;; one row at a time
    (dolist (country-data data)
      (format stream "~A" (car country-data))
      (dolist (dimn (cdr country-data))
        (format stream "~A~A" separator (cdr dimn)))
      (format stream "~%"))))
  
(defun args-of-list (list args)
  "Returns a new list that consists of the elements in list corresponding to
   the positions that the args specifies. If no element exists in that position
   the list contains nil there"
  (let ((args-list nil))
    (dolist (arg args)
      (push (nth arg list) args-list))
    (values (reverse args-list))))

(defun selected-facts (facts selectfun args)
  "If filterfun returns non-nil, then pushes a new list that comprises of the 
   args of the fact, and returns the list of such filtered-facts for all the 
   given facts"
  (let ((selected-facts nil))
    (dolist (fact facts)
      (if (funcall selectfun fact) 
        (push (args-of-list fact args) selected-facts)))
    (values (nreverse selected-facts))))

(defun always-true? (fact)
  (declare (ignore fact))
  t)

;;; Exports
(defun exports-fact? (fact)
  (and (eql (length fact) 3)
       (equal (car fact) 'wfb::exports)))
(defun exports-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (list (car fact) (second (second fact)))) 
    (selected-facts wfb-facts #'exports-fact? '(1 2))))

;; Net Defense expenditures, in USD
;; (wfb::defense_expenditures egypt (* 3.5e9 $) );; line 33479
(defun defense-expenditures-fact? (fact)
  (and (eql (length fact) 3)
       (equal (car fact) 'wfb::defense_expenditures)
       ;; want the net $, not the fraction
       (equal (third (third fact)) '$)))
(defun defense-expenditures-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (list (car fact) (second (second fact)))) 
    (selected-facts wfb-facts #'defense-expenditures-fact? '(1 2))))

;; Defense expenditures, as a fraction of GDP
;; (wfb::defense_expenditures algeria (* 2.7 %_of_gdp) );; line 2095
(defun defense-expenditures-fraction-fact? (fact)
  (and (eql (length fact) 3)
       (equal (car fact) 'wfb::defense_expenditures)
       (equal (third (third fact)) '%_of_gdp)))
(defun defense-expenditures-fraction-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (list (car fact) (second (second fact)))) 
    (selected-facts wfb-facts #'defense-expenditures-fraction-fact? '(1 2))))

;;; Population 
(defun population-fact? (fact)
  (and (eql (length fact) 3)
       (equal (car fact) 'wfb::population)))
(defun population-data (wfb-facts)
  (selected-facts wfb-facts #'population-fact? '(1 2)))

;;; Birth Rate
(defun birth-rate-fact? (fact)
  (and (eql (length fact) 3)
       (equal (car fact) 'wfb::birth_rate)))
(defun birth-rate-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (list (car fact) (second (second fact)))) 
    (selected-facts wfb-facts #'birth-rate-fact? '(1 2))))

;; Labor-force in government
;; (wfb::labor_force_by_occupation algeria government (* 29.5 %) );; line 1745
(defun labor-force-govt-fact? (fact)
  (and (eql (length fact) 4)
       (equal (car fact) 'wfb::labor_force_by_occupation)
       (equal (third fact) 'government)
       (not (equal (fourth fact) 'n/a))))

(defun labor-force-govt-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (if (equal fact 'n/a) 
                nil
                (list (car fact) (second (second fact))))) 
    (selected-facts wfb-facts #'labor-force-govt-fact? '(1 3))))

;; Percentage of arable land
;; (wfb::land_use el_salvador arable_land (* 27 %) );; line 33543
(defun arable-land-fact? (fact)
  (and (eql (length fact) 4)
       (equal (car fact) 'wfb::land_use)
       (equal (third fact) 'arable_land)
       (not (equal (fourth fact) 'n/a))))

(defun arable-land-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (if (equal fact 'n/a) 
                nil
                (list (car fact) (second (second fact))))) 
    (selected-facts wfb-facts #'arable-land-fact? '(1 3))))

;; Life expectancy at birth
;; (wfb::life_expectancy_at_birth el_salvador total_population (* 67.5 years) );; line 33613
(defun life-expectancy-at-birth-fact? (fact)
  (and (eql (length fact) 4)
       (equal (car fact) 'wfb::life_expectancy_at_birth)
       (equal (third fact) 'total_population)
       (not (equal (fourth fact) 'n/a))))
(defun life-expectancy-at-birth-data (wfb-facts)
  (mapcar #'(lambda (fact)
              (if (equal fact 'n/a) 
                nil
                (list (car fact) (second (second fact))))) 
    (selected-facts wfb-facts #'life-expectancy-at-birth-fact? '(1 3))))



(defun dump-list-as-table (list &optional (fname nil) (separator "  "))
  "For a list of lists, prints out each list as a row with elements
   of the list separated by separator, and a newline separating each row
   to a file. If no file is specified, the output is sent to standard
   output" 
  (if (not fname)
    (dump-list-as-table-1 list t separator) 
    (with-open-file (fout fname
                          :direction :output
                          :if-does-not-exist :create
                          :if-exists :supersede)
      (dump-list-as-table-1 list fout separator))))

(defun dump-list-as-table-1 (list stream separator)
  (dolist (l list)
    (mapcar #'(lambda (elt)
                (format stream "~A~A" elt separator)) l)
    (format stream "~%")))

(defun good-fact? (fact)
  (equal (car fact) 'a))

(defparameter test-facts '((b 1 2) (a 3 5) (a (f) g) (x y xz)))


;; Cyc dimensions
(defparameter wfb->cyc-dimensions 
  '((total_area (areaOfRegion ?x ?y)) 
    (land_area (landAreaOfRegion ?x ?y)) 
    (total_borders) 
    (arable_land) 
    (population (populationDuring ?x ?y ?z))
    (population_growth_rate) ;;missing 
    
    (birth_rate (birthRate ?x ?y)) 
    (death_rate (deathRate ?x ?y)) 
    (net_migration_rate (netMigrationRate ?x ?y))   
    (infant_mortality_rate (infantMortalityRate ?x ?y))
    
    (life_expectancy_at_birth_total) 
    (literacy_total_population) ;;missing
    (total_fertility_rate) (total_labor_force) 
    
    (national_product (grossDomesticProduct ?x ?y ?z))
    (national_product_real_growth_rate (realGDPGrowthRateDuring ?x ?y ?z)) 
    (national_product_per_capita ) ;;missing
    (unemployment_rate) ;;missing 
    (budget_expenditures)
    
    (exports_total (exportAmountDuring ?x Product ?y ?z)) 
    (imports_total (importAmountDuring ?x Product ?y ?z))
    
    (external_debt) 
    (electricity_capacity (productionCapacity ?x Electricity ?y)) 
    (electricity_production)
    (electricity_consumption_per_capita) (highways_total) (airports_total)
    (television_broadcast_stations) (defense_expenditures_number)))