Twitter importer

src/foursquare.lisp

@@ -2,10 +2,11 @@
 (defpackage #:timeliner.foursquare
   (:use :cl #:timeliner.utils :cl-mongo)
   (:export
+   #:*access-token*
    #:on-cron))
 (in-package #:timeliner.foursquare)
 
-(defvar *access-token* "ZLLBUQAWZJLWURZNGQRYN5J5XYYFDHMOSMTTCQYXYTSZUYCE")
+(defvar *access-token*)
 
 (defun make-checkin-doc (checkin)
   (let* ((venue (gethash "venue" checkin))
@@ -47,14 +48,10 @@
      for doc = (make-checkin-doc item)
      when doc collect doc))
 
-(defun save-checkins (checkins)
-  (dolist (checkin checkins)
-    (db.insert "events" checkin)))
-
 (defun on-cron ()
   (let* ((last-checkin (first (docs (db.sort "events" ($ "type" "checkin") :field "ts"
                                              :asc nil :limit 1))))
          (from (and last-checkin (ms->ts (cl-mongo::raw (get-element "ts" last-checkin)))))
          (checkins (load-checkins from)))
     (format t "Got ~A checkins from ~A~%" (length checkins) from)
-    (save-checkins checkins)))
+    (save-events checkins)))

src/locations.lisp

@@ -184,7 +184,7 @@
        for doc in docs
        for docMs = (get-element "timestampMs" doc)
        when (<= docMs ms) do (setf less doc)
-       when (and (not greater) (>= docMs ms)) do (setf greater doc))
+       when (and (not greater) (> docMs ms)) do (setf greater doc))
     (when (and less greater)
       (geo:interpolate-between-points
        (doc->point less)

src/twitter.lisp

@@ -0,0 +1,57 @@
+(in-package :cl-user)
+(defpackage #:timeliner.twitter
+  (:use :cl #:timeliner.utils :cl-mongo)
+  (:export
+   #:*access-token*
+   #:on-cron))
+(in-package #:timeliner.twitter)
+
+(defvar *access-token*)
+
+(defun make-twitter-doc (twit)
+  (let* ((ts (local-time:universal-to-timestamp
+              (cl-date-time-parser:parse-date-time
+               (aget "created_at" twit)))))
+    (kv
+     (kv :ts ts)
+     (kv :type "twitter")
+     (kv :title (aget "text" twit))
+     (kv :language (aget "lang" twit))
+     (kv :twitter (kv
+                   (kv "id" (aget "id" twit))
+                   (kv "reply_to" (aget "in_reply_to_status_id" twit))
+                   (kv "retweeted" (aget "retweeted" twit))))
+     (kv :loc (timeliner.locations:point->doc
+               (timeliner.locations:find-location-at (ts->ms ts)))))))
+
+(defparameter *timeline-url* "https://api.twitter.com/1.1/statuses/user_timeline.json")
+
+(defun load-twits (since-id)
+  (loop
+     for max-id = nil then (1- (aget "id" (car (last data))))
+     for data = (yason:parse
+                 (flexi-streams:octets-to-string
+                  (cl-oauth:access-protected-resource
+                   (format nil "~A?~A"
+                           *timeline-url*
+                           (cl-oauth::alist->query-string
+                            (remove-if
+                             (complement #'cdr)
+                             (list
+                              (cons "count" 200)
+                              (cons "trim_user" 1)
+                              (cons "since_id" since-id)
+                              (cons "max_id" max-id)))
+                            :include-leading-ampersand nil))
+                   *access-token*))
+                 :object-as :alist)
+     while data
+     append (mapcar #'make-twitter-doc data)))
+
+(defun on-cron ()
+  (let* ((last-twit (first (docs (db.sort "events" ($ "type" "twitter") :field "ts"
+                                          :asc nil :limit 1))))
+         (since-id (if last-twit (get-element "twitter.id" last-twit) 1))
+         (twits (load-twits since-id)))
+    (format t "Got ~A twits from ~A~%" (length twits) since-id)
+    (save-events twits)))

src/utils.lisp

@@ -11,7 +11,8 @@
    #:aget
    #:doc->plist
    #:load-chrome-cookie-jar
-   #:today))
+   #:today
+   #:save-events))
 (in-package :timeliner.utils)
 
 (defun ts->ms (ts)
@@ -29,6 +30,10 @@
     (set :sec 0)
     (set :nsec 0)))
 
+(defun save-events (events)
+  (dolist (e events)
+    (db.insert "events" e)))
+
 (defgeneric $between (a from to)
   (:documentation "cl-mongo between query"))
 
@@ -48,7 +53,7 @@
            with))
 
 (defmacro aget (key alist)
-  `(cdr (assoc ,key ,alist :test #'string=)))
+  `(cdr (assoc ,key ,alist :test #'equal)))
 
 (defvar *decode-doc-to* :plist "Document object type")
 (defgeneric decode-element (element)

src/web.lisp

@@ -15,13 +15,16 @@
 (defvar *run-cron* t "Controls if starting a web should run cron tasks")
 (defvar *crons* (list
                  (list #'timeliner.locations:on-cron '(:minute 0 :hour *))
-                 (list #'timeliner.financisto:on-cron '(:minute 0 :hour 7))
-                 (list #'timeliner.foursquare:on-cron '(:minute 5 :hour *)))
+                 (list #'timeliner.twitter:on-cron '(:minute 1 :hour *))
+                 (list #'timeliner.foursquare:on-cron '(:minute 5 :hour *))
+                 (list #'timeliner.financisto:on-cron '(:minute 0 :hour 7)))
   "List of cron functions with their schedules")
 (defvar *cron-timers* nil)
 
 (defmethod restas:initialize-module-instance :before ((module (eql #.*package*)) context)
   (restas:with-context context
+    (alexandria:when-let (file (probe-file "config.lisp"))
+      (load file))
     (cl-mongo:mongo :host "10.8.0.6")
     (cl-mongo:db.use "timeline")
     (when *run-cron*
@@ -349,7 +352,8 @@
                             (case (! this model (get :type))
                               (:place "glyphicon glyphicon-map-marker")
                               (:finance "glyphicon glyphicon-usd")
-                              (:checkin "glyphicon glyphicon-ok-circle")))
+                              (:checkin "glyphicon glyphicon-ok-circle")
+                              (:twitter "glyphicon glyphicon-pencil")))
                render (lambda ()
                         (! this $el (attr (create
                                            :href "#"

timeliner.asd

@@ -48,17 +48,21 @@ THE SOFTWARE.
                :css-lite
                :sqlite
                :gzip-stream
-               :clon)
+               :clon
+               :cl-oauth
+               :cl-date-time-parser)
   :components ((:module "src"
                 :components
                 ((:file "utils")
                  (:file "locations" :depends-on ("utils"))
                  (:file "financisto" :depends-on ("utils"))
                  (:file "foursquare" :depends-on ("utils"))
+                 (:file "twitter" :depends-on ("utils"))
                  (:file "web" :depends-on ("utils"
                                            "locations"
                                            "financisto"
-                                           "foursquare")))))
+                                           "foursquare"
+                                           "twitter")))))
   :description "Personal Timeline"
   :long-description
   #.(with-open-file (stream (merge-pathnames