New google-location export API

src/locations.lisp

@@ -9,6 +9,7 @@
    #:on-cron))
 (in-package #:timeliner.locations)
 
+(defparameter *google-kml-url* "https://www.google.com/maps/timeline/kml")
 
 (defun reverse-geocode (lat lon)
   (let* ((data (yason:parse
@@ -101,37 +102,68 @@
        finally (return ts))))
 
 (defvar *google-cookie-jar*)
-(defvar *google-manual-header*)
+;; (defvar *google-manual-header*)
+
+;; (defun load-history (from &optional (to (local-time:timestamp+ from 1 :day)) (trim nil))
+;;   (loop for loc in (cadadr (yason:parse
+;;                             (flexi-streams:octets-to-string
+;;                              (drakma:http-request
+;;                               "https://maps.google.com/locationhistory/b/0/apps/pvjson?t=0"
+;;                               :method :post
+;;                               :user-agent "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36"
+;;                               :content (format nil "[null,~A,~A,~A]"
+;;                                                (ts->ms from)
+;;                                                (ts->ms to)
+;;                                                (if trim "true" "false"))
+;;                               :cookie-jar *google-cookie-jar*
+;;                               :additional-headers (list
+;;                                                    (cons "x-manualheader" *google-manual-header*)
+;;                                                    '("referer" . "https://maps.google.com/locationhistory/b/0"))))))
+;;      collect (cons (ms->ts (parse-integer (cadr loc)))
+;;                    (geo:point-deg (caddr loc) (cadddr loc)))))
+
+;; (defun get-history-xsrf-token ()
+;;   (let* ((page (drakma:http-request
+;;                 "https://maps.google.com/locationhistory/b/0/"
+;;                 :user-agent "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36"
+;;                 :cookie-jar *google-cookie-jar*))
+;;          (needle "window.LatitudeServerConstants['XsrfToken'] = '")
+;;          (start (search needle page)))
+;;     (when start
+;;       (subseq page
+;;               (+ start (length needle))
+;;               (position #\' page :start (+ start (length needle)))))))
+
+(defun get-kml-url (from to)
+  (format nil "~A?authuser=0&pb=!1m8!1m3!1i~D!2i~D!3i~D!2m3!1i~D!2i~D!3i~D"
+          *google-kml-url*
+          (local-time:timestamp-year from)
+          (1- (local-time:timestamp-month from))
+          (local-time:timestamp-day from)
+          (local-time:timestamp-year to)
+          (1- (local-time:timestamp-month to))
+          (local-time:timestamp-day to)))
 
 (defun load-history (from &optional (to (local-time:timestamp+ from 1 :day)) (trim nil))
-  (loop for loc in (cadadr (yason:parse
-                            (flexi-streams:octets-to-string
-                             (drakma:http-request
-                              "https://maps.google.com/locationhistory/b/0/apps/pvjson?t=0"
-                              :method :post
-                              :user-agent "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36"
-                              :content (format nil "[null,~A,~A,~A]"
-                                               (ts->ms from)
-                                               (ts->ms to)
-                                               (if trim "true" "false"))
-                              :cookie-jar *google-cookie-jar*
-                              :additional-headers (list
-                                                   (cons "x-manualheader" *google-manual-header*)
-                                                   '("referer" . "https://maps.google.com/locationhistory/b/0"))))))
-     collect (cons (ms->ts (parse-integer (cadr loc)))
-                   (geo:point-deg (caddr loc) (cadddr loc)))))
-
-(defun get-history-xsrf-token ()
-  (let* ((page (drakma:http-request
-                "https://maps.google.com/locationhistory/b/0/"
-                :user-agent "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36"
-                :cookie-jar *google-cookie-jar*))
-         (needle "window.LatitudeServerConstants['XsrfToken'] = '")
-         (start (search needle page)))
-    (when start
-      (subseq page
-              (+ start (length needle))
-              (position #\' page :start (+ start (length needle)))))))
+  (declare (ignore trim))
+  (remove-if #'(lambda (p) (not (local-time:timestamp< from (car p) to)))
+             (extract-points (get-track (s-xml:parse-xml
+                                         (drakma:http-request (get-kml-url from to)
+                                                              :cookie-jar *google-cookie-jar*
+                                                              :want-stream t))))))
+
+(defun get-track (kml)
+  "Returns when-gx:coord list in <gx:track>"
+  (cddadr (cddadr (cddddr (cddadr kml)))))
+
+(defun loc-to-point (string-loc)
+  (let ((triple (read-from-string (concatenate 'string "(" string-loc ")"))))
+    (geo:point-deg (second triple) (first triple))))
+
+(defun extract-points (kml-track)
+  (loop for ((a w) (b c)) on kml-track by #'cddr
+     collect (cons (local-time:parse-timestring w)
+                   (loc-to-point c))))
 
 (defun point->doc (point)
   (when point
@@ -161,7 +193,7 @@
 
 (defun import-location-events (from &optional (to (local-time:timestamp+ from 1 :day)))
   (let* ((*google-cookie-jar* (load-chrome-cookie-jar ".google.com"))
-         (*google-manual-header* (get-history-xsrf-token))
+         ;; (*google-manual-header* (get-history-xsrf-token))
          (locs (load-history from to nil)))
     (log:info "Saving ~A locations for ~A~%" (length locs) from)
     (save-locations locs)

src/web.lisp

@@ -18,7 +18,8 @@
                  (list #'timeliner.locations:on-cron '(:minute 0 :hour *))
                  (list #'timeliner.twitter:on-cron '(:minute 3 :hour *))
                  (list #'timeliner.foursquare:on-cron '(:minute 5 :hour *))
-                 (list #'timeliner.financisto:on-cron '(:minute 0 :hour 7)))
+                 (list #'timeliner.financisto:on-cron '(:minute 0 :hour 7))
+                 (list #'timeliner.photos:on-cron '(:minute 0 :hour 6)))
   "List of cron functions with their schedules")
 (defvar *cron-timers* nil)
 

timeliner.asd

@@ -53,7 +53,8 @@ THE SOFTWARE.
                :cl-date-time-parser
                :log4cl
                :cl-fad
-               :zpb-exif)
+               :zpb-exif
+               :s-xml)
   :components ((:module "src"
                 :components
                 ((:file "utils")