Financisto import

process-financisto.lisp

@@ -1,4 +1,4 @@
-(defvar *financisto-backup-path* #P"/home/enikesha/Documents/backups/financisto/")
+(defvar *financisto-backup-path* #P"/home/enikesha/dev/lisp/timeliner/")
 
 (defun find-last-backup ()
   (first (sort (mapcar #'namestring
@@ -52,19 +52,21 @@
          (from-account (gethash (parse-integer (getf e :from-account-id)) accounts))
          (from-currency (gethash (parse-integer (getf from-account :currency-id)) currencies))
          (from-decimals (expt 10 (parse-integer (getf from-currency :decimals))))
-         (from-amount (/ (parse-integer (getf e :from-amount)) from-decimals))
-         (from-balance (/ (gethash (parse-integer (getf from-account :-id)) balances 0) from-decimals))
+         (from-amount (float (/ (parse-integer (getf e :from-amount)) from-decimals)))
+         (from-balance (float (/ (gethash (parse-integer (getf from-account :-id)) balances 0) from-decimals)))
          (spend (< from-amount 0))         
          (original-currency (gethash (parse-integer (getf e :original-currency-id)) currencies))
-         (original-from-amount (and original-currency (/ (parse-integer (getf e :original-from-amount)) (expt 10 (parse-integer (getf original-currency :decimals))))))
+         (original-from-amount (and original-currency
+                                    (float (/ (parse-integer (getf e :original-from-amount))
+                                              (expt 10 (parse-integer (getf original-currency :decimals)))))))
          (payee (gethash (parse-integer (getf e :payee-id)) (gethash "payee" db)))
          (category (gethash (parse-integer (getf e :category-id)) (gethash "category" db)))
          
          (to-account (gethash (parse-integer (getf e :to-account-id)) accounts))
          (to-currency (and to-account (gethash (parse-integer (getf to-account :currency-id)) currencies)))
          (to-decimals (and to-account (expt 10 (parse-integer (getf to-currency :decimals)))))
-         (to-amount (and to-account (/ (parse-integer (getf e :to-amount)) to-decimals)))
-         (to-balance (and to-account (/ (gethash (parse-integer (getf to-account :-id)) balances 0) to-decimals)))
+         (to-amount (and to-account (float (/ (parse-integer (getf e :to-amount)) to-decimals))))
+         (to-balance (and to-account (float (/ (gethash (parse-integer (getf to-account :-id)) balances 0) to-decimals))))
 
          (transfer (cond
                      ((and to-account (equal (getf from-account :type)
@@ -112,8 +114,24 @@
                      (and payee (getf payee :title))
                      (getf e :note)
                      from-balance (getf from-currency :symbol))))
-         (type (if transfer (string-downcase (string transfer)) (if spend "spend" "earned"))))
-    
+         (type (if transfer (string-downcase (string transfer)) (if spend "spend" "earned")))
+         (financisto (if transfer
+                         (kv "transfer" (kv
+                                         (kv "from_account" (getf from-account :title))
+                                         (kv "from_amount" from-amount)
+                                         (kv "from_currency" (getf from-currency :name))
+                                         (kv "from_balance" from-balance)
+                                         (kv "to_account" (getf to-account :title))
+                                         (kv "to_amount" to-amount)
+                                         (kv "to_currency" (getf to-currency :name))
+                                         (kv "to_balance" to-balance)))
+                         (kv "transaction" (kv
+                                            (kv "account" (getf from-account :title))
+                                            (kv "amount" from-amount)
+                                            (kv "currency" (getf from-currency :name))
+                                            (kv "balance" from-balance)
+                                            (kv "original_amount" original-from-amount)
+                                            (kv "original_currency" (getf original-currency :name)))))))
     (kv
      (kv "ts" (parse-integer (getf e :datetime)))
      (kv "type" "finance")
@@ -122,9 +140,10 @@
          (kv
           (kv "id" (parse-integer (getf e :-id)))
           (kv "type" type)
-          ))
-     )
-    ))
+          (kv "payee" (getf payee :title))
+          (kv "category" (and category (not (equal (getf category :-id) "0")) (getf category :title)))
+          (kv "note" (getf e :note))
+          financisto)))))
 
 (defun financisto-import ()
   (let* ((filename (find-last-backup))
@@ -135,15 +154,28 @@
                              #'<
                              :key #'(lambda (i) (parse-integer (getf i :datetime)))))
          (balances (make-hash-table :size (hash-table-count (gethash "account" db)))))
-    (format t "Got ~A with ~A transactions" filename (length transactions))
+    (format t "Got ~A with ~A transactions~%" filename (length transactions))
     (dolist (entity transactions)
       (destructuring-bind (&key from-account-id from-amount
                                 to-account-id to-amount
-                                &allow-other-keys) entity        
+                                -id datetime
+                                &allow-other-keys) entity
         (incf (gethash (parse-integer from-account-id) balances 0) (parse-integer from-amount))
         (when (not (string= to-account-id "0"))
           (incf (gethash (parse-integer to-account-id) balances 0) (parse-integer to-amount)))
-        
-        ))
-    (loop for k being the hash-keys in balances using (hash-value v)
-       collect (list k (getf (gethash k (gethash "account" db)) :title) v))))
+        (let* ((new-doc (make-finance-doc entity db balances))
+               (existing-doc (first (docs (db.find "events"
+                                                   (kv ($ "type" "finance")
+                                                       ($ "financisto.id" (parse-integer -id)))))))
+               (new-ts (parse-integer datetime)))
+          (cl-mongo::kv-container-add
+           (kv "loc"
+               (if (and existing-doc (= new-ts (get-element "ts" existing-doc)))
+                   (get-element "loc" existing-doc)
+                   (point->doc (find-location-at new-ts))))
+           new-doc)
+          (if existing-doc
+              (progn
+                (cl-mongo::kv-container-add (kv "_id" (doc-id existing-doc)) new-doc)
+                (db.save "events" new-doc))
+              (db.insert "events" new-doc)))))))

process-locations.lisp

@@ -115,14 +115,16 @@
      collect (cons (ms->ts (parse-integer (cadr loc)))
                    (geo:point-deg (caddr loc) (cadddr loc)))))
 
+(defun point->doc (point)
+  (when point
+    (kv (kv :type "Point") ; GeoJSON Point
+        (kv :coordinates (list (geo:longitude-deg point)
+                               (geo:latitude-deg point))))))
+
 (defun make-location-doc (loc)
-  (let* ((point (cdr loc))
-         (lat (geo:latitude-deg point))
-         (lon (geo:longitude-deg point)))
-    (kv
-     (kv "timestampMs" (ts->ms (car loc)))
-     (kv "loc" (kv (kv :type "Point") ; GeoJSON Point
-                   (kv :coordinates (list lon lat)))))))
+  (kv
+   (kv "timestampMs" (ts->ms (car loc)))
+   (kv "loc" (point->doc (cdr loc)))))
 
 (defun save-locations (locations)
   (loop for loc in locations
@@ -135,3 +137,26 @@
     (save-locations locs)
     (format t "Saving ~A place events for ~A~%" (length places) from)
     (save-places places)))
+
+(defun find-location-at (ms)
+  (let* ((ts (ms->ts ms))
+         (docs (docs (db.sort "locations"
+                              ($between
+                               "timestampMs"
+                               (ts->ms (local-time:timestamp- ts 5 :minute))
+                               (ts->ms (local-time:timestamp+ ts 5 :minute)))
+                              :field "timestampMs" :limit 15)))
+         less
+         greater)
+    (loop
+       for doc in docs
+       for docMs = (get-element "timestampMs" doc)
+       when (<= docMs ms) do (setf less doc)
+       when (and (not greater) (>= docMs ms)) do (setf greater doc))
+    (when (and less greater)
+      (geo:interpolate-between-points
+       (doc->point less)
+       (doc->point greater)
+       (/ (- ms (get-element "timestampMs" less))
+          (- (get-element "timestampMs" greater)
+             (get-element "timestampMs" less)))))))