Scraping fixes

plugins/tescort.lisp

@@ -56,6 +56,7 @@
                "#form_PopUp"
                (("username" . escortnews-login)
                 ("password" . escortnews-pass))))
+     (:validate-init . "button.signout")
      (:validate . ".countSearchBar"))
     ))
 
@@ -68,7 +69,12 @@
       (apply #'http-request url args)
     (unless (stringp body)
       (setf body (babel:octets-to-string body :encoding :utf-8)))
-    (values (plump:parse (agets (yason:parse body :object-as object-as) "html")) status headers uri)))
+    (values (plump:parse
+	     (agets
+	      (or (ignore-errors (yason:parse body :object-as object-as))
+		  `(("html" . ,body)))
+	      "html"))
+	    status headers uri)))
 
 (defvar *tescort-login*)
 (defvar *tescort-pass*)

scraping.lisp

@@ -108,16 +108,17 @@
          (request (agets scrape :request))
          (processor (or (agets scrape :processor) #'xml-request))
          (init (agets info :init))
-         (validate (agets info :validate)))
+         (validate (agets info :validate))
+	 (validate-init (or (agets info :validate-init) validate)))
     (labels ((return-info ()
                (extract-items-info (agets scrape :items)
                                    (agets scrape :info))))
       (unless (have-cookies request)
-        (do-form init context validate))
+        (do-form init context validate-init))
       (with-request (request context processor)
         (if (do-validate validate)
             (return-info)
             (progn
-              (do-form init context validate)
+              (do-form init context validate-init)
               (with-request (request context)
                 (return-info))))))))