|
|
@@ -52,7 +52,7 @@
|
|
|
(sleep backoff)
|
|
|
(setf backoff (min *backoff-max*
|
|
|
(* 2 backoff))))
|
|
|
- (bordeaux-threads:timeout (e)
|
|
|
+ (usocket:timeout-error (e)
|
|
|
(log:error e)
|
|
|
(log:info "Backing off for" backoff)
|
|
|
(sleep backoff)
|
|
|
@@ -145,33 +145,54 @@ is replaced with replacement."
|
|
|
(defun spaced (list)
|
|
|
(format nil "~{~A~^ ~}" list))
|
|
|
|
|
|
-(defun http-default (url)
|
|
|
- (let ((uri (quri:uri url)))
|
|
|
- (quri:render-uri
|
|
|
- (if (null (quri:uri-scheme uri))
|
|
|
- (quri:uri (format nil "http://~A" url))
|
|
|
- uri))))
|
|
|
+(defun http-default (url &optional parameters)
|
|
|
+ (let* ((uri (quri:uri url))
|
|
|
+ (userinfo (quri:uri-userinfo uri)))
|
|
|
+ (when parameters
|
|
|
+ (let ((query (quri:url-encode-params parameters :encoding :utf-8)))
|
|
|
+ (setf (quri:uri-query uri)
|
|
|
+ (if (and (quri:uri-query uri)
|
|
|
+ (string-not-equal (quri:uri-query uri) ""))
|
|
|
+ (concatenate 'string (quri:uri-query uri) "&" query)
|
|
|
+ query))))
|
|
|
+ (when userinfo
|
|
|
+ (setf (quri:uri-userinfo uri) nil))
|
|
|
+ (unless (quri:uri-scheme uri)
|
|
|
+ (setf (quri:uri-scheme uri) "http"))
|
|
|
+ (values uri userinfo)))
|
|
|
+
|
|
|
+(defun http-request (url &rest args &key method version parameters content headers basic-auth cookie-jar keep-alive use-connection-pool (max-redirects 5) timeout force-binary want-stream ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path user-agent)
|
|
|
+ (declare (ignore method version content basic-auth cookie-jar keep-alive use-connection-pool max-redirects timeout force-binary want-stream ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path))
|
|
|
+ (multiple-value-bind (uri userinfo)
|
|
|
+ (http-default url parameters)
|
|
|
+ (when userinfo
|
|
|
+ (push (cons :authorization (concatenate 'string "Basic "
|
|
|
+ (base64:string-to-base64-string userinfo)))
|
|
|
+ headers))
|
|
|
+ (when user-agent
|
|
|
+ (push (cons :user-agent user-agent) headers)
|
|
|
+ (remf args :user-agent))
|
|
|
+ (remf args :parameters)
|
|
|
+ (apply #'dex:request uri :headers headers args)))
|
|
|
|
|
|
;; XML processing
|
|
|
-(defun xml-request (url &key encoding parameters)
|
|
|
- (multiple-value-bind (raw-body status headers uri http-stream)
|
|
|
- (drakma:http-request (http-default url)
|
|
|
- :parameters parameters
|
|
|
- :external-format-out :utf-8
|
|
|
- :force-binary t
|
|
|
- :decode-content t)
|
|
|
- (declare (ignore status http-stream))
|
|
|
+(defun xml-request (url &rest args &key method parameters content headers basic-auth cookie-jar keep-alive use-connection-pool timeout ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path user-agent encoding)
|
|
|
+ (declare (ignore method parameters headers content basic-auth cookie-jar keep-alive use-connection-pool timeout ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path user-agent))
|
|
|
+ (remf args :encoding)
|
|
|
+ (multiple-value-bind (raw-body status headers uri)
|
|
|
+ (apply #'http-request url :force-binary t args)
|
|
|
(let ((encoding
|
|
|
(or
|
|
|
;; 1. Provided encoding
|
|
|
encoding
|
|
|
;; 2. Content-type header
|
|
|
(ignore-errors
|
|
|
- (let ((ct (aget :content-type headers)))
|
|
|
+ (let ((ct (gethash "content-type" headers)))
|
|
|
(subseq ct (1+ (position #\= ct)))))
|
|
|
;; 3. Parse first 1000 bytes
|
|
|
(ignore-errors
|
|
|
- (let ((dom (plump:parse (flex:octets-to-string (subseq raw-body 0 1000)))))
|
|
|
+ (let ((dom (plump:parse (flex:octets-to-string
|
|
|
+ (subseq raw-body 0 (1+ (position (char-code #\>) raw-body :start 1000)))))))
|
|
|
(or
|
|
|
;; 3.1 Content-type from http-equiv
|
|
|
(ignore-errors
|
|
|
@@ -190,7 +211,7 @@ is replaced with replacement."
|
|
|
(lambda (c) (use-value #\? c))))
|
|
|
(plump:parse
|
|
|
(flex:octets-to-string raw-body :external-format (intern encoding 'keyword))))
|
|
|
- uri))))
|
|
|
+ status headers uri))))
|
|
|
|
|
|
(defun get-by-tag (node tag)
|
|
|
(nreverse (org.shirakumo.plump.dom::get-elements-by-tag-name node tag)))
|
|
|
@@ -200,20 +221,16 @@ is replaced with replacement."
|
|
|
(string-trim '(#\Newline #\Space #\Return) (plump:text (elt (clss:select selector node) 0)))))
|
|
|
|
|
|
;; JSON processing
|
|
|
-(defun json-request (url &key (method :get) parameters content additional-headers (object-as :alist))
|
|
|
- (multiple-value-bind (stream status headers uri http-stream)
|
|
|
- (drakma:http-request (http-default url) :method method
|
|
|
- :parameters parameters
|
|
|
- :content content :content-type "application/json"
|
|
|
- :additional-headers additional-headers
|
|
|
- :external-format-out :utf-8
|
|
|
- :force-binary t :want-stream t :decode-content t)
|
|
|
- (declare (ignore status headers))
|
|
|
- (unwind-protect
|
|
|
- (progn
|
|
|
- (setf (flex:flexi-stream-external-format stream) :utf-8)
|
|
|
- (values (yason:parse stream :object-as object-as) uri))
|
|
|
- (ignore-errors (close http-stream)))))
|
|
|
+(defun json-request (url &rest args &key method parameters content headers basic-auth cookie-jar keep-alive use-connection-pool timeout ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path user-agent (object-as :alist))
|
|
|
+ (declare (ignore method parameters basic-auth cookie-jar keep-alive use-connection-pool timeout ssl-key-file ssl-cert-file ssl-key-password stream verbose proxy insecure ca-path user-agent))
|
|
|
+ (remf args :object-as)
|
|
|
+ (when content
|
|
|
+ (push (cons :content-type "application/json") headers))
|
|
|
+ (multiple-value-bind (body status headers uri)
|
|
|
+ (apply #'http-request url args)
|
|
|
+ (unless (stringp body)
|
|
|
+ (setf body (babel:octets-to-string body :encoding :utf-8)))
|
|
|
+ (values (yason:parse body :object-as object-as) status headers uri)))
|
|
|
|
|
|
(defun plist-hash (plist &optional skip-nil (format-key #'identity) &rest hash-table-initargs)
|
|
|
(cond
|