From 2fd6bfa9b56008dfb06ebd3a04cd9acb38f5b0f3 Mon Sep 17 00:00:00 2001 From: tbussmann Date: Thu, 27 Oct 2016 18:10:26 +0200 Subject: [PATCH 1/4] Add HEAD method, prepare for version 1.3 - add a http_method HEAD which does not follow redirections - prepare for a version 1.3 incl. migration from 1.1 --- META.json | 6 ++-- Makefile | 2 +- README.md | 23 +++++++++++++- http--1.1--1.3.sql | 14 +++++++++ http--1.3.sql | 77 ++++++++++++++++++++++++++++++++++++++++++++++ http.c | 20 +++++++++--- http.control | 2 +- 7 files changed, 133 insertions(+), 11 deletions(-) create mode 100755 http--1.1--1.3.sql create mode 100644 http--1.3.sql diff --git a/META.json b/META.json index 3c9a960..a49d8e8 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "http", "abstract": "HTTP client for PostgreSQL", "description": "HTTP allows you to get the content of a web page in a SQL function call.", - "version": "1.1", + "version": "1.3", "maintainer": [ "Paul Ramsey " ], @@ -21,9 +21,9 @@ }, "provides": { "http": { - "file": "http--1.1.sql", + "file": "http--1.3.sql", "docfile": "README.md", - "version": "1.1", + "version": "1.3", "abstract": "HTTP client for PostgreSQL" }, }, diff --git a/Makefile b/Makefile index f70b041..8951470 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ MODULE_big = http OBJS = http.o EXTENSION = http -DATA = http--1.1.sql http--1.0--1.1.sql +DATA = http--1.3.sql http--1.1.sql http--1.0--1.1.sql REGRESS = http EXTRA_CLEAN = diff --git a/README.md b/README.md index 142313c..b644eb8 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,26 @@ To access binary content, you must coerce the content from the default `varchar` --------------+---------------+---------------- image/gif | 31958 | 31958 +To access only the header you can do a HEAD-Request. This will not follow redirections. + + WITH + http AS ( + SELECT * FROM http_head('http://google.com') + ), + headers AS ( + SELECT (unnest(headers)).* FROM http + ) + SELECT + http.status, + headers.value AS location + FROM http + LEFT OUTER JOIN headers + ON (headers.field = 'Location'); + + status | location + --------+----------------------------------------------------------- + 302 | http://www.google.ch/?gfe_rd=cr&ei=ACESWLy_KuvI8zeghL64Ag + ## Concepts Every HTTP call is a made up of an `http_request` and an `http_response`. @@ -133,7 +153,7 @@ Every HTTP call is a made up of an `http_request` and an `http_response`. headers | http_header[] | content | character varying | -The utility functions, `http_get()`, `http_post()`, `http_put()`, and `http_delete()` are just wrappers around a master function, `http(http_request)` that returns `http_response`. +The utility functions, `http_get()`, `http_post()`, `http_put()`, `http_delete()` and `http_head()` are just wrappers around a master function, `http(http_request)` that returns `http_response`. The `headers` field for requests and response is a PostgreSQL array of type `http_header` which is just a simple tuple. @@ -167,6 +187,7 @@ By default a 5 second timeout is set for the completion of a request. If a diff * `http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` * `http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` * `http_delete(uri VARCHAR)` returns `http_resonse` +* `http_head(uri VARCHAR)` returns `http_resonse` * `urlencode(string VARCHAR)` returns `text` ## Installation diff --git a/http--1.1--1.3.sql b/http--1.1--1.3.sql new file mode 100755 index 0000000..6c8b1fd --- /dev/null +++ b/http--1.1--1.3.sql @@ -0,0 +1,14 @@ + +ALTER DOMAIN http_method DROP CONSTRAINT http_method_check; +ALTER DOMAIN http_method ADD CHECK ( + VALUE ILIKE 'get' OR + VALUE ILIKE 'post' OR + VALUE ILIKE 'put' OR + VALUE ILIKE 'delete' OR + VALUE ILIKE 'head' +); + +CREATE OR REPLACE FUNCTION http_head(uri VARCHAR) + RETURNS http_response + AS $$ SELECT http(('HEAD', $1, NULL, NULL, NULL)::http_request) $$ + LANGUAGE 'sql'; diff --git a/http--1.3.sql b/http--1.3.sql new file mode 100644 index 0000000..f1f408e --- /dev/null +++ b/http--1.3.sql @@ -0,0 +1,77 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION http" to load this file. \quit + +CREATE DOMAIN http_method AS text +CHECK ( + VALUE ILIKE 'get' OR + VALUE ILIKE 'post' OR + VALUE ILIKE 'put' OR + VALUE ILIKE 'delete' OR + VALUE ILIKE 'head' +); + +CREATE DOMAIN content_type AS text +CHECK ( + VALUE ~ '^\S+\/\S+' +); + +CREATE TYPE http_header AS ( + field VARCHAR, + value VARCHAR +); + +CREATE TYPE http_response AS ( + status INTEGER, + content_type VARCHAR, + headers http_header[], + content VARCHAR +); + +CREATE TYPE http_request AS ( + method http_method, + uri VARCHAR, + headers http_header[], + content_type VARCHAR, + content VARCHAR +); + +CREATE OR REPLACE FUNCTION http_header (field VARCHAR, value VARCHAR) + RETURNS http_header + AS $$ SELECT $1, $2 $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION http(request http_request) + RETURNS http_response + AS 'MODULE_PATHNAME', 'http_request' + LANGUAGE 'c'; + +CREATE OR REPLACE FUNCTION http_get(uri VARCHAR) + RETURNS http_response + AS $$ SELECT http(('GET', $1, NULL, NULL, NULL)::http_request) $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) + RETURNS http_response + AS $$ SELECT http(('POST', $1, NULL, $3, $2)::http_request) $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) + RETURNS http_response + AS $$ SELECT http(('PUT', $1, NULL, $3, $2)::http_request) $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR) + RETURNS http_response + AS $$ SELECT http(('DELETE', $1, NULL, NULL, NULL)::http_request) $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION http_head(uri VARCHAR) + RETURNS http_response + AS $$ SELECT http(('HEAD', $1, NULL, NULL, NULL)::http_request) $$ + LANGUAGE 'sql'; + +CREATE OR REPLACE FUNCTION urlencode(string VARCHAR) + RETURNS TEXT + AS 'MODULE_PATHNAME' + LANGUAGE 'c' + IMMUTABLE STRICT; diff --git a/http.c b/http.c index ec0f8b4..3226faa 100644 --- a/http.c +++ b/http.c @@ -70,7 +70,8 @@ typedef enum { HTTP_GET, HTTP_POST, HTTP_DELETE, - HTTP_PUT + HTTP_PUT, + HTTP_HEAD } http_method; /* Components (and postitions) of the http_request tuple type */ @@ -213,6 +214,8 @@ request_type(const char *method) return HTTP_PUT; else if ( strcasecmp(method, "DELETE") == 0 ) return HTTP_DELETE; + else if ( strcasecmp(method, "HEAD") == 0 ) + return HTTP_HEAD; else return HTTP_GET; } @@ -604,10 +607,13 @@ Datum http_request(PG_FUNCTION_ARGS) /* Set the HTTP content encoding to gzip */ /*curl_easy_setopt(g_http_handle, CURLOPT_ACCEPT_ENCODING, HTTP_ENCODING);*/ - /* Follow redirects, as many as 5 */ - CURL_SETOPT(g_http_handle, CURLOPT_FOLLOWLOCATION, 1); - CURL_SETOPT(g_http_handle, CURLOPT_MAXREDIRS, 5); - + if ( method != HTTP_HEAD ) + { + /* Follow redirects, as many as 5 */ + CURL_SETOPT(g_http_handle, CURLOPT_FOLLOWLOCATION, 1); + CURL_SETOPT(g_http_handle, CURLOPT_MAXREDIRS, 5); + } + if ( g_use_keepalive ) { /* Add a keep alive option to the headers to reuse network sockets */ @@ -678,6 +684,10 @@ Datum http_request(PG_FUNCTION_ARGS) { CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "DELETE"); } + else if ( method == HTTP_HEAD ) + { + CURL_SETOPT(g_http_handle, CURLOPT_NOBODY, 1); + } /* Set the headers */ CURL_SETOPT(g_http_handle, CURLOPT_HTTPHEADER, headers); diff --git a/http.control b/http.control index 942e687..a452b0b 100644 --- a/http.control +++ b/http.control @@ -1,4 +1,4 @@ -default_version = '1.1' +default_version = '1.3' module_pathname = '$libdir/http' relocatable = true comment = 'HTTP client for PostgreSQL, allows web page retrieval inside the database.' From 898dd0a7d85c70554f82a3f7bf996023c979dbf4 Mon Sep 17 00:00:00 2001 From: tbussmann Date: Thu, 27 Oct 2016 18:21:10 +0200 Subject: [PATCH 2/4] changed from 1.3 to 1.2 --- META.json | 6 +++--- Makefile | 2 +- http--1.1--1.3.sql => http--1.1--1.2.sql | 0 http--1.3.sql => http--1.2.sql | 0 http.control | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename http--1.1--1.3.sql => http--1.1--1.2.sql (100%) rename http--1.3.sql => http--1.2.sql (100%) diff --git a/META.json b/META.json index a49d8e8..990d9a6 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "http", "abstract": "HTTP client for PostgreSQL", "description": "HTTP allows you to get the content of a web page in a SQL function call.", - "version": "1.3", + "version": "1.2", "maintainer": [ "Paul Ramsey " ], @@ -21,9 +21,9 @@ }, "provides": { "http": { - "file": "http--1.3.sql", + "file": "http--1.2.sql", "docfile": "README.md", - "version": "1.3", + "version": "1.2", "abstract": "HTTP client for PostgreSQL" }, }, diff --git a/Makefile b/Makefile index 8951470..bccf427 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ MODULE_big = http OBJS = http.o EXTENSION = http -DATA = http--1.3.sql http--1.1.sql http--1.0--1.1.sql +DATA = http--1.2.sql http--1.1--1.2.sql http--1.0--1.1.sql REGRESS = http EXTRA_CLEAN = diff --git a/http--1.1--1.3.sql b/http--1.1--1.2.sql similarity index 100% rename from http--1.1--1.3.sql rename to http--1.1--1.2.sql diff --git a/http--1.3.sql b/http--1.2.sql similarity index 100% rename from http--1.3.sql rename to http--1.2.sql diff --git a/http.control b/http.control index a452b0b..5270225 100644 --- a/http.control +++ b/http.control @@ -1,4 +1,4 @@ -default_version = '1.3' +default_version = '1.2' module_pathname = '$libdir/http' relocatable = true comment = 'HTTP client for PostgreSQL, allows web page retrieval inside the database.' From c154e5afb38ca18249c135aab637be7d36e70006 Mon Sep 17 00:00:00 2001 From: tbussmann Date: Thu, 27 Oct 2016 21:53:29 +0200 Subject: [PATCH 3/4] remove obsolte full install script for version 1.1 --- http--1.1--1.2.sql | 0 http--1.1.sql | 71 ---------------------------------------------- 2 files changed, 71 deletions(-) mode change 100755 => 100644 http--1.1--1.2.sql delete mode 100644 http--1.1.sql diff --git a/http--1.1--1.2.sql b/http--1.1--1.2.sql old mode 100755 new mode 100644 diff --git a/http--1.1.sql b/http--1.1.sql deleted file mode 100644 index 603d4fb..0000000 --- a/http--1.1.sql +++ /dev/null @@ -1,71 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION http" to load this file. \quit - -CREATE DOMAIN http_method AS text -CHECK ( - VALUE ILIKE 'get' OR - VALUE ILIKE 'post' OR - VALUE ILIKE 'put' OR - VALUE ILIKE 'delete' -); - -CREATE DOMAIN content_type AS text -CHECK ( - VALUE ~ '^\S+\/\S+' -); - -CREATE TYPE http_header AS ( - field VARCHAR, - value VARCHAR -); - -CREATE TYPE http_response AS ( - status INTEGER, - content_type VARCHAR, - headers http_header[], - content VARCHAR -); - -CREATE TYPE http_request AS ( - method http_method, - uri VARCHAR, - headers http_header[], - content_type VARCHAR, - content VARCHAR -); - -CREATE OR REPLACE FUNCTION http_header (field VARCHAR, value VARCHAR) - RETURNS http_header - AS $$ SELECT $1, $2 $$ - LANGUAGE 'sql'; - -CREATE OR REPLACE FUNCTION http(request http_request) - RETURNS http_response - AS 'MODULE_PATHNAME', 'http_request' - LANGUAGE 'c'; - -CREATE OR REPLACE FUNCTION http_get(uri VARCHAR) - RETURNS http_response - AS $$ SELECT http(('GET', $1, NULL, NULL, NULL)::http_request) $$ - LANGUAGE 'sql'; - -CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) - RETURNS http_response - AS $$ SELECT http(('POST', $1, NULL, $3, $2)::http_request) $$ - LANGUAGE 'sql'; - -CREATE OR REPLACE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) - RETURNS http_response - AS $$ SELECT http(('PUT', $1, NULL, $3, $2)::http_request) $$ - LANGUAGE 'sql'; - -CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR) - RETURNS http_response - AS $$ SELECT http(('DELETE', $1, NULL, NULL, NULL)::http_request) $$ - LANGUAGE 'sql'; - -CREATE OR REPLACE FUNCTION urlencode(string VARCHAR) - RETURNS TEXT - AS 'MODULE_PATHNAME' - LANGUAGE 'c' - IMMUTABLE STRICT; From fcc35a5edba58cd72b2e66b68478744dc56f05d6 Mon Sep 17 00:00:00 2001 From: tbussmann Date: Thu, 27 Oct 2016 22:08:57 +0200 Subject: [PATCH 4/4] use a different approach for http_head example in readme.md --- README.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b644eb8..2586ba2 100644 --- a/README.md +++ b/README.md @@ -112,21 +112,17 @@ To access binary content, you must coerce the content from the default `varchar` --------------+---------------+---------------- image/gif | 31958 | 31958 -To access only the header you can do a HEAD-Request. This will not follow redirections. +To access only the headers you can do a HEAD-Request. This will not follow redirections. - WITH - http AS ( - SELECT * FROM http_head('http://google.com') - ), - headers AS ( - SELECT (unnest(headers)).* FROM http - ) SELECT - http.status, - headers.value AS location - FROM http - LEFT OUTER JOIN headers - ON (headers.field = 'Location'); + http.status, + headers.value AS location + FROM + http_head('http://google.com') AS http + LEFT OUTER JOIN LATERAL (SELECT value + FROM unnest(http.headers) + WHERE field = 'Location') AS headers + ON true; status | location --------+-----------------------------------------------------------