79 #define CLIENT_ERR_MIN 400
80 #define CLIENT_ERR_MAX 417
84 "Unauthorized: Contact the server administrator.",
86 "Forbidden: Contact the server administrator.",
87 "Not Found: The data source or server could not be found.\n\
88 Often this means that the OPeNDAP server is missing or needs attention;\n\
89 Please contact the server administrator.",
90 "Method Not Allowed.",
92 "Proxy Authentication Required.",
97 "Precondition Failed.",
98 "Request Entity Too Large.",
99 "Request URI Too Large.",
100 "Unsupported Media Type.",
101 "Requested Range Not Satisfiable.",
102 "Expectation Failed."
105 #define SERVER_ERR_MIN 500
106 #define SERVER_ERR_MAX 505
109 "Internal Server Error.",
112 "Service Unavailable.",
114 "HTTP Version Not Supported."
120 http_status_to_string(
int status)
127 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
134 class ParseHeader :
public unary_function<const string &, void>
142 ParseHeader() : type(
unknown_type), server(
"dods/0.0"), protocol(
"2.0")
145 void operator()(
const string &line)
149 if (name ==
"content-description") {
150 DBG2(cerr << name <<
": " << value << endl);
156 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
157 DBG2(cerr << name <<
": " << value << endl);
160 else if (name ==
"xopendap-server") {
161 DBG2(cerr << name <<
": " << value << endl);
164 else if (name ==
"xdap") {
165 DBG2(cerr << name <<
": " << value << endl);
168 else if (server ==
"dods/0.0" && name ==
"server") {
169 DBG2(cerr << name <<
": " << value << endl);
172 else if (name ==
"location") {
173 DBG2(cerr << name <<
": " << value << endl);
177 && line.find(
"text/html") != string::npos) {
178 DBG2(cerr << name <<
": text/html..." << endl);
193 string get_protocol()
198 string get_location() {
220 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
222 DBG2(cerr <<
"Inside the header parser." << endl);
223 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
226 string complete_line;
227 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
228 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
230 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
233 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
234 DBG(cerr <<
"Header line: " << complete_line << endl);
235 hdrs->push_back(complete_line);
243 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
245 string message(msg, size);
249 cerr <<
"Text: " << message;
break;
250 case CURLINFO_HEADER_IN:
251 cerr <<
"Header in: " << message;
break;
252 case CURLINFO_HEADER_OUT:
253 cerr <<
"Header out: " << message;
break;
254 case CURLINFO_DATA_IN:
255 cerr <<
"Data in: " << message;
break;
256 case CURLINFO_DATA_OUT:
257 cerr <<
"Data out: " << message;
break;
259 cerr <<
"End: " << message;
break;
260 #ifdef CURLINFO_SSL_DATA_IN
261 case CURLINFO_SSL_DATA_IN:
262 cerr <<
"SSL Data in: " << message;
break;
264 #ifdef CURLINFO_SSL_DATA_OUT
265 case CURLINFO_SSL_DATA_OUT:
266 cerr <<
"SSL Data out: " << message;
break;
269 cerr <<
"Curl info: " << message;
break;
278 HTTPConnect::www_lib_init()
280 d_curl = curl_easy_init();
282 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
288 if (!d_rcr->get_proxy_server_host().empty()) {
289 DBG(cerr <<
"Setting up a proxy server." << endl);
290 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
292 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
294 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
296 curl_easy_setopt(d_curl, CURLOPT_PROXY,
297 d_rcr->get_proxy_server_host().c_str());
298 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
299 d_rcr->get_proxy_server_port());
302 #ifdef CURLOPT_PROXYAUTH
303 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
307 if (!d_rcr->get_proxy_server_userpw().empty())
308 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
309 d_rcr->get_proxy_server_userpw().c_str());
312 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
315 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
320 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
322 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
323 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
324 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
329 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
330 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
333 if (!d_rcr->get_validate_ssl() == 0) {
334 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
335 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
342 if (!d_cookie_jar.empty()) {
343 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
344 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
345 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
349 cerr <<
"Curl version: " << curl_version() << endl;
350 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
351 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
358 class BuildHeaders :
public unary_function<const string &, void>
360 struct curl_slist *d_cl;
363 BuildHeaders() : d_cl(0)
366 void operator()(
const string &header)
368 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list."
370 d_cl = curl_slist_append(d_cl, header.c_str());
373 struct curl_slist *get_headers()
394 HTTPConnect::read_url(
const string &url, FILE *stream,
395 vector<string> *resp_hdrs,
396 const vector<string> *headers)
398 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
408 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
409 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
411 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
414 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
415 ostream_iterator<string>(cerr,
"\n")));
417 BuildHeaders req_hdrs;
418 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
421 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
422 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
425 bool temporary_proxy =
false;
426 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
427 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
428 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
431 string::size_type at_sign = url.find(
'@');
435 if (at_sign != url.npos)
436 d_upstring = url.substr(7, at_sign - 7);
438 if (!d_upstring.empty())
439 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
444 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
449 CURLcode res = curl_easy_perform(d_curl);
452 curl_slist_free_all(req_hdrs.get_headers());
453 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
456 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
457 curl_easy_setopt(d_curl, CURLOPT_PROXY,
458 d_rcr->get_proxy_server_host().c_str());
461 throw Error(d_error_buffer);
464 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
466 throw Error(d_error_buffer);
475 HTTPConnect::url_uses_proxy_for(
const string &url)
throw()
477 if (d_rcr->is_proxy_for_used()) {
478 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
479 int index = 0, matchlen;
480 return host_regex.search(url.c_str(), url.size(), matchlen, index) != -1;
490 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
492 return d_rcr->is_no_proxy_for_used()
493 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
504 HTTPConnect::HTTPConnect(
RCReader *rcr) : d_username(
""), d_password(
""),
506 d_dap_client_protocol_major(2),
507 d_dap_client_protocol_minor(0)
517 d_request_headers.push_back(
string(
"Pragma:"));
518 string user_agent = string(
"User-Agent: ") + string(
CNAME)
519 + string(
"/") + string(
CVER);
520 d_request_headers.push_back(user_agent);
521 if (d_accept_deflate)
522 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
530 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
549 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
551 curl_easy_cleanup(d_curl);
553 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
572 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
578 stream = caching_fetch_url(url);
581 stream = plain_fetch_url(url);
586 ss <<
"HTTP/1.0 " << stream->
get_status() <<
" -" << endl;
587 for (
size_t i = 0; i < stream->
get_headers()->size(); i++) {
599 cout << endl << endl;
603 if (parser.get_location() !=
"" &&
604 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
609 stream->
set_type(parser.get_object_type());
627 get_tempfile_template(
const string &file_template)
634 Regex directory(
"[-a-zA-Z0-9_:\\]*");
639 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
640 goto valid_temp_directory;
643 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
644 goto valid_temp_directory;
649 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
650 goto valid_temp_directory;
652 #else // Unix/Linux/OSX has another...
654 Regex directory(
"[-a-zA-Z0-9_/]*");
656 c = getenv(
"TMPDIR");
657 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
658 goto valid_temp_directory;
663 if (access(P_tmpdir, W_OK | R_OK) == 0) {
665 goto valid_temp_directory;
671 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
672 goto valid_temp_directory;
679 valid_temp_directory:
682 c +=
"\\" + file_template;
684 c +=
"/" + file_template;
711 string dods_temp = get_tempfile_template((
string)
"dodsXXXXXX");
713 vector<char> pathname(dods_temp.length() + 1);
715 strncpy(&pathname[0], dods_temp.c_str(), dods_temp.length());
717 DBG(cerr <<
"pathanme: " << &pathname[0] <<
" (" << dods_temp.length() + 1 <<
")" << endl);
720 #if defined(WIN32) || defined(TEST_WIN32_TEMPS)
721 stream = fopen(_mktemp(&pathname[0]),
"w+b");
725 stream = fdopen(mkstemp(&pathname[0]),
"w+");
730 "Failed to open a temporary file for the data values ("
734 dods_temp = &pathname[0];
746 res = unlink(name.c_str());
773 HTTPConnect::caching_fetch_url(
const string &url)
775 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
777 vector<string> *headers =
new vector<string>;
782 DBGN(cerr <<
"no; getting response and caching." << endl);
783 delete headers; headers = 0;
784 time_t now = time(0);
785 HTTPResponse *rs = plain_fetch_url(url);
786 d_http_cache->
cache_response(url, now, *(rs->get_headers()), rs->get_stream());
791 DBGN(cerr <<
"yes... ");
794 DBGN(cerr <<
"and it's valid; using cached response." << endl);
795 HTTPCacheResponse *crs =
new HTTPCacheResponse(s, 200, headers, file_name, d_http_cache);
799 DBGN(cerr <<
"but it's not valid; validating... ");
806 time_t now = time(0);
810 http_status = read_url(url, body, headers, &cond_hdrs);
819 switch (http_status) {
821 DBGN(cerr <<
"read a new response; caching." << endl);
824 HTTPResponse *rs =
new HTTPResponse(body, http_status, headers, dods_temp);
830 DBGN(cerr <<
"cached response valid; updating." << endl);
836 HTTPCacheResponse *crs =
new HTTPCacheResponse(hs, 304, headers, file_name, d_http_cache);
842 if (http_status >= 400) {
843 delete headers; headers = 0;
844 string msg =
"Error while reading the URL: ";
847 +=
".\nThe OPeNDAP server returned the following message:\n";
848 msg += http_status_to_string(http_status);
852 delete headers; headers = 0;
853 throw InternalErr(__FILE__, __LINE__,
854 "Bad response from the HTTP server: " +
long_to_string(http_status));
861 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
876 HTTPConnect::plain_fetch_url(
const string &url)
878 DBG(cerr <<
"Getting URL: " << url << endl);
881 vector<string> *resp_hdrs =
new vector<string>;
885 status = read_url(url, stream, resp_hdrs);
888 string msg =
"Error while reading the URL: ";
890 msg +=
".\nThe OPeNDAP server returned the following message:\n";
891 msg += http_status_to_string(status);
904 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
923 if (d_accept_deflate) {
924 if (find(d_request_headers.begin(), d_request_headers.end(),
925 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
926 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
927 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
928 ostream_iterator<string>(cerr,
"\n")));
931 vector<string>::iterator i;
932 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
933 bind2nd(equal_to<string>(),
934 string(
"Accept-Encoding: deflate, gzip, compress")));
935 d_request_headers.erase(i, d_request_headers.end());
940 class HeaderMatch :
public unary_function<const string &, bool> {
941 const string &d_header;
943 HeaderMatch(
const string &header) : d_header(header) {}
944 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
959 vector<string>::iterator i;
960 i = find_if(d_request_headers.begin(), d_request_headers.end(),
961 HeaderMatch(
"XDAP-Accept:"));
962 if (i != d_request_headers.end())
963 d_request_headers.erase(i);
966 d_dap_client_protocol_major = major;
967 d_dap_client_protocol_minor = minor;
968 ostringstream xdap_accept;
969 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
971 d_request_headers.push_back(xdap_accept.str());
973 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
974 ostream_iterator<string>(cerr,
"\n")));
1002 d_upstring = u +
":" + p;