1 // Copyright 2013-2020, Adam D. Ruppe. 2 3 // FIXME: eaders are supposed to be case insensitive. ugh. 4 5 // FIXME: need timeout controls 6 // FIXME: 100 continue. tho we never Expect it so should never happen, never kno, 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to be support HTTPS. Compile with 14 `-version=with_openssl` to enable such support. 15 16 http2.d, despite its name, does NOT implement HTTP/2.0, but this 17 shouldn't matter for 99.9% of usage, since all servers will continue 18 to support HTTP/1.1 for a very long time. 19 20 +/ 21 module arsd.http2; 22 23 // FIXME: I think I want to disable sigpipe here too. 24 25 import std.uri : encodeComponent; 26 27 debug(arsd_http2_verbose) debug=arsd_http2; 28 29 debug(arsd_http2) import std.stdio : writeln; 30 31 version=arsd_http_internal_implementation; 32 33 version(without_openssl) {} 34 else { 35 version=use_openssl; 36 version=with_openssl; 37 version(older_openssl) {} else 38 version=newer_openssl; 39 } 40 41 version(arsd_http_winhttp_implementation) { 42 pragma(lib, "winhttp") 43 import core.sys.windows.winhttp; 44 // FIXME: alter the dub package file too 45 46 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 47 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 48 49 50 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 51 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 52 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 53 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 54 } 55 56 57 58 /++ 59 Demonstrates core functionality, using the [HttpClient], 60 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 61 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 62 63 +/ 64 unittest { 65 import arsd.http2; 66 67 void main() { 68 auto client = new HttpClient(); 69 auto request = client.navigateTo(Uri("http://dlang.org/")); 70 auto response = request.waitForCompletion(); 71 72 string returnedHtml = response.contentText; 73 } 74 } 75 76 // FIXME: multipart encoded file uploads needs implementation 77 // future: do web client api stuff 78 79 debug import std.stdio; 80 81 import std.socket; 82 import core.time; 83 84 // FIXME: check Transfer-Encoding: gzip always 85 86 version(with_openssl) { 87 //pragma(lib, "crypto"); 88 //pragma(lib, "ssl"); 89 } 90 91 /+ 92 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 93 return null; 94 } 95 +/ 96 97 /** 98 auto request = get("http://arsdnet.net/"); 99 request.send(); 100 101 auto response = get("http://arsdnet.net/").waitForCompletion(); 102 */ 103 HttpRequest get(string url) { 104 auto client = new HttpClient(); 105 auto request = client.navigateTo(Uri(url)); 106 return request; 107 } 108 109 /** 110 Do not forget to call `waitForCompletion()` on the returned object! 111 */ 112 HttpRequest post(string url, string[string] req) { 113 auto client = new HttpClient(); 114 ubyte[] bdata; 115 foreach(k, v; req) { 116 if(bdata.length) 117 bdata ~= cast(ubyte[]) "&"; 118 bdata ~= cast(ubyte[]) encodeComponent(k); 119 bdata ~= cast(ubyte[]) "="; 120 bdata ~= cast(ubyte[]) encodeComponent(v); 121 } 122 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 123 return request; 124 } 125 126 /// gets the text off a url. basic operation only. 127 string getText(string url) { 128 auto request = get(url); 129 auto response = request.waitForCompletion(); 130 return cast(string) response.content; 131 } 132 133 /+ 134 ubyte[] getBinary(string url, string[string] cookies = null) { 135 auto hr = httpRequest("GET", url, null, cookies); 136 if(hr.code != 200) 137 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 138 return hr.content; 139 } 140 141 /** 142 Gets a textual document, ignoring headers. Throws on non-text or error. 143 */ 144 string get(string url, string[string] cookies = null) { 145 auto hr = httpRequest("GET", url, null, cookies); 146 if(hr.code != 200) 147 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 148 if(hr.contentType.indexOf("text/") == -1) 149 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 150 return cast(string) hr.content; 151 152 } 153 154 static import std.uri; 155 156 string post(string url, string[string] args, string[string] cookies = null) { 157 string content; 158 159 foreach(name, arg; args) { 160 if(content.length) 161 content ~= "&"; 162 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 163 } 164 165 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 166 if(hr.code != 200) 167 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 168 if(hr.contentType.indexOf("text/") == -1) 169 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 170 171 return cast(string) hr.content; 172 } 173 174 +/ 175 176 /// 177 struct HttpResponse { 178 /++ 179 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 180 181 Code 0 - initial value, nothing happened 182 Code 1 - you called request.abort 183 Code 2 - connection refused 184 Code 3 - connection succeeded, but server disconnected early 185 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 186 Code 5 - request timed out 187 188 Code >= 100 - a HTTP response 189 +/ 190 int code; 191 string codeText; /// 192 193 string httpVersion; /// 194 195 string statusLine; /// 196 197 string contentType; /// The content type header 198 string location; /// The location header 199 200 /++ 201 202 History: 203 Added December 5, 2020 (version 9.1) 204 +/ 205 bool wasSuccessful() { 206 return code >= 200 && code < 400; 207 } 208 209 /// the charset out of content type, if present. `null` if not. 210 string contentTypeCharset() { 211 auto idx = contentType.indexOf("charset="); 212 if(idx == -1) 213 return null; 214 auto c = contentType[idx + "charset=".length .. $].strip; 215 if(c.length) 216 return c; 217 return null; 218 } 219 220 string[string] cookies; /// Names and values of cookies set in the response. 221 222 string[] headers; /// Array of all headers returned. 223 string[string] headersHash; /// 224 225 ubyte[] content; /// The raw content returned in the response body. 226 string contentText; /// [content], but casted to string (for convenience) 227 228 alias responseText = contentText; // just cuz I do this so often. 229 //alias body = content; 230 231 /++ 232 returns `new Document(this.contentText)`. Requires [arsd.dom]. 233 +/ 234 auto contentDom()() { 235 import arsd.dom; 236 return new Document(this.contentText); 237 238 } 239 240 /++ 241 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 242 +/ 243 auto contentJson()() { 244 import arsd.jsvar; 245 return var.fromJson(this.contentText); 246 } 247 248 HttpRequestParameters requestParameters; /// 249 250 LinkHeader[] linksStored; 251 bool linksLazilyParsed; 252 253 HttpResponse deepCopy() const { 254 HttpResponse h = cast(HttpResponse) this; 255 h.cookies = h.cookies.dup; 256 h.headers = h.headers.dup; 257 h.headersHash = h.headersHash.dup; 258 h.content = h.content.dup; 259 h.linksStored = h.linksStored.dup; 260 return h; 261 } 262 263 /// Returns links header sorted by "rel" attribute. 264 /// It returns a new array on each call. 265 LinkHeader[string] linksHash() { 266 auto links = this.links(); 267 LinkHeader[string] ret; 268 foreach(link; links) 269 ret[link.rel] = link; 270 return ret; 271 } 272 273 /// Returns the Link header, parsed. 274 LinkHeader[] links() { 275 if(linksLazilyParsed) 276 return linksStored; 277 linksLazilyParsed = true; 278 LinkHeader[] ret; 279 280 auto hdrPtr = "Link" in headersHash; 281 if(hdrPtr is null) 282 return ret; 283 284 auto header = *hdrPtr; 285 286 LinkHeader current; 287 288 while(header.length) { 289 char ch = header[0]; 290 291 if(ch == '<') { 292 // read url 293 header = header[1 .. $]; 294 size_t idx; 295 while(idx < header.length && header[idx] != '>') 296 idx++; 297 current.url = header[0 .. idx]; 298 header = header[idx .. $]; 299 } else if(ch == ';') { 300 // read attribute 301 header = header[1 .. $]; 302 header = header.stripLeft; 303 304 size_t idx; 305 while(idx < header.length && header[idx] != '=') 306 idx++; 307 308 string name = header[0 .. idx]; 309 header = header[idx + 1 .. $]; 310 311 string value; 312 313 if(header.length && header[0] == '"') { 314 // quoted value 315 header = header[1 .. $]; 316 idx = 0; 317 while(idx < header.length && header[idx] != '\"') 318 idx++; 319 value = header[0 .. idx]; 320 header = header[idx .. $]; 321 322 } else if(header.length) { 323 // unquoted value 324 idx = 0; 325 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 326 idx++; 327 328 value = header[0 .. idx]; 329 header = header[idx .. $].stripLeft; 330 } 331 332 name = name.toLower; 333 if(name == "rel") 334 current.rel = value; 335 else 336 current.attributes[name] = value; 337 338 } else if(ch == ',') { 339 // start another 340 ret ~= current; 341 current = LinkHeader.init; 342 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 343 // ignore 344 } 345 346 if(header.length) 347 header = header[1 .. $]; 348 } 349 350 ret ~= current; 351 352 linksStored = ret; 353 354 return ret; 355 } 356 } 357 358 /// 359 struct LinkHeader { 360 string url; /// 361 string rel; /// 362 string[string] attributes; /// like title, rev, media, whatever attributes 363 } 364 365 import std..string; 366 static import std.algorithm; 367 import std.conv; 368 import std.range; 369 370 371 private AddressFamily family(string unixSocketPath) { 372 if(unixSocketPath.length) 373 return AddressFamily.UNIX; 374 else // FIXME: what about ipv6? 375 return AddressFamily.INET; 376 } 377 378 version(Windows) 379 private class UnixAddress : Address { 380 this(string) { 381 throw new Exception("No unix address support on this system in lib yet :("); 382 } 383 override sockaddr* name() { assert(0); } 384 override const(sockaddr)* name() const { assert(0); } 385 override int nameLen() const { assert(0); } 386 } 387 388 389 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 390 /// 391 struct Uri { 392 alias toString this; // blargh idk a url really is a string, but should it be implicit? 393 394 // scheme//userinfo@host:port/path?query#fragment 395 396 string scheme; /// e.g. "http" in "http://example.com/" 397 string userinfo; /// the username (and possibly a password) in the uri 398 string host; /// the domain name 399 int port; /// port number, if given. Will be zero if a port was not explicitly given 400 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 401 string query; /// the stuff after the ? in a uri 402 string fragment; /// the stuff after the # in a uri. 403 404 /// Breaks down a uri string to its components 405 this(string uri) { 406 reparse(uri); 407 } 408 409 private string unixSocketPath = null; 410 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 411 Uri viaUnixSocket(string path) const { 412 Uri copy = this; 413 copy.unixSocketPath = path; 414 return copy; 415 } 416 417 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 418 version(linux) 419 Uri viaAbstractSocket(string path) const { 420 Uri copy = this; 421 copy.unixSocketPath = "\0" ~ path; 422 return copy; 423 } 424 425 private void reparse(string uri) { 426 // from RFC 3986 427 // the ctRegex triples the compile time and makes ugly errors for no real benefit 428 // it was a nice experiment but just not worth it. 429 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 430 /* 431 Captures: 432 0 = whole url 433 1 = scheme, with : 434 2 = scheme, no : 435 3 = authority, with // 436 4 = authority, no // 437 5 = path 438 6 = query string, with ? 439 7 = query string, no ? 440 8 = anchor, with # 441 9 = anchor, no # 442 */ 443 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 444 // instead, I will DIY and cut that down to 0.6s on the same computer. 445 /* 446 447 Note that authority is 448 user:password@domain:port 449 where the user:password@ part is optional, and the :port is optional. 450 451 Regex translation: 452 453 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 454 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 455 Path cannot have any ? or # in it. It is optional. 456 Query must start with ? and must not have # in it. It is optional. 457 Anchor must start with # and can have anything else in it to end of string. It is optional. 458 */ 459 460 this = Uri.init; // reset all state 461 462 // empty uri = nothing special 463 if(uri.length == 0) { 464 return; 465 } 466 467 size_t idx; 468 469 scheme_loop: foreach(char c; uri[idx .. $]) { 470 switch(c) { 471 case ':': 472 case '/': 473 case '?': 474 case '#': 475 break scheme_loop; 476 default: 477 } 478 idx++; 479 } 480 481 if(idx == 0 && uri[idx] == ':') { 482 // this is actually a path! we skip way ahead 483 goto path_loop; 484 } 485 486 if(idx == uri.length) { 487 // the whole thing is a path, apparently 488 path = uri; 489 return; 490 } 491 492 if(idx > 0 && uri[idx] == ':') { 493 scheme = uri[0 .. idx]; 494 idx++; 495 } else { 496 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 497 idx = 0; 498 } 499 500 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 501 // we have an authority.... 502 idx += 2; 503 504 auto authority_start = idx; 505 authority_loop: foreach(char c; uri[idx .. $]) { 506 switch(c) { 507 case '/': 508 case '?': 509 case '#': 510 break authority_loop; 511 default: 512 } 513 idx++; 514 } 515 516 auto authority = uri[authority_start .. idx]; 517 518 auto idx2 = authority.indexOf("@"); 519 if(idx2 != -1) { 520 userinfo = authority[0 .. idx2]; 521 authority = authority[idx2 + 1 .. $]; 522 } 523 524 if(authority.length && authority[0] == '[') { 525 // ipv6 address special casing 526 idx2 = authority.indexOf(']'); 527 if(idx2 != -1) { 528 auto end = authority[idx2 + 1 .. $]; 529 if(end.length && end[0] == ':') 530 idx2 = idx2 + 1; 531 else 532 idx2 = -1; 533 } 534 } else { 535 idx2 = authority.indexOf(":"); 536 } 537 538 if(idx2 == -1) { 539 port = 0; // 0 means not specified; we should use the default for the scheme 540 host = authority; 541 } else { 542 host = authority[0 .. idx2]; 543 port = to!int(authority[idx2 + 1 .. $]); 544 } 545 } 546 547 path_loop: 548 auto path_start = idx; 549 550 foreach(char c; uri[idx .. $]) { 551 if(c == '?' || c == '#') 552 break; 553 idx++; 554 } 555 556 path = uri[path_start .. idx]; 557 558 if(idx == uri.length) 559 return; // nothing more to examine... 560 561 if(uri[idx] == '?') { 562 idx++; 563 auto query_start = idx; 564 foreach(char c; uri[idx .. $]) { 565 if(c == '#') 566 break; 567 idx++; 568 } 569 query = uri[query_start .. idx]; 570 } 571 572 if(idx < uri.length && uri[idx] == '#') { 573 idx++; 574 fragment = uri[idx .. $]; 575 } 576 577 // uriInvalidated = false; 578 } 579 580 private string rebuildUri() const { 581 string ret; 582 if(scheme.length) 583 ret ~= scheme ~ ":"; 584 if(userinfo.length || host.length) 585 ret ~= "//"; 586 if(userinfo.length) 587 ret ~= userinfo ~ "@"; 588 if(host.length) 589 ret ~= host; 590 if(port) 591 ret ~= ":" ~ to!string(port); 592 593 ret ~= path; 594 595 if(query.length) 596 ret ~= "?" ~ query; 597 598 if(fragment.length) 599 ret ~= "#" ~ fragment; 600 601 // uri = ret; 602 // uriInvalidated = false; 603 return ret; 604 } 605 606 /// Converts the broken down parts back into a complete string 607 string toString() const { 608 // if(uriInvalidated) 609 return rebuildUri(); 610 } 611 612 /// Returns a new absolute Uri given a base. It treats this one as 613 /// relative where possible, but absolute if not. (If protocol, domain, or 614 /// other info is not set, the new one inherits it from the base.) 615 /// 616 /// Browsers use a function like this to figure out links in html. 617 Uri basedOn(in Uri baseUrl) const { 618 Uri n = this; // copies 619 // n.uriInvalidated = true; // make sure we regenerate... 620 621 // userinfo is not inherited... is this wrong? 622 623 // if anything is given in the existing url, we don't use the base anymore. 624 if(n.scheme.empty) { 625 n.scheme = baseUrl.scheme; 626 if(n.host.empty) { 627 n.host = baseUrl.host; 628 if(n.port == 0) { 629 n.port = baseUrl.port; 630 if(n.path.length > 0 && n.path[0] != '/') { 631 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 632 if(b.length == 0) 633 b = "/"; 634 n.path = b ~ n.path; 635 } else if(n.path.length == 0) { 636 n.path = baseUrl.path; 637 } 638 } 639 } 640 } 641 642 n.removeDots(); 643 644 // if still basically talking to the same thing, we should inherit the unix path 645 // too since basically the unix path is saying for this service, always use this override. 646 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 647 n.unixSocketPath = baseUrl.unixSocketPath; 648 649 return n; 650 } 651 652 void removeDots() { 653 auto parts = this.path.split("/"); 654 string[] toKeep; 655 foreach(part; parts) { 656 if(part == ".") { 657 continue; 658 } else if(part == "..") { 659 toKeep = toKeep[0 .. $-1]; 660 continue; 661 } else { 662 toKeep ~= part; 663 } 664 } 665 666 this.path = toKeep.join("/"); 667 } 668 669 } 670 671 /* 672 void main(string args[]) { 673 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 674 } 675 */ 676 677 /// 678 struct BasicAuth { 679 string username; /// 680 string password; /// 681 } 682 683 /** 684 Represents a HTTP request. You usually create these through a [HttpClient]. 685 686 687 --- 688 auto request = new HttpRequest(); 689 // set any properties here 690 691 // synchronous usage 692 auto reply = request.perform(); 693 694 // async usage, type 1: 695 request.send(); 696 request2.send(); 697 698 // wait until the first one is done, with the second one still in-flight 699 auto response = request.waitForCompletion(); 700 701 // async usage, type 2: 702 request.onDataReceived = (HttpRequest hr) { 703 if(hr.state == HttpRequest.State.complete) { 704 // use hr.responseData 705 } 706 }; 707 request.send(); // send, using the callback 708 709 // before terminating, be sure you wait for your requests to finish! 710 711 request.waitForCompletion(); 712 --- 713 */ 714 class HttpRequest { 715 716 /// Automatically follow a redirection? 717 bool followLocation = false; 718 719 this() { 720 } 721 722 /// 723 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds) { 724 populateFromInfo(where, method); 725 setTimeout(timeout); 726 this.cache = cache; 727 } 728 729 /++ 730 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 731 732 History: 733 Added March 31, 2021 734 +/ 735 void setTimeout(Duration timeout) { 736 this.requestParameters.timeoutFromInactivity = timeout; 737 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 738 } 739 740 private MonoTime timeoutFromInactivity; 741 742 private Uri where; 743 744 private ICache cache; 745 746 /// Final url after any redirections 747 string finalUrl; 748 749 void populateFromInfo(Uri where, HttpVerb method) { 750 auto parts = where.basedOn(this.where); 751 this.where = parts; 752 finalUrl = where.toString(); 753 requestParameters.method = method; 754 requestParameters.unixSocketPath = where.unixSocketPath; 755 requestParameters.host = parts.host; 756 requestParameters.port = cast(ushort) parts.port; 757 requestParameters.ssl = parts.scheme == "https"; 758 if(parts.port == 0) 759 requestParameters.port = requestParameters.ssl ? 443 : 80; 760 requestParameters.uri = parts.path.length ? parts.path : "/"; 761 if(parts.query.length) { 762 requestParameters.uri ~= "?"; 763 requestParameters.uri ~= parts.query; 764 } 765 } 766 767 ~this() { 768 } 769 770 ubyte[] sendBuffer; 771 772 HttpResponse responseData; 773 private HttpClient parentClient; 774 775 size_t bodyBytesSent; 776 size_t bodyBytesReceived; 777 778 State state_; 779 State state() { return state_; } 780 State state(State s) { 781 assert(state_ != State.complete); 782 return state_ = s; 783 } 784 /// Called when data is received. Check the state to see what data is available. 785 void delegate(HttpRequest) onDataReceived; 786 787 enum State { 788 /// The request has not yet been sent 789 unsent, 790 791 /// The send() method has been called, but no data is 792 /// sent on the socket yet because the connection is busy. 793 pendingAvailableConnection, 794 795 /// The headers are being sent now 796 sendingHeaders, 797 798 /// The body is being sent now 799 sendingBody, 800 801 /// The request has been sent but we haven't received any response yet 802 waitingForResponse, 803 804 /// We have received some data and are currently receiving headers 805 readingHeaders, 806 807 /// All headers are available but we're still waiting on the body 808 readingBody, 809 810 /// The request is complete. 811 complete, 812 813 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 814 aborted 815 } 816 817 /// Sends now and waits for the request to finish, returning the response. 818 HttpResponse perform() { 819 send(); 820 return waitForCompletion(); 821 } 822 823 /// Sends the request asynchronously. 824 void send() { 825 sendPrivate(true); 826 } 827 828 private void sendPrivate(bool advance) { 829 if(state != State.unsent && state != State.aborted) 830 return; // already sent 831 832 if(cache !is null) { 833 auto res = cache.getCachedResponse(this.requestParameters); 834 if(res !is null) { 835 state = State.complete; 836 responseData = (*res).deepCopy(); 837 return; 838 } 839 } 840 841 string headers; 842 843 headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; 844 if(requestParameters.useHttp11) 845 headers ~= " HTTP/1.1\r\n"; 846 else 847 headers ~= " HTTP/1.0\r\n"; 848 849 // the whole authority section is supposed to be there, but curl doesn't send if default port 850 // so I'll copy what they do 851 headers ~= "Host: "; 852 headers ~= requestParameters.host; 853 if(requestParameters.port != 80 && requestParameters.port != 443) { 854 headers ~= ":"; 855 headers ~= to!string(requestParameters.port); 856 } 857 headers ~= "\r\n"; 858 859 if(requestParameters.userAgent.length) 860 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 861 if(requestParameters.contentType.length) 862 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 863 if(requestParameters.authorization.length) 864 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 865 if(requestParameters.bodyData.length) 866 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 867 if(requestParameters.acceptGzip) 868 headers ~= "Accept-Encoding: gzip\r\n"; 869 if(requestParameters.keepAlive) 870 headers ~= "Connection: keep-alive\r\n"; 871 872 foreach(header; requestParameters.headers) 873 headers ~= header ~ "\r\n"; 874 875 headers ~= "\r\n"; 876 877 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 878 879 // import std.stdio; writeln("******* ", sendBuffer); 880 881 responseData = HttpResponse.init; 882 responseData.requestParameters = requestParameters; 883 bodyBytesSent = 0; 884 bodyBytesReceived = 0; 885 state = State.pendingAvailableConnection; 886 887 bool alreadyPending = false; 888 foreach(req; pending) 889 if(req is this) { 890 alreadyPending = true; 891 break; 892 } 893 if(!alreadyPending) { 894 pending ~= this; 895 } 896 897 if(advance) 898 HttpRequest.advanceConnections(); 899 } 900 901 902 /// Waits for the request to finish or timeout, whichever comes first. 903 HttpResponse waitForCompletion() { 904 while(state != State.aborted && state != State.complete) { 905 if(state == State.unsent) { 906 send(); 907 continue; 908 } 909 if(auto err = HttpRequest.advanceConnections()) { 910 switch(err) { 911 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 912 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 913 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 914 } 915 } 916 } 917 918 if(state == State.complete && responseData.code >= 200) 919 if(cache !is null) 920 cache.cacheResponse(this.requestParameters, this.responseData); 921 922 return responseData; 923 } 924 925 /// Aborts this request. 926 void abort() { 927 this.state = State.aborted; 928 this.responseData.code = 1; 929 this.responseData.codeText = "request.abort called"; 930 // FIXME actually cancel it? 931 } 932 933 HttpRequestParameters requestParameters; /// 934 935 version(arsd_http_winhttp_implementation) { 936 public static void resetInternals() { 937 938 } 939 940 static assert(0, "implementation not finished"); 941 } 942 943 944 version(arsd_http_internal_implementation) { 945 private static { 946 // we manage the actual connections. When a request is made on a particular 947 // host, we try to reuse connections. We may open more than one connection per 948 // host to do parallel requests. 949 // 950 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 951 Socket[][string] socketsPerHost; 952 953 void loseSocket(string host, ushort port, bool ssl, Socket s) { 954 import std..string; 955 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 956 957 if(auto list = key in socketsPerHost) { 958 for(int a = 0; a < (*list).length; a++) { 959 if((*list)[a] is s) { 960 961 for(int b = a; b < (*list).length - 1; b++) 962 (*list)[b] = (*list)[b+1]; 963 (*list) = (*list)[0 .. $-1]; 964 break; 965 } 966 } 967 } 968 } 969 970 Socket getOpenSocketOnHost(string host, ushort port, bool ssl, string unixSocketPath) { 971 972 Socket openNewConnection() { 973 Socket socket; 974 if(ssl) { 975 version(with_openssl) { 976 loadOpenSsl(); 977 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host); 978 } else 979 throw new Exception("SSL not compiled in"); 980 } else 981 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 982 983 if(unixSocketPath) { 984 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 985 socket.connect(new UnixAddress(unixSocketPath)); 986 } else { 987 // FIXME: i should prolly do ipv6 if available too. 988 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 989 throw new Exception("No host given for request"); 990 socket.connect(new InternetAddress(host, port)); 991 } 992 993 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket); 994 assert(socket.handle() !is socket_t.init); 995 return socket; 996 } 997 998 import std..string; 999 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1000 1001 if(auto hostListing = key in socketsPerHost) { 1002 // try to find an available socket that is already open 1003 foreach(socket; *hostListing) { 1004 if(socket !in activeRequestOnSocket) { 1005 // let's see if it has closed since we last tried 1006 // e.g. a server timeout or something. If so, we need 1007 // to lose this one and immediately open a new one. 1008 static SocketSet readSet = null; 1009 if(readSet is null) 1010 readSet = new SocketSet(); 1011 readSet.reset(); 1012 assert(socket !is null); 1013 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1014 readSet.add(socket); 1015 auto got = Socket.select(readSet, null, null, 5.msecs /* timeout */); 1016 if(got > 0) { 1017 // we can read something off this... but there aren't 1018 // any active requests. Assume it is EOF and open a new one 1019 1020 socket.close(); 1021 loseSocket(host, port, ssl, socket); 1022 goto openNew; 1023 } 1024 return socket; 1025 } 1026 } 1027 1028 // if not too many already open, go ahead and do a new one 1029 if((*hostListing).length < 6) { 1030 auto socket = openNewConnection(); 1031 (*hostListing) ~= socket; 1032 return socket; 1033 } else 1034 return null; // too many, you'll have to wait 1035 } 1036 1037 openNew: 1038 1039 auto socket = openNewConnection(); 1040 socketsPerHost[key] ~= socket; 1041 return socket; 1042 } 1043 1044 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1045 HttpRequest[Socket] activeRequestOnSocket; 1046 HttpRequest[] pending; // and these are the requests that are waiting 1047 1048 SocketSet readSet; 1049 SocketSet writeSet; 1050 1051 1052 int advanceConnections() { 1053 if(readSet is null) 1054 readSet = new SocketSet(); 1055 if(writeSet is null) 1056 writeSet = new SocketSet(); 1057 1058 ubyte[2048] buffer; 1059 1060 HttpRequest[16] removeFromPending; 1061 size_t removeFromPendingCount = 0; 1062 1063 bool hadAbortedRequest; 1064 1065 // are there pending requests? let's try to send them 1066 foreach(idx, pc; pending) { 1067 if(removeFromPendingCount == removeFromPending.length) 1068 break; 1069 1070 if(pc.state == HttpRequest.State.aborted) { 1071 removeFromPending[removeFromPendingCount++] = pc; 1072 hadAbortedRequest = true; 1073 continue; 1074 } 1075 1076 Socket socket; 1077 1078 try { 1079 socket = getOpenSocketOnHost(pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath); 1080 } catch(SocketException e) { 1081 // connection refused or timed out (I should disambiguate somehow)... 1082 pc.state = HttpRequest.State.aborted; 1083 1084 pc.responseData.code = 2; 1085 pc.responseData.codeText = "connection failed"; 1086 1087 hadAbortedRequest = true; 1088 1089 removeFromPending[removeFromPendingCount++] = pc; 1090 continue; 1091 } 1092 1093 if(socket !is null) { 1094 activeRequestOnSocket[socket] = pc; 1095 assert(pc.sendBuffer.length); 1096 pc.state = State.sendingHeaders; 1097 1098 removeFromPending[removeFromPendingCount++] = pc; 1099 } 1100 } 1101 1102 import std.algorithm : remove; 1103 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1104 pending = pending.remove!((a) => a is rp)(); 1105 1106 readSet.reset(); 1107 writeSet.reset(); 1108 1109 bool hadOne = false; 1110 1111 Duration minTimeout = 10.seconds; 1112 auto now = MonoTime.currTime; 1113 1114 // active requests need to be read or written to 1115 foreach(sock, request; activeRequestOnSocket) { 1116 // check the other sockets just for EOF, if they close, take them out of our list, 1117 // we'll reopen if needed upon request. 1118 readSet.add(sock); 1119 hadOne = true; 1120 1121 Duration timeo; 1122 if(request.timeoutFromInactivity <= now) 1123 timeo = 0.seconds; 1124 else 1125 timeo = request.timeoutFromInactivity - now; 1126 1127 if(timeo < minTimeout) 1128 minTimeout = timeo; 1129 1130 if(request.state == State.sendingHeaders || request.state == State.sendingBody) { 1131 writeSet.add(sock); 1132 hadOne = true; 1133 } 1134 } 1135 1136 if(!hadOne) { 1137 if(hadAbortedRequest) 1138 return 0; // something got aborted, that's progress 1139 return 2; // automatic timeout, nothing to do 1140 } 1141 1142 tryAgain: 1143 1144 1145 Socket[16] inactive; 1146 int inactiveCount = 0; 1147 void killInactives() { 1148 foreach(s; inactive[0 .. inactiveCount]) { 1149 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1150 activeRequestOnSocket.remove(s); 1151 } 1152 } 1153 1154 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1155 if(selectGot == 0) { /* timeout */ 1156 now = MonoTime.currTime; 1157 foreach(sock, request; activeRequestOnSocket) { 1158 1159 if(request.timeoutFromInactivity <= now) { 1160 request.state = HttpRequest.State.aborted; 1161 request.responseData.code = 5; 1162 request.responseData.codeText = "Request timed out"; 1163 1164 inactive[inactiveCount++] = sock; 1165 sock.close(); 1166 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1167 } 1168 } 1169 killInactives(); 1170 return 0; 1171 // return 1; was an error to time out but now im making it on the individual request 1172 } else if(selectGot == -1) { /* interrupted */ 1173 /* 1174 version(Posix) { 1175 import core.stdc.errno; 1176 if(errno != EINTR) 1177 throw new Exception("select error: " ~ to!string(errno)); 1178 } 1179 */ 1180 goto tryAgain; 1181 } else { /* ready */ 1182 foreach(sock, request; activeRequestOnSocket) { 1183 if(readSet.isSet(sock)) { 1184 keep_going: 1185 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1186 auto got = sock.receive(buffer); 1187 debug(arsd_http2_verbose) writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); 1188 if(got < 0) { 1189 throw new Exception("receive error"); 1190 } else if(got == 0) { 1191 // remote side disconnected 1192 debug(arsd_http2) writeln("remote disconnect"); 1193 if(request.state != State.complete) { 1194 request.state = State.aborted; 1195 1196 request.responseData.code = 3; 1197 request.responseData.codeText = "server disconnected"; 1198 } 1199 inactive[inactiveCount++] = sock; 1200 sock.close(); 1201 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1202 } else { 1203 // data available 1204 bool stillAlive; 1205 1206 try { 1207 stillAlive = request.handleIncomingData(buffer[0 .. got]); 1208 } catch (Exception e) { 1209 request.state = HttpRequest.State.aborted; 1210 request.responseData.code = 4; 1211 request.responseData.codeText = e.msg; 1212 1213 inactive[inactiveCount++] = sock; 1214 sock.close(); 1215 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1216 continue; 1217 } 1218 1219 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 1220 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 1221 inactive[inactiveCount++] = sock; 1222 continue; 1223 // reuse the socket for another pending request, if we can 1224 } 1225 } 1226 1227 if(request.onDataReceived) 1228 request.onDataReceived(request); 1229 1230 version(with_openssl) 1231 if(auto s = cast(SslClientSocket) sock) { 1232 // select doesn't handle the case with stuff 1233 // left in the ssl buffer so i'm checking it separately 1234 if(s.dataPending()) { 1235 goto keep_going; 1236 } 1237 } 1238 } 1239 1240 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1241 if(writeSet.isSet(sock)) { 1242 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1243 assert(request.sendBuffer.length); 1244 auto sent = sock.send(request.sendBuffer); 1245 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1246 if(sent <= 0) 1247 throw new Exception("send error " ~ lastSocketError); 1248 request.sendBuffer = request.sendBuffer[sent .. $]; 1249 if(request.sendBuffer.length == 0) { 1250 request.state = State.waitingForResponse; 1251 } 1252 } 1253 } 1254 } 1255 1256 killInactives(); 1257 1258 // we've completed a request, are there any more pending connection? if so, send them now 1259 1260 return 0; 1261 } 1262 } 1263 1264 public static void resetInternals() { 1265 socketsPerHost = null; 1266 activeRequestOnSocket = null; 1267 pending = null; 1268 1269 } 1270 1271 struct HeaderReadingState { 1272 bool justSawLf; 1273 bool justSawCr; 1274 bool atStartOfLine = true; 1275 bool readingLineContinuation; 1276 } 1277 HeaderReadingState headerReadingState; 1278 1279 struct BodyReadingState { 1280 bool isGzipped; 1281 bool isDeflated; 1282 1283 bool isChunked; 1284 int chunkedState; 1285 1286 // used for the chunk size if it is chunked 1287 int contentLengthRemaining; 1288 } 1289 BodyReadingState bodyReadingState; 1290 1291 bool closeSocketWhenComplete; 1292 1293 import std.zlib; 1294 UnCompress uncompress; 1295 1296 const(ubyte)[] leftoverDataFromLastTime; 1297 1298 bool handleIncomingData(scope const ubyte[] dataIn) { 1299 bool stillAlive = true; 1300 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 1301 if(state == State.waitingForResponse) { 1302 state = State.readingHeaders; 1303 headerReadingState = HeaderReadingState.init; 1304 bodyReadingState = BodyReadingState.init; 1305 } 1306 1307 const(ubyte)[] data; 1308 if(leftoverDataFromLastTime.length) 1309 data = leftoverDataFromLastTime ~ dataIn[]; 1310 else 1311 data = dataIn[]; 1312 1313 if(state == State.readingHeaders) { 1314 void parseLastHeader() { 1315 assert(responseData.headers.length); 1316 if(responseData.headers.length == 1) { 1317 responseData.statusLine = responseData.headers[0]; 1318 import std.algorithm; 1319 auto parts = responseData.statusLine.splitter(" "); 1320 responseData.httpVersion = parts.front; 1321 parts.popFront(); 1322 if(parts.empty) 1323 throw new Exception("Corrupted response, bad status line"); 1324 responseData.code = to!int(parts.front()); 1325 parts.popFront(); 1326 responseData.codeText = ""; 1327 while(!parts.empty) { 1328 // FIXME: this sucks! 1329 responseData.codeText ~= parts.front(); 1330 parts.popFront(); 1331 if(!parts.empty) 1332 responseData.codeText ~= " "; 1333 } 1334 } else { 1335 // parse the new header 1336 auto header = responseData.headers[$-1]; 1337 1338 auto colon = header.indexOf(":"); 1339 if(colon == -1) 1340 return; 1341 auto name = header[0 .. colon]; 1342 if(colon + 1 == header.length || colon + 2 == header.length) // assuming a space there 1343 return; // empty header, idk 1344 assert(colon + 2 < header.length, header); 1345 auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space 1346 1347 switch(name) { 1348 case "Connection": 1349 case "connection": 1350 if(value == "close") 1351 closeSocketWhenComplete = true; 1352 break; 1353 case "Content-Type": 1354 case "content-type": 1355 responseData.contentType = value; 1356 break; 1357 case "Location": 1358 case "location": 1359 responseData.location = value; 1360 break; 1361 case "Content-Length": 1362 case "content-length": 1363 bodyReadingState.contentLengthRemaining = to!int(value); 1364 break; 1365 case "Transfer-Encoding": 1366 case "transfer-encoding": 1367 // note that if it is gzipped, it zips first, then chunks the compressed stream. 1368 // so we should always dechunk first, then feed into the decompressor 1369 if(value.strip == "chunked") 1370 bodyReadingState.isChunked = true; 1371 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 1372 break; 1373 case "Content-Encoding": 1374 case "content-encoding": 1375 if(value == "gzip") { 1376 bodyReadingState.isGzipped = true; 1377 uncompress = new UnCompress(); 1378 } else if(value == "deflate") { 1379 bodyReadingState.isDeflated = true; 1380 uncompress = new UnCompress(); 1381 } else throw new Exception("Unknown Content-Encoding: " ~ value); 1382 break; 1383 case "Set-Cookie": 1384 case "set-cookie": 1385 // FIXME handle 1386 break; 1387 default: 1388 // ignore 1389 } 1390 1391 responseData.headersHash[name] = value; 1392 } 1393 } 1394 1395 size_t position = 0; 1396 for(position = 0; position < dataIn.length; position++) { 1397 if(headerReadingState.readingLineContinuation) { 1398 if(data[position] == ' ' || data[position] == '\t') 1399 continue; 1400 headerReadingState.readingLineContinuation = false; 1401 } 1402 1403 if(headerReadingState.atStartOfLine) { 1404 headerReadingState.atStartOfLine = false; 1405 if(data[position] == '\r' || data[position] == '\n') { 1406 // done with headers 1407 if(data[position] == '\r' && (position + 1) < data.length && data[position + 1] == '\n') 1408 position++; 1409 if(this.requestParameters.method == HttpVerb.HEAD) 1410 state = State.complete; 1411 else 1412 state = State.readingBody; 1413 position++; // skip the newline 1414 break; 1415 } else if(data[position] == ' ' || data[position] == '\t') { 1416 // line continuation, ignore all whitespace and collapse it into a space 1417 headerReadingState.readingLineContinuation = true; 1418 responseData.headers[$-1] ~= ' '; 1419 } else { 1420 // new header 1421 if(responseData.headers.length) 1422 parseLastHeader(); 1423 responseData.headers ~= ""; 1424 } 1425 } 1426 1427 if(data[position] == '\r') { 1428 headerReadingState.justSawCr = true; 1429 continue; 1430 } else 1431 headerReadingState.justSawCr = false; 1432 1433 if(data[position] == '\n') { 1434 headerReadingState.justSawLf = true; 1435 headerReadingState.atStartOfLine = true; 1436 continue; 1437 } else 1438 headerReadingState.justSawLf = false; 1439 1440 responseData.headers[$-1] ~= data[position]; 1441 } 1442 1443 parseLastHeader(); 1444 data = data[position .. $]; 1445 } 1446 1447 if(state == State.readingBody) { 1448 if(bodyReadingState.isChunked) { 1449 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 1450 // read binary data of that length. it is our content 1451 // repeat until a zero sized chunk 1452 // then read footers as headers. 1453 1454 start_over: 1455 for(int a = 0; a < data.length; a++) { 1456 final switch(bodyReadingState.chunkedState) { 1457 case 0: // reading hex 1458 char c = data[a]; 1459 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 1460 // just keep reading 1461 } else { 1462 int power = 1; 1463 bodyReadingState.contentLengthRemaining = 0; 1464 assert(a != 0, cast(string) data); 1465 for(int b = a-1; b >= 0; b--) { 1466 char cc = data[b]; 1467 if(cc >= 'a' && cc <= 'z') 1468 cc -= 0x20; 1469 int val = 0; 1470 if(cc >= '0' && cc <= '9') 1471 val = cc - '0'; 1472 else 1473 val = cc - 'A' + 10; 1474 1475 assert(val >= 0 && val <= 15, to!string(val)); 1476 bodyReadingState.contentLengthRemaining += power * val; 1477 power *= 16; 1478 } 1479 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 1480 bodyReadingState.chunkedState = 1; 1481 data = data[a + 1 .. $]; 1482 goto start_over; 1483 } 1484 break; 1485 case 1: // reading until end of line 1486 char c = data[a]; 1487 if(c == '\n') { 1488 if(bodyReadingState.contentLengthRemaining == 0) 1489 bodyReadingState.chunkedState = 5; 1490 else 1491 bodyReadingState.chunkedState = 2; 1492 } 1493 data = data[a + 1 .. $]; 1494 goto start_over; 1495 case 2: // reading data 1496 auto can = a + bodyReadingState.contentLengthRemaining; 1497 if(can > data.length) 1498 can = cast(int) data.length; 1499 1500 auto newData = data[a .. can]; 1501 data = data[can .. $]; 1502 1503 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1504 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 1505 //else 1506 responseData.content ~= newData; 1507 1508 bodyReadingState.contentLengthRemaining -= newData.length; 1509 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 1510 assert(bodyReadingState.contentLengthRemaining >= 0); 1511 if(bodyReadingState.contentLengthRemaining == 0) { 1512 bodyReadingState.chunkedState = 3; 1513 } else { 1514 // will continue grabbing more 1515 } 1516 goto start_over; 1517 case 3: // reading 13/10 1518 assert(data[a] == 13); 1519 bodyReadingState.chunkedState++; 1520 data = data[a + 1 .. $]; 1521 goto start_over; 1522 case 4: // reading 10 at end of packet 1523 assert(data[a] == 10); 1524 data = data[a + 1 .. $]; 1525 bodyReadingState.chunkedState = 0; 1526 goto start_over; 1527 case 5: // reading footers 1528 //goto done; // FIXME 1529 state = State.complete; 1530 1531 bodyReadingState.chunkedState = 0; 1532 1533 while(data[a] != 10) { 1534 a++; 1535 if(a == data.length) 1536 return stillAlive; // in the footer state we're just discarding everything until we're done so this should be ok 1537 } 1538 data = data[a + 1 .. $]; 1539 1540 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1541 auto n = uncompress.uncompress(responseData.content); 1542 n ~= uncompress.flush(); 1543 responseData.content = cast(ubyte[]) n; 1544 } 1545 1546 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 1547 1548 responseData.contentText = cast(string) responseData.content; 1549 1550 goto done; 1551 } 1552 } 1553 1554 done: 1555 // FIXME 1556 //if(closeSocketWhenComplete) 1557 //socket.close(); 1558 } else { 1559 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1560 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 1561 //else 1562 responseData.content ~= data; 1563 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 1564 int use = cast(int) data.length; 1565 if(use > bodyReadingState.contentLengthRemaining) 1566 use = bodyReadingState.contentLengthRemaining; 1567 bodyReadingState.contentLengthRemaining -= use; 1568 data = data[use .. $]; 1569 if(bodyReadingState.contentLengthRemaining == 0) { 1570 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1571 auto n = uncompress.uncompress(responseData.content); 1572 n ~= uncompress.flush(); 1573 responseData.content = cast(ubyte[]) n; 1574 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 1575 } 1576 if(followLocation && responseData.location.length) { 1577 static bool first = true; 1578 //version(DigitalMars) if(!first) asm { int 3; } 1579 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 1580 //import std.stdio; writeln("redirected to ", responseData.location); 1581 first = false; 1582 responseData = HttpResponse.init; 1583 headerReadingState = HeaderReadingState.init; 1584 bodyReadingState = BodyReadingState.init; 1585 state = State.unsent; 1586 stillAlive = false; 1587 sendPrivate(false); 1588 } else { 1589 state = State.complete; 1590 responseData.contentText = cast(string) responseData.content; 1591 // FIXME 1592 //if(closeSocketWhenComplete) 1593 //socket.close(); 1594 } 1595 } 1596 } 1597 } 1598 1599 if(data.length) 1600 leftoverDataFromLastTime = data.dup; 1601 else 1602 leftoverDataFromLastTime = null; 1603 1604 return stillAlive; 1605 } 1606 1607 } 1608 } 1609 1610 /// 1611 struct HttpRequestParameters { 1612 // FIXME: implement these 1613 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 1614 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 1615 1616 // debugging 1617 bool useHttp11 = true; /// 1618 bool acceptGzip = true; /// 1619 bool keepAlive = true; /// 1620 1621 // the request itself 1622 HttpVerb method; /// 1623 string host; /// 1624 ushort port; /// 1625 string uri; /// 1626 1627 bool ssl; /// 1628 1629 string userAgent; /// 1630 string authorization; /// 1631 1632 string[string] cookies; /// 1633 1634 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 1635 1636 string contentType; /// 1637 ubyte[] bodyData; /// 1638 1639 string unixSocketPath; 1640 } 1641 1642 interface IHttpClient { 1643 1644 } 1645 1646 /// 1647 enum HttpVerb { 1648 /// 1649 GET, 1650 /// 1651 HEAD, 1652 /// 1653 POST, 1654 /// 1655 PUT, 1656 /// 1657 DELETE, 1658 /// 1659 OPTIONS, 1660 /// 1661 TRACE, 1662 /// 1663 CONNECT, 1664 /// 1665 PATCH, 1666 /// 1667 MERGE 1668 } 1669 1670 /** 1671 Usage: 1672 1673 --- 1674 auto client = new HttpClient("localhost", 80); 1675 // relative links work based on the current url 1676 HttpRequest request = client.get("foo/bar"); 1677 request = client.get("baz"); // gets foo/baz 1678 1679 // requests are not sent until you tell them to; 1680 // they are just objects representing potential. 1681 // to realize it and fetch the response, use waitForCompletion: 1682 1683 HttpResponse response = request.waitForCompletion(); 1684 1685 // now you can use response.headers, response.contentText, etc 1686 --- 1687 */ 1688 1689 /// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 1690 class HttpClient { 1691 /* Protocol restrictions, useful to disable when debugging servers */ 1692 bool useHttp11 = true; /// 1693 bool acceptGzip = true; /// 1694 bool keepAlive = true; /// 1695 1696 /// 1697 @property Uri location() { 1698 return currentUrl; 1699 } 1700 1701 /++ 1702 Default timeout for requests created on this client. 1703 1704 History: 1705 Added March 31, 2021 1706 +/ 1707 Duration defaultTimeout = 10.seconds; 1708 1709 /// High level function that works similarly to entering a url 1710 /// into a browser. 1711 /// 1712 /// Follows locations, updates the current url. 1713 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 1714 currentUrl = where.basedOn(currentUrl); 1715 currentDomain = where.host; 1716 1717 auto request = this.request(currentUrl, method); 1718 request.followLocation = true; 1719 1720 return request; 1721 } 1722 1723 /++ 1724 Creates a request without updating the current url state 1725 (but will still save cookies btw... when that is implemented) 1726 +/ 1727 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 1728 auto request = new HttpRequest(uri, method, cache, defaultTimeout); 1729 1730 request.requestParameters.userAgent = userAgent; 1731 request.requestParameters.authorization = authorization; 1732 1733 request.requestParameters.useHttp11 = this.useHttp11; 1734 request.requestParameters.acceptGzip = this.acceptGzip; 1735 request.requestParameters.keepAlive = this.keepAlive; 1736 1737 request.requestParameters.bodyData = bodyData; 1738 request.requestParameters.contentType = contentType; 1739 1740 return request; 1741 1742 } 1743 1744 /// ditto 1745 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 1746 return request(uri, method, fd.toBytes, fd.contentType); 1747 } 1748 1749 1750 private Uri currentUrl; 1751 private string currentDomain; 1752 private ICache cache; 1753 1754 this(ICache cache = null) { 1755 this.cache = cache; 1756 1757 } 1758 1759 // FIXME: add proxy 1760 // FIXME: some kind of caching 1761 1762 /// 1763 void setCookie(string name, string value, string domain = null) { 1764 if(domain == null) 1765 domain = currentDomain; 1766 1767 cookies[domain][name] = value; 1768 } 1769 1770 /// 1771 void clearCookies(string domain = null) { 1772 if(domain is null) 1773 cookies = null; 1774 else 1775 cookies[domain] = null; 1776 } 1777 1778 // If you set these, they will be pre-filled on all requests made with this client 1779 string userAgent = "D arsd.html2"; /// 1780 string authorization; /// 1781 1782 /* inter-request state */ 1783 string[string][string] cookies; 1784 } 1785 1786 interface ICache { 1787 /++ 1788 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 1789 1790 Return null if the cache does not provide. 1791 +/ 1792 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 1793 1794 /++ 1795 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 1796 1797 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 1798 +/ 1799 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 1800 } 1801 1802 /+ 1803 // / Provides caching behavior similar to a real web browser 1804 class HttpCache : ICache { 1805 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 1806 return null; 1807 } 1808 } 1809 1810 // / Gives simple maximum age caching, ignoring the actual http headers 1811 class SimpleCache : ICache { 1812 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 1813 return null; 1814 } 1815 } 1816 +/ 1817 1818 /++ 1819 A pseudo-cache to provide a mock server. Construct one of these, 1820 populate it with test responses, and pass it to [HttpClient] to 1821 do a network-free test. 1822 1823 You should populate it with the [populate] method. Any request not 1824 pre-populated will return a "server refused connection" response. 1825 +/ 1826 class HttpMockProvider : ICache { 1827 /+ + 1828 1829 +/ 1830 version(none) 1831 this(Uri baseUrl, string defaultResponseContentType) { 1832 1833 } 1834 1835 this() {} 1836 1837 HttpResponse defaultResponse; 1838 1839 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 1840 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 1841 import std.conv; 1842 auto defaultPort = request.ssl ? 443 : 80; 1843 string identifier = text( 1844 request.method, " ", 1845 request.ssl ? "https" : "http", "://", 1846 request.host, 1847 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 1848 request.uri 1849 ); 1850 1851 if(auto res = identifier in population) 1852 return res; 1853 return &defaultResponse; 1854 } 1855 1856 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 1857 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 1858 return false; 1859 } 1860 1861 /++ 1862 Convenience method to populate simple responses. For more complex 1863 work, use one of the other overloads where you build complete objects 1864 yourself. 1865 1866 Params: 1867 request = a verb and complete URL to mock as one string. 1868 For example "GET http://example.com/". If you provide only 1869 a partial URL, it will be based on the `baseUrl` you gave 1870 in the `HttpMockProvider` constructor. 1871 1872 responseCode = the HTTP response code, like 200 or 404. 1873 1874 response = the response body as a string. It is assumed 1875 to be of the `defaultResponseContentType` you passed in the 1876 `HttpMockProvider` constructor. 1877 +/ 1878 void populate(string request, int responseCode, string response) { 1879 1880 // FIXME: absolute-ize the URL in the request 1881 1882 HttpResponse r; 1883 r.code = responseCode; 1884 r.codeText = getHttpCodeText(r.code); 1885 1886 r.content = cast(ubyte[]) response; 1887 r.contentText = response; 1888 1889 population[request] = r; 1890 } 1891 1892 version(none) 1893 void populate(string method, string url, HttpResponse response) { 1894 // FIXME 1895 } 1896 1897 private HttpResponse[string] population; 1898 } 1899 1900 // modified from the one in cgi.d to just have the text 1901 private static string getHttpCodeText(int code) pure nothrow @nogc { 1902 switch(code) { 1903 // this module's proprietary extensions 1904 case 0: return null; 1905 case 1: return "request.abort called"; 1906 case 2: return "connection failed"; 1907 case 3: return "server disconnected"; 1908 case 4: return "exception thrown"; // actually should be some other thing 1909 case 5: return "Request timed out"; 1910 1911 // * * * standard ones * * * 1912 1913 // 1xx skipped since they shouldn't happen 1914 1915 // 1916 case 200: return "OK"; 1917 case 201: return "Created"; 1918 case 202: return "Accepted"; 1919 case 203: return "Non-Authoritative Information"; 1920 case 204: return "No Content"; 1921 case 205: return "Reset Content"; 1922 // 1923 case 300: return "Multiple Choices"; 1924 case 301: return "Moved Permanently"; 1925 case 302: return "Found"; 1926 case 303: return "See Other"; 1927 case 307: return "Temporary Redirect"; 1928 case 308: return "Permanent Redirect"; 1929 // 1930 case 400: return "Bad Request"; 1931 case 403: return "Forbidden"; 1932 case 404: return "Not Found"; 1933 case 405: return "Method Not Allowed"; 1934 case 406: return "Not Acceptable"; 1935 case 409: return "Conflict"; 1936 case 410: return "Gone"; 1937 // 1938 case 500: return "Internal Server Error"; 1939 case 501: return "Not Implemented"; 1940 case 502: return "Bad Gateway"; 1941 case 503: return "Service Unavailable"; 1942 // 1943 default: assert(0, "Unsupported http code"); 1944 } 1945 } 1946 1947 1948 /// 1949 struct HttpCookie { 1950 string name; /// 1951 string value; /// 1952 string domain; /// 1953 string path; /// 1954 //SysTime expirationDate; /// 1955 bool secure; /// 1956 bool httpOnly; /// 1957 } 1958 1959 // FIXME: websocket 1960 1961 version(testing) 1962 void main() { 1963 import std.stdio; 1964 auto client = new HttpClient(); 1965 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 1966 request.send(); 1967 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 1968 request2.send(); 1969 1970 { 1971 auto response = request2.waitForCompletion(); 1972 //write(cast(string) response.content); 1973 } 1974 1975 auto response = request.waitForCompletion(); 1976 write(cast(string) response.content); 1977 1978 writeln(HttpRequest.socketsPerHost); 1979 } 1980 1981 1982 // From sslsocket.d, but this is the maintained version! 1983 version(use_openssl) { 1984 alias SslClientSocket = OpenSslSocket; 1985 1986 // macros in the original C 1987 SSL_METHOD* SSLv23_client_method() { 1988 if(ossllib.SSLv23_client_method) 1989 return ossllib.SSLv23_client_method(); 1990 else 1991 return ossllib.TLS_client_method(); 1992 } 1993 1994 struct SSL {} 1995 struct SSL_CTX {} 1996 struct SSL_METHOD {} 1997 enum SSL_VERIFY_NONE = 0; 1998 1999 struct ossllib { 2000 __gshared static extern(C) { 2001 /* these are only on older openssl versions { */ 2002 int function() SSL_library_init; 2003 void function() SSL_load_error_strings; 2004 SSL_METHOD* function() SSLv23_client_method; 2005 /* } */ 2006 2007 void function(ulong, void*) OPENSSL_init_ssl; 2008 2009 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 2010 SSL* function(SSL_CTX*) SSL_new; 2011 int function(SSL*, int) SSL_set_fd; 2012 int function(SSL*) SSL_connect; 2013 int function(SSL*, const void*, int) SSL_write; 2014 int function(SSL*, void*, int) SSL_read; 2015 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 2016 void function(SSL*) SSL_free; 2017 void function(SSL_CTX*) SSL_CTX_free; 2018 2019 int function(const SSL*) SSL_pending; 2020 2021 void function(SSL*, int, void*) SSL_set_verify; 2022 2023 void function(SSL*, int, c_long, void*) SSL_ctrl; 2024 2025 SSL_METHOD* function() SSLv3_client_method; 2026 SSL_METHOD* function() TLS_client_method; 2027 2028 } 2029 } 2030 2031 import core.stdc.config; 2032 2033 struct eallib { 2034 __gshared static extern(C) { 2035 /* these are only on older openssl versions { */ 2036 void function() OpenSSL_add_all_ciphers; 2037 void function() OpenSSL_add_all_digests; 2038 /* } */ 2039 2040 void function(ulong, void*) OPENSSL_init_crypto; 2041 2042 void function(FILE*) ERR_print_errors_fp; 2043 } 2044 } 2045 2046 2047 SSL_CTX* SSL_CTX_new(const SSL_METHOD* a) { 2048 if(ossllib.SSL_CTX_new) 2049 return ossllib.SSL_CTX_new(a); 2050 else throw new Exception("SSL_CTX_new not loaded"); 2051 } 2052 SSL* SSL_new(SSL_CTX* a) { 2053 if(ossllib.SSL_new) 2054 return ossllib.SSL_new(a); 2055 else throw new Exception("SSL_new not loaded"); 2056 } 2057 int SSL_set_fd(SSL* a, int b) { 2058 if(ossllib.SSL_set_fd) 2059 return ossllib.SSL_set_fd(a, b); 2060 else throw new Exception("SSL_set_fd not loaded"); 2061 } 2062 int SSL_connect(SSL* a) { 2063 if(ossllib.SSL_connect) 2064 return ossllib.SSL_connect(a); 2065 else throw new Exception("SSL_connect not loaded"); 2066 } 2067 int SSL_write(SSL* a, const void* b, int c) { 2068 if(ossllib.SSL_write) 2069 return ossllib.SSL_write(a, b, c); 2070 else throw new Exception("SSL_write not loaded"); 2071 } 2072 int SSL_read(SSL* a, void* b, int c) { 2073 if(ossllib.SSL_read) 2074 return ossllib.SSL_read(a, b, c); 2075 else throw new Exception("SSL_read not loaded"); 2076 } 2077 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 2078 if(ossllib.SSL_shutdown) 2079 return ossllib.SSL_shutdown(a); 2080 assert(0); 2081 } 2082 void SSL_free(SSL* a) { 2083 if(ossllib.SSL_free) 2084 return ossllib.SSL_free(a); 2085 else throw new Exception("SSL_free not loaded"); 2086 } 2087 void SSL_CTX_free(SSL_CTX* a) { 2088 if(ossllib.SSL_CTX_free) 2089 return ossllib.SSL_CTX_free(a); 2090 else throw new Exception("SSL_CTX_free not loaded"); 2091 } 2092 2093 int SSL_pending(const SSL* a) { 2094 if(ossllib.SSL_pending) 2095 return ossllib.SSL_pending(a); 2096 else throw new Exception("SSL_pending not loaded"); 2097 } 2098 void SSL_set_verify(SSL* a, int b, void* c) { 2099 if(ossllib.SSL_set_verify) 2100 return ossllib.SSL_set_verify(a, b, c); 2101 else throw new Exception("SSL_set_verify not loaded"); 2102 } 2103 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 2104 if(ossllib.SSL_ctrl) 2105 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 2106 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 2107 } 2108 2109 SSL_METHOD* SSLv3_client_method() { 2110 if(ossllib.SSLv3_client_method) 2111 return ossllib.SSLv3_client_method(); 2112 else throw new Exception("SSLv3_client_method not loaded"); 2113 } 2114 SSL_METHOD* TLS_client_method() { 2115 if(ossllib.TLS_client_method) 2116 return ossllib.TLS_client_method(); 2117 else throw new Exception("TLS_client_method not loaded"); 2118 } 2119 void ERR_print_errors_fp(FILE* a) { 2120 if(eallib.ERR_print_errors_fp) 2121 return eallib.ERR_print_errors_fp(a); 2122 else throw new Exception("ERR_print_errors_fp not loaded"); 2123 } 2124 2125 2126 private __gshared void* ossllib_handle; 2127 version(Windows) 2128 private __gshared void* oeaylib_handle; 2129 else 2130 alias oeaylib_handle = ossllib_handle; 2131 version(Posix) 2132 private import core.sys.posix.dlfcn; 2133 else version(Windows) 2134 private import core.sys.windows.windows; 2135 2136 import core.stdc.stdio; 2137 2138 private __gshared Object loadSslMutex = new Object; 2139 private __gshared bool sslLoaded = false; 2140 2141 void loadOpenSsl() { 2142 if(sslLoaded) 2143 return; 2144 synchronized(loadSslMutex) { 2145 2146 version(OSX) { 2147 // newest box 2148 ossllib_handle = dlopen("libssl.1.1.dylib", RTLD_NOW); 2149 // other boxes 2150 if(ossllib_handle is null) 2151 ossllib_handle = dlopen("libssl.dylib", RTLD_NOW); 2152 // old ones like my laptop test 2153 if(ossllib_handle is null) 2154 ossllib_handle = dlopen("/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", RTLD_NOW); 2155 2156 } else version(Posix) { 2157 ossllib_handle = dlopen("libssl.so.1.1", RTLD_NOW); 2158 if(ossllib_handle is null) 2159 ossllib_handle = dlopen("libssl.so", RTLD_NOW); 2160 } else version(Windows) { 2161 ossllib_handle = LoadLibraryW("libssl32.dll"w.ptr); 2162 oeaylib_handle = LoadLibraryW("libeay32.dll"w.ptr); 2163 } 2164 2165 if(ossllib_handle is null) 2166 throw new Exception("libssl library not found"); 2167 if(oeaylib_handle is null) 2168 throw new Exception("libeay32 library not found"); 2169 2170 foreach(memberName; __traits(allMembers, ossllib)) { 2171 alias t = typeof(__traits(getMember, ossllib, memberName)); 2172 version(Posix) 2173 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 2174 else version(Windows) { 2175 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 2176 } 2177 } 2178 2179 foreach(memberName; __traits(allMembers, eallib)) { 2180 alias t = typeof(__traits(getMember, eallib, memberName)); 2181 version(Posix) 2182 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 2183 else version(Windows) { 2184 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 2185 } 2186 } 2187 2188 2189 if(ossllib.SSL_library_init) 2190 ossllib.SSL_library_init(); 2191 else if(ossllib.OPENSSL_init_ssl) 2192 ossllib.OPENSSL_init_ssl(0, null); 2193 else throw new Exception("couldn't init openssl"); 2194 2195 if(eallib.OpenSSL_add_all_ciphers) { 2196 eallib.OpenSSL_add_all_ciphers(); 2197 if(eallib.OpenSSL_add_all_digests is null) 2198 throw new Exception("no add digests"); 2199 eallib.OpenSSL_add_all_digests(); 2200 } else if(eallib.OPENSSL_init_crypto) 2201 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 2202 else throw new Exception("couldn't init crypto openssl"); 2203 2204 if(ossllib.SSL_load_error_strings) 2205 ossllib.SSL_load_error_strings(); 2206 else if(ossllib.OPENSSL_init_ssl) 2207 ossllib.OPENSSL_init_ssl(0x00200000L, null); 2208 else throw new Exception("couldn't load openssl errors"); 2209 2210 sslLoaded = true; 2211 } 2212 } 2213 2214 /+ 2215 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 2216 // might have trouble being run from the GC after this module is unloaded. 2217 shared static ~this() { 2218 if(ossllib_handle) { 2219 version(Windows) { 2220 FreeLibrary(oeaylib_handle); 2221 FreeLibrary(ossllib_handle); 2222 } else version(Posix) 2223 dlclose(ossllib_handle); 2224 ossllib_handle = null; 2225 } 2226 ossllib.tupleof = ossllib.tupleof.init; 2227 } 2228 +/ 2229 2230 //pragma(lib, "crypto"); 2231 //pragma(lib, "ssl"); 2232 2233 class OpenSslSocket : Socket { 2234 private SSL* ssl; 2235 private SSL_CTX* ctx; 2236 private void initSsl(bool verifyPeer, string hostname) { 2237 ctx = SSL_CTX_new(SSLv23_client_method()); 2238 assert(ctx !is null); 2239 2240 ssl = SSL_new(ctx); 2241 2242 if(hostname.length) 2243 SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 2244 2245 if(!verifyPeer) 2246 SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 2247 SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 2248 } 2249 2250 bool dataPending() { 2251 return SSL_pending(ssl) > 0; 2252 } 2253 2254 @trusted 2255 override void connect(Address to) { 2256 super.connect(to); 2257 if(SSL_connect(ssl) == -1) { 2258 ERR_print_errors_fp(core.stdc.stdio.stderr); 2259 int i; 2260 //printf("wtf\n"); 2261 //scanf("%d\n", i); 2262 throw new Exception("ssl connect"); 2263 } 2264 } 2265 2266 @trusted 2267 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 2268 //import std.stdio;writeln(cast(string) buf); 2269 auto retval = SSL_write(ssl, buf.ptr, cast(uint) buf.length); 2270 if(retval == -1) { 2271 ERR_print_errors_fp(core.stdc.stdio.stderr); 2272 int i; 2273 //printf("wtf\n"); 2274 //scanf("%d\n", i); 2275 throw new Exception("ssl send"); 2276 } 2277 return retval; 2278 2279 } 2280 override ptrdiff_t send(scope const(void)[] buf) { 2281 return send(buf, SocketFlags.NONE); 2282 } 2283 @trusted 2284 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 2285 auto retval = SSL_read(ssl, buf.ptr, cast(int)buf.length); 2286 if(retval == -1) { 2287 ERR_print_errors_fp(core.stdc.stdio.stderr); 2288 int i; 2289 //printf("wtf\n"); 2290 //scanf("%d\n", i); 2291 throw new Exception("ssl send"); 2292 } 2293 return retval; 2294 } 2295 override ptrdiff_t receive(scope void[] buf) { 2296 return receive(buf, SocketFlags.NONE); 2297 } 2298 2299 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 2300 super(af, type); 2301 initSsl(verifyPeer, hostname); 2302 } 2303 2304 override void close() { 2305 if(ssl) SSL_shutdown(ssl); 2306 super.close(); 2307 } 2308 2309 this(socket_t sock, AddressFamily af, string hostname) { 2310 super(sock, af); 2311 initSsl(true, hostname); 2312 } 2313 2314 ~this() { 2315 SSL_free(ssl); 2316 SSL_CTX_free(ctx); 2317 ssl = null; 2318 } 2319 } 2320 } 2321 2322 2323 /++ 2324 An experimental component for working with REST apis. Note that it 2325 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 2326 or you will get "HttpApiClient is used as a type" compile errors. 2327 2328 This will probably not work for you yet, and I might change it significantly. 2329 2330 Requires [arsd.jsvar]. 2331 2332 2333 Here's a snippet to create a pull request on GitHub to Phobos: 2334 2335 --- 2336 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 2337 2338 // create the arguments object 2339 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 2340 var args = var.emptyObject; 2341 args.title = "My Pull Request"; 2342 args.head = "yourusername:" ~ branchName; 2343 args.base = "master"; 2344 // note it is ["body"] instead of .body because `body` is a D keyword 2345 args["body"] = "My cool PR is opened by the API!"; 2346 args.maintainer_can_modify = true; 2347 2348 /+ 2349 Fun fact, you can also write that: 2350 2351 var args = [ 2352 "title": "My Pull Request".var, 2353 "head": "yourusername:" ~ branchName.var, 2354 "base" : "master".var, 2355 "body" : "My cool PR is opened by the API!".var, 2356 "maintainer_can_modify": true.var 2357 ]; 2358 2359 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 2360 +/ 2361 2362 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 2363 // containing `args` as json, then immediately grabs the json result and extracts 2364 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 2365 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 2366 2367 writeln("Created: ", prUrl); 2368 --- 2369 2370 Why use this instead of just building the URL? Well, of course you can! This just makes 2371 it a bit more convenient than string concatenation and manages a few headers for you. 2372 2373 Subtypes could potentially add static type checks too. 2374 +/ 2375 class HttpApiClient() { 2376 import arsd.jsvar; 2377 2378 HttpClient httpClient; 2379 2380 alias HttpApiClientType = typeof(this); 2381 2382 string urlBase; 2383 string oauth2Token; 2384 string submittedContentType; 2385 2386 /++ 2387 Params: 2388 2389 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 2390 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 2391 submittedContentType = the content-type of POST, PUT, etc. bodies. 2392 httpClient = an injected http client, or null if you want to use a default-constructed one 2393 2394 History: 2395 The `httpClient` param was added on December 26, 2020. 2396 +/ 2397 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 2398 if(httpClient is null) 2399 this.httpClient = new HttpClient(); 2400 else 2401 this.httpClient = httpClient; 2402 2403 assert(urlBase[0] == 'h'); 2404 assert(urlBase[$-1] == '/'); 2405 2406 this.urlBase = urlBase; 2407 this.oauth2Token = oauth2Token; 2408 this.submittedContentType = submittedContentType; 2409 } 2410 2411 /// 2412 static struct HttpRequestWrapper { 2413 HttpApiClientType apiClient; /// 2414 HttpRequest request; /// 2415 HttpResponse _response; 2416 2417 /// 2418 this(HttpApiClientType apiClient, HttpRequest request) { 2419 this.apiClient = apiClient; 2420 this.request = request; 2421 } 2422 2423 /// Returns the full [HttpResponse] object so you can inspect the headers 2424 @property HttpResponse response() { 2425 if(_response is HttpResponse.init) 2426 _response = request.waitForCompletion(); 2427 return _response; 2428 } 2429 2430 /++ 2431 Returns the parsed JSON from the body of the response. 2432 2433 Throws on non-2xx responses. 2434 +/ 2435 var result() { 2436 return apiClient.throwOnError(response); 2437 } 2438 2439 alias request this; 2440 } 2441 2442 /// 2443 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 2444 if(uri[0] == '/') 2445 uri = uri[1 .. $]; 2446 2447 auto u = Uri(uri).basedOn(Uri(urlBase)); 2448 2449 auto req = httpClient.navigateTo(u, requestMethod); 2450 2451 if(oauth2Token.length) 2452 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 2453 req.requestParameters.contentType = submittedContentType; 2454 req.requestParameters.bodyData = bodyBytes; 2455 2456 return HttpRequestWrapper(this, req); 2457 } 2458 2459 /// 2460 var throwOnError(HttpResponse res) { 2461 if(res.code < 200 || res.code >= 300) 2462 throw new Exception(res.codeText ~ " " ~ res.contentText); 2463 2464 var response = var.fromJson(res.contentText); 2465 if(response.errors) { 2466 throw new Exception(response.errors.toJson()); 2467 } 2468 2469 return response; 2470 } 2471 2472 /// 2473 @property RestBuilder rest() { 2474 return RestBuilder(this, null, null); 2475 } 2476 2477 // hipchat.rest.room["Tech Team"].history 2478 // gives: "/room/Tech%20Team/history" 2479 // 2480 // hipchat.rest.room["Tech Team"].history("page", "12) 2481 /// 2482 static struct RestBuilder { 2483 HttpApiClientType apiClient; 2484 string[] pathParts; 2485 string[2][] queryParts; 2486 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 2487 this.apiClient = apiClient; 2488 this.pathParts = pathParts; 2489 this.queryParts = queryParts; 2490 } 2491 2492 RestBuilder _SELF() { 2493 return this; 2494 } 2495 2496 /// The args are so you can call opCall on the returned 2497 /// object, despite @property being broken af in D. 2498 RestBuilder opDispatch(string str, T)(string n, T v) { 2499 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 2500 } 2501 2502 /// 2503 RestBuilder opDispatch(string str)() { 2504 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2505 } 2506 2507 2508 /// 2509 RestBuilder opIndex(string str) { 2510 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2511 } 2512 /// 2513 RestBuilder opIndex(var str) { 2514 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 2515 } 2516 /// 2517 RestBuilder opIndex(int i) { 2518 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 2519 } 2520 2521 /// 2522 RestBuilder opCall(T)(string name, T value) { 2523 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 2524 } 2525 2526 /// 2527 string toUri() { 2528 import std.uri; 2529 string result; 2530 foreach(idx, part; pathParts) { 2531 if(idx) 2532 result ~= "/"; 2533 result ~= encodeComponent(part); 2534 } 2535 result ~= "?"; 2536 foreach(idx, part; queryParts) { 2537 if(idx) 2538 result ~= "&"; 2539 result ~= encodeComponent(part[0]); 2540 result ~= "="; 2541 result ~= encodeComponent(part[1]); 2542 } 2543 2544 return result; 2545 } 2546 2547 /// 2548 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 2549 /// ditto 2550 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 2551 2552 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 2553 /// ditto 2554 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 2555 /// ditto 2556 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 2557 /// ditto 2558 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 2559 2560 struct ToBytesResult { 2561 ubyte[] bytes; 2562 string contentType; 2563 } 2564 2565 private ToBytesResult toBytes(T...)(T t) { 2566 import std.conv : to; 2567 static if(T.length == 0) 2568 return ToBytesResult(null, null); 2569 else static if(T.length == 1 && is(T[0] == var)) 2570 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 2571 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 2572 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 2573 else static if(T.length == 1 && is(T[0] : FormData)) 2574 return ToBytesResult(t[0].toBytes, t[0].contentType); 2575 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 2576 // string -> value pairs for a POST request 2577 string answer; 2578 foreach(idx, val; t) { 2579 static if(idx % 2 == 0) { 2580 if(answer.length) 2581 answer ~= "&"; 2582 answer ~= encodeComponent(val); // it had better be a string! lol 2583 answer ~= "="; 2584 } else { 2585 answer ~= encodeComponent(to!string(val)); 2586 } 2587 } 2588 2589 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 2590 } 2591 else 2592 static assert(0); // FIXME 2593 2594 } 2595 2596 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 2597 return apiClient.request(uri, verb, bodyBytes); 2598 } 2599 2600 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 2601 auto r = apiClient.request(uri, verb, tbr.bytes); 2602 if(tbr.contentType !is null) 2603 r.requestParameters.contentType = tbr.contentType; 2604 return r; 2605 } 2606 } 2607 } 2608 2609 2610 // see also: arsd.cgi.encodeVariables 2611 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 2612 class FormData { 2613 struct MimePart { 2614 string name; 2615 const(void)[] data; 2616 string contentType; 2617 string filename; 2618 } 2619 2620 MimePart[] parts; 2621 2622 /// 2623 void append(string key, in void[] value, string contentType = null, string filename = null) { 2624 parts ~= MimePart(key, value, contentType, filename); 2625 } 2626 2627 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 2628 2629 string contentType() { 2630 return "multipart/form-data; boundary=" ~ boundary; 2631 } 2632 2633 /// 2634 ubyte[] toBytes() { 2635 string data; 2636 2637 foreach(part; parts) { 2638 data ~= "--" ~ boundary ~ "\r\n"; 2639 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 2640 if(part.filename !is null) 2641 data ~= "; filename=\""~part.filename~"\""; 2642 data ~= "\r\n"; 2643 if(part.contentType !is null) 2644 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 2645 data ~= "\r\n"; 2646 2647 data ~= cast(string) part.data; 2648 2649 data ~= "\r\n"; 2650 } 2651 2652 data ~= "--" ~ boundary ~ "--\r\n"; 2653 2654 return cast(ubyte[]) data; 2655 } 2656 } 2657 2658 private bool bicmp(in ubyte[] item, in char[] search) { 2659 if(item.length != search.length) return false; 2660 2661 foreach(i; 0 .. item.length) { 2662 ubyte a = item[i]; 2663 ubyte b = search[i]; 2664 if(a >= 'A' && a <= 'Z') 2665 a += 32; 2666 //if(b >= 'A' && b <= 'Z') 2667 //b += 32; 2668 if(a != b) 2669 return false; 2670 } 2671 2672 return true; 2673 } 2674 2675 /++ 2676 WebSocket client, based on the browser api, though also with other api options. 2677 2678 --- 2679 auto ws = new WebSocket(URI("ws://....")); 2680 2681 ws.onmessage = (in char[] msg) { 2682 ws.send("a reply"); 2683 }; 2684 2685 ws.connect(); 2686 2687 WebSocket.eventLoop(); 2688 --- 2689 2690 Symbol_groups: 2691 foundational = 2692 Used with all API styles. 2693 2694 browser_api = 2695 API based on the standard in the browser. 2696 2697 event_loop_integration = 2698 Integrating with external event loops is done through static functions. You should 2699 call these BEFORE doing anything else with the WebSocket module or class. 2700 2701 $(PITFALL NOT IMPLEMENTED) 2702 --- 2703 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 2704 // or something like that. it is not implemented yet. 2705 --- 2706 $(PITFALL NOT IMPLEMENTED) 2707 2708 blocking_api = 2709 The blocking API is best used when you only need basic functionality with a single connection. 2710 2711 --- 2712 WebSocketFrame msg; 2713 do { 2714 // FIXME good demo 2715 } while(msg); 2716 --- 2717 2718 Or to check for blocks before calling: 2719 2720 --- 2721 try_to_process_more: 2722 while(ws.isMessageBuffered()) { 2723 auto msg = ws.waitForNextMessage(); 2724 // process msg 2725 } 2726 if(ws.isDataPending()) { 2727 ws.lowLevelReceive(); 2728 goto try_to_process_more; 2729 } else { 2730 // nothing ready, you can do other things 2731 // or at least sleep a while before trying 2732 // to process more. 2733 if(ws.readyState == WebSocket.OPEN) { 2734 Thread.sleep(1.seconds); 2735 goto try_to_process_more; 2736 } 2737 } 2738 --- 2739 2740 +/ 2741 class WebSocket { 2742 private Uri uri; 2743 private string[string] cookies; 2744 private string origin; 2745 2746 private string host; 2747 private ushort port; 2748 private bool ssl; 2749 2750 private MonoTime timeoutFromInactivity; 2751 private MonoTime nextPing; 2752 2753 /++ 2754 wss://echo.websocket.org 2755 +/ 2756 /// Group: foundational 2757 this(Uri uri, Config config = Config.init) 2758 //in (uri.scheme == "ws" || uri.scheme == "wss") 2759 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 2760 { 2761 this.uri = uri; 2762 this.config = config; 2763 2764 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 2765 2766 host = uri.host; 2767 ssl = uri.scheme == "wss"; 2768 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 2769 2770 if(ssl) { 2771 version(with_openssl) { 2772 loadOpenSsl(); 2773 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host); 2774 } else 2775 throw new Exception("SSL not compiled in"); 2776 } else 2777 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 2778 2779 } 2780 2781 /++ 2782 2783 +/ 2784 /// Group: foundational 2785 void connect() { 2786 if(uri.unixSocketPath) 2787 socket.connect(new UnixAddress(uri.unixSocketPath)); 2788 else 2789 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 2790 // FIXME: websocket handshake could and really should be async too. 2791 2792 auto uri = this.uri.path.length ? this.uri.path : "/"; 2793 if(this.uri.query.length) { 2794 uri ~= "?"; 2795 uri ~= this.uri.query; 2796 } 2797 2798 // the headers really shouldn't be bigger than this, at least 2799 // the chunks i need to process 2800 ubyte[4096] bufferBacking = void; 2801 ubyte[] buffer = bufferBacking[]; 2802 size_t pos; 2803 2804 void append(in char[][] items...) { 2805 foreach(what; items) { 2806 if((pos + what.length) > buffer.length) { 2807 buffer.length += 4096; 2808 } 2809 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 2810 pos += what.length; 2811 } 2812 } 2813 2814 append("GET ", uri, " HTTP/1.1\r\n"); 2815 append("Host: ", this.uri.host, "\r\n"); 2816 2817 append("Upgrade: websocket\r\n"); 2818 append("Connection: Upgrade\r\n"); 2819 append("Sec-WebSocket-Version: 13\r\n"); 2820 2821 // FIXME: randomize this 2822 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 2823 2824 if(config.protocol.length) 2825 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 2826 if(config.origin.length) 2827 append("Origin: ", origin, "\r\n"); 2828 2829 foreach(h; config.additionalHeaders) { 2830 append(h); 2831 append("\r\n"); 2832 } 2833 2834 append("\r\n"); 2835 2836 auto remaining = buffer[0 .. pos]; 2837 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 2838 while(remaining.length) { 2839 auto r = socket.send(remaining); 2840 if(r < 0) 2841 throw new Exception(lastSocketError()); 2842 if(r == 0) 2843 throw new Exception("unexpected connection termination"); 2844 remaining = remaining[r .. $]; 2845 } 2846 2847 // the response shouldn't be especially large at this point, just 2848 // headers for the most part. gonna try to get it in the stack buffer. 2849 // then copy stuff after headers, if any, to the frame buffer. 2850 ubyte[] used; 2851 2852 void more() { 2853 auto r = socket.receive(buffer[used.length .. $]); 2854 2855 if(r < 0) 2856 throw new Exception(lastSocketError()); 2857 if(r == 0) 2858 throw new Exception("unexpected connection termination"); 2859 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 2860 2861 used = buffer[0 .. used.length + r]; 2862 } 2863 2864 more(); 2865 2866 import std.algorithm; 2867 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 2868 throw new Exception("didn't get a websocket answer"); 2869 // skip the status line 2870 while(used.length && used[0] != '\n') 2871 used = used[1 .. $]; 2872 2873 if(used.length == 0) 2874 throw new Exception("Remote server disconnected or didn't send enough information"); 2875 2876 if(used.length < 1) 2877 more(); 2878 2879 used = used[1 .. $]; // skip the \n 2880 2881 if(used.length == 0) 2882 more(); 2883 2884 // checks on the protocol from ehaders 2885 bool isWebsocket; 2886 bool isUpgrade; 2887 const(ubyte)[] protocol; 2888 const(ubyte)[] accept; 2889 2890 while(used.length) { 2891 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 2892 used = used[2 .. $]; 2893 break; // all done 2894 } 2895 int idxColon; 2896 while(idxColon < used.length && used[idxColon] != ':') 2897 idxColon++; 2898 if(idxColon == used.length) 2899 more(); 2900 auto idxStart = idxColon + 1; 2901 while(idxStart < used.length && used[idxStart] == ' ') 2902 idxStart++; 2903 if(idxStart == used.length) 2904 more(); 2905 auto idxEnd = idxStart; 2906 while(idxEnd < used.length && used[idxEnd] != '\r') 2907 idxEnd++; 2908 if(idxEnd == used.length) 2909 more(); 2910 2911 auto headerName = used[0 .. idxColon]; 2912 auto headerValue = used[idxStart .. idxEnd]; 2913 2914 // move past this header 2915 used = used[idxEnd .. $]; 2916 // and the \r\n 2917 if(2 <= used.length) 2918 used = used[2 .. $]; 2919 2920 if(headerName.bicmp("upgrade")) { 2921 if(headerValue.bicmp("websocket")) 2922 isWebsocket = true; 2923 } else if(headerName.bicmp("connection")) { 2924 if(headerValue.bicmp("upgrade")) 2925 isUpgrade = true; 2926 } else if(headerName.bicmp("sec-websocket-accept")) { 2927 accept = headerValue; 2928 } else if(headerName.bicmp("sec-websocket-protocol")) { 2929 protocol = headerValue; 2930 } 2931 2932 if(!used.length) { 2933 more(); 2934 } 2935 } 2936 2937 2938 if(!isWebsocket) 2939 throw new Exception("didn't answer as websocket"); 2940 if(!isUpgrade) 2941 throw new Exception("didn't answer as upgrade"); 2942 2943 2944 // FIXME: check protocol if config requested one 2945 // FIXME: check accept for the right hash 2946 2947 receiveBuffer[0 .. used.length] = used[]; 2948 receiveBufferUsedLength = used.length; 2949 2950 readyState_ = OPEN; 2951 2952 if(onopen) 2953 onopen(); 2954 2955 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 2956 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 2957 2958 registerActiveSocket(this); 2959 } 2960 2961 /++ 2962 Is data pending on the socket? Also check [isMessageBuffered] to see if there 2963 is already a message in memory too. 2964 2965 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 2966 again. 2967 +/ 2968 /// Group: blocking_api 2969 public bool isDataPending(Duration timeout = 0.seconds) { 2970 static SocketSet readSet; 2971 if(readSet is null) 2972 readSet = new SocketSet(); 2973 2974 version(with_openssl) 2975 if(auto s = cast(SslClientSocket) socket) { 2976 // select doesn't handle the case with stuff 2977 // left in the ssl buffer so i'm checking it separately 2978 if(s.dataPending()) { 2979 return true; 2980 } 2981 } 2982 2983 readSet.add(socket); 2984 2985 //tryAgain: 2986 auto selectGot = Socket.select(readSet, null, null, timeout); 2987 if(selectGot == 0) { /* timeout */ 2988 // timeout 2989 return false; 2990 } else if(selectGot == -1) { /* interrupted */ 2991 return false; 2992 } else { /* ready */ 2993 if(readSet.isSet(socket)) { 2994 return true; 2995 } 2996 } 2997 2998 return false; 2999 } 3000 3001 private void llsend(ubyte[] d) { 3002 if(readyState == CONNECTING) 3003 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 3004 //connect(); 3005 while(d.length) { 3006 auto r = socket.send(d); 3007 if(r <= 0) throw new Exception("Socket send failed"); 3008 d = d[r .. $]; 3009 } 3010 } 3011 3012 private void llclose() { 3013 socket.shutdown(SocketShutdown.SEND); 3014 } 3015 3016 /++ 3017 Waits for more data off the low-level socket and adds it to the pending buffer. 3018 3019 Returns `true` if the connection is still active. 3020 +/ 3021 /// Group: blocking_api 3022 public bool lowLevelReceive() { 3023 if(readyState == CONNECTING) 3024 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 3025 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 3026 if(r == 0) 3027 return false; 3028 if(r <= 0) 3029 throw new Exception("Socket receive failed"); 3030 receiveBufferUsedLength += r; 3031 return true; 3032 } 3033 3034 private Socket socket; 3035 3036 /* copy/paste section { */ 3037 3038 private int readyState_; 3039 private ubyte[] receiveBuffer; 3040 private size_t receiveBufferUsedLength; 3041 3042 private Config config; 3043 3044 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 3045 enum OPEN = 1; /// The connection is open and ready to communicate. 3046 enum CLOSING = 2; /// The connection is in the process of closing. 3047 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 3048 3049 /++ 3050 3051 +/ 3052 /// Group: foundational 3053 static struct Config { 3054 /++ 3055 These control the size of the receive buffer. 3056 3057 It starts at the initial size, will temporarily 3058 balloon up to the maximum size, and will reuse 3059 a buffer up to the likely size. 3060 3061 Anything larger than the maximum size will cause 3062 the connection to be aborted and an exception thrown. 3063 This is to protect you against a peer trying to 3064 exhaust your memory, while keeping the user-level 3065 processing simple. 3066 +/ 3067 size_t initialReceiveBufferSize = 4096; 3068 size_t likelyReceiveBufferSize = 4096; /// ditto 3069 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 3070 3071 /++ 3072 Maximum combined size of a message. 3073 +/ 3074 size_t maximumMessageSize = 10 * 1024 * 1024; 3075 3076 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 3077 string origin; /// Origin URL to send with the handshake, if desired. 3078 string protocol; /// the protocol header, if desired. 3079 3080 /++ 3081 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 3082 3083 --- 3084 Config config; 3085 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 3086 --- 3087 3088 History: 3089 Added February 19, 2021 (included in dub version 9.2) 3090 +/ 3091 string[] additionalHeaders; 3092 3093 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 3094 3095 /++ 3096 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 3097 3098 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 3099 3100 History: 3101 Added March 31, 2021 (included in dub version 9.4) 3102 +/ 3103 Duration timeoutFromInactivity = 1.minutes; 3104 } 3105 3106 /++ 3107 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 3108 +/ 3109 int readyState() { 3110 return readyState_; 3111 } 3112 3113 /++ 3114 Closes the connection, sending a graceful teardown message to the other side. 3115 +/ 3116 /// Group: foundational 3117 void close(int code = 0, string reason = null) 3118 //in (reason.length < 123) 3119 in { assert(reason.length < 123); } do 3120 { 3121 if(readyState_ != OPEN) 3122 return; // it cool, we done 3123 WebSocketFrame wss; 3124 wss.fin = true; 3125 wss.opcode = WebSocketOpcode.close; 3126 wss.data = cast(ubyte[]) reason; 3127 wss.send(&llsend); 3128 3129 readyState_ = CLOSING; 3130 3131 llclose(); 3132 } 3133 3134 /++ 3135 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 3136 +/ 3137 /// Group: foundational 3138 void ping() { 3139 WebSocketFrame wss; 3140 wss.fin = true; 3141 wss.opcode = WebSocketOpcode.ping; 3142 wss.send(&llsend); 3143 } 3144 3145 // automatically handled.... 3146 void pong() { 3147 WebSocketFrame wss; 3148 wss.fin = true; 3149 wss.opcode = WebSocketOpcode.pong; 3150 wss.send(&llsend); 3151 } 3152 3153 /++ 3154 Sends a text message through the websocket. 3155 +/ 3156 /// Group: foundational 3157 void send(in char[] textData) { 3158 WebSocketFrame wss; 3159 wss.fin = true; 3160 wss.opcode = WebSocketOpcode.text; 3161 wss.data = cast(ubyte[]) textData; 3162 wss.send(&llsend); 3163 } 3164 3165 /++ 3166 Sends a binary message through the websocket. 3167 +/ 3168 /// Group: foundational 3169 void send(in ubyte[] binaryData) { 3170 WebSocketFrame wss; 3171 wss.fin = true; 3172 wss.opcode = WebSocketOpcode.binary; 3173 wss.data = cast(ubyte[]) binaryData; 3174 wss.send(&llsend); 3175 } 3176 3177 /++ 3178 Waits for and returns the next complete message on the socket. 3179 3180 Note that the onmessage function is still called, right before 3181 this returns. 3182 +/ 3183 /// Group: blocking_api 3184 public WebSocketFrame waitForNextMessage() { 3185 do { 3186 auto m = processOnce(); 3187 if(m.populated) 3188 return m; 3189 } while(lowLevelReceive()); 3190 3191 return WebSocketFrame.init; // FIXME? maybe. 3192 } 3193 3194 /++ 3195 Tells if [waitForNextMessage] would block. 3196 +/ 3197 /// Group: blocking_api 3198 public bool waitForNextMessageWouldBlock() { 3199 checkAgain: 3200 if(isMessageBuffered()) 3201 return false; 3202 if(!isDataPending()) 3203 return true; 3204 while(isDataPending()) 3205 lowLevelReceive(); 3206 goto checkAgain; 3207 } 3208 3209 /++ 3210 Is there a message in the buffer already? 3211 If `true`, [waitForNextMessage] is guaranteed to return immediately. 3212 If `false`, check [isDataPending] as the next step. 3213 +/ 3214 /// Group: blocking_api 3215 public bool isMessageBuffered() { 3216 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3217 auto s = d; 3218 if(d.length) { 3219 auto orig = d; 3220 auto m = WebSocketFrame.read(d); 3221 // that's how it indicates that it needs more data 3222 if(d !is orig) 3223 return true; 3224 } 3225 3226 return false; 3227 } 3228 3229 private ubyte continuingType; 3230 private ubyte[] continuingData; 3231 //private size_t continuingDataLength; 3232 3233 private WebSocketFrame processOnce() { 3234 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3235 //import std.stdio; writeln(d); 3236 auto s = d; 3237 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 3238 WebSocketFrame m; 3239 if(d.length) { 3240 auto orig = d; 3241 m = WebSocketFrame.read(d); 3242 // that's how it indicates that it needs more data 3243 if(d is orig) 3244 return WebSocketFrame.init; 3245 m.unmaskInPlace(); 3246 switch(m.opcode) { 3247 case WebSocketOpcode.continuation: 3248 if(continuingData.length + m.data.length > config.maximumMessageSize) 3249 throw new Exception("message size exceeded"); 3250 3251 continuingData ~= m.data; 3252 if(m.fin) { 3253 if(ontextmessage) 3254 ontextmessage(cast(char[]) continuingData); 3255 if(onbinarymessage) 3256 onbinarymessage(continuingData); 3257 3258 continuingData = null; 3259 } 3260 break; 3261 case WebSocketOpcode.text: 3262 if(m.fin) { 3263 if(ontextmessage) 3264 ontextmessage(m.textData); 3265 } else { 3266 continuingType = m.opcode; 3267 //continuingDataLength = 0; 3268 continuingData = null; 3269 continuingData ~= m.data; 3270 } 3271 break; 3272 case WebSocketOpcode.binary: 3273 if(m.fin) { 3274 if(onbinarymessage) 3275 onbinarymessage(m.data); 3276 } else { 3277 continuingType = m.opcode; 3278 //continuingDataLength = 0; 3279 continuingData = null; 3280 continuingData ~= m.data; 3281 } 3282 break; 3283 case WebSocketOpcode.close: 3284 readyState_ = CLOSED; 3285 if(onclose) 3286 onclose(); 3287 3288 unregisterActiveSocket(this); 3289 break; 3290 case WebSocketOpcode.ping: 3291 pong(); 3292 break; 3293 case WebSocketOpcode.pong: 3294 // just really references it is still alive, nbd. 3295 break; 3296 default: // ignore though i could and perhaps should throw too 3297 } 3298 } 3299 3300 if(d.length) { 3301 m.data = m.data.dup(); 3302 } 3303 3304 import core.stdc..string; 3305 memmove(receiveBuffer.ptr, d.ptr, d.length); 3306 receiveBufferUsedLength = d.length; 3307 3308 return m; 3309 } 3310 3311 private void autoprocess() { 3312 // FIXME 3313 do { 3314 processOnce(); 3315 } while(lowLevelReceive()); 3316 } 3317 3318 3319 void delegate() onclose; /// 3320 void delegate() onerror; /// 3321 void delegate(in char[]) ontextmessage; /// 3322 void delegate(in ubyte[]) onbinarymessage; /// 3323 void delegate() onopen; /// 3324 3325 /++ 3326 3327 +/ 3328 /// Group: browser_api 3329 void onmessage(void delegate(in char[]) dg) { 3330 ontextmessage = dg; 3331 } 3332 3333 /// ditto 3334 void onmessage(void delegate(in ubyte[]) dg) { 3335 onbinarymessage = dg; 3336 } 3337 3338 /* } end copy/paste */ 3339 3340 /* 3341 const int bufferedAmount // amount pending 3342 const string extensions 3343 3344 const string protocol 3345 const string url 3346 */ 3347 3348 static { 3349 /++ 3350 3351 +/ 3352 void eventLoop() { 3353 3354 static SocketSet readSet; 3355 3356 if(readSet is null) 3357 readSet = new SocketSet(); 3358 3359 loopExited = false; 3360 3361 outermost: while(!loopExited) { 3362 readSet.reset(); 3363 3364 Duration timeout = 10.seconds; 3365 3366 auto now = MonoTime.currTime; 3367 bool hadAny; 3368 foreach(sock; activeSockets) { 3369 if(now >= sock.timeoutFromInactivity) { 3370 // timeout 3371 if(sock.onerror) 3372 sock.onerror(); 3373 3374 sock.socket.close(); 3375 sock.readyState_ = CLOSED; 3376 unregisterActiveSocket(sock); 3377 continue outermost; 3378 } 3379 3380 if(now >= sock.nextPing) { 3381 sock.ping(); 3382 sock.nextPing = now + sock.config.pingFrequency.msecs; 3383 } 3384 3385 auto timeo = sock.timeoutFromInactivity - now; 3386 if(timeo < timeout) 3387 timeout = timeo; 3388 3389 readSet.add(sock.socket); 3390 hadAny = true; 3391 } 3392 3393 if(!hadAny) 3394 return; 3395 3396 tryAgain: 3397 auto selectGot = Socket.select(readSet, null, null, timeout); 3398 if(selectGot == 0) { /* timeout */ 3399 // timeout 3400 continue; // it will be handled at the top of the loop 3401 } else if(selectGot == -1) { /* interrupted */ 3402 goto tryAgain; 3403 } else { 3404 foreach(sock; activeSockets) { 3405 if(readSet.isSet(sock.socket)) { 3406 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 3407 if(!sock.lowLevelReceive()) { 3408 sock.readyState_ = CLOSED; 3409 unregisterActiveSocket(sock); 3410 continue outermost; 3411 } 3412 while(sock.processOnce().populated) {} 3413 selectGot--; 3414 if(selectGot <= 0) 3415 break; 3416 } 3417 } 3418 } 3419 } 3420 } 3421 3422 private bool loopExited; 3423 /++ 3424 3425 +/ 3426 void exitEventLoop() { 3427 loopExited = true; 3428 } 3429 3430 WebSocket[] activeSockets; 3431 void registerActiveSocket(WebSocket s) { 3432 activeSockets ~= s; 3433 } 3434 void unregisterActiveSocket(WebSocket s) { 3435 foreach(i, a; activeSockets) 3436 if(s is a) { 3437 activeSockets[i] = activeSockets[$-1]; 3438 activeSockets = activeSockets[0 .. $-1]; 3439 break; 3440 } 3441 } 3442 } 3443 } 3444 3445 template addToSimpledisplayEventLoop() { 3446 import arsd.simpledisplay; 3447 void addToSimpledisplayEventLoop(WebSocket ws, SimpleWindow window) { 3448 3449 void midprocess() { 3450 if(!ws.lowLevelReceive()) { 3451 ws.readyState_ = WebSocket.CLOSED; 3452 WebSocket.unregisterActiveSocket(ws); 3453 return; 3454 } 3455 while(ws.processOnce().populated) {} 3456 } 3457 3458 version(linux) { 3459 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 3460 } else version(Windows) { 3461 auto reader = new WindowsHandleReader(&midprocess, ws.socket.handle); 3462 } else static assert(0, "unsupported OS"); 3463 } 3464 } 3465 3466 3467 /* copy/paste from cgi.d */ 3468 public { 3469 enum WebSocketOpcode : ubyte { 3470 continuation = 0, 3471 text = 1, 3472 binary = 2, 3473 // 3, 4, 5, 6, 7 RESERVED 3474 close = 8, 3475 ping = 9, 3476 pong = 10, 3477 // 11,12,13,14,15 RESERVED 3478 } 3479 3480 public struct WebSocketFrame { 3481 private bool populated; 3482 bool fin; 3483 bool rsv1; 3484 bool rsv2; 3485 bool rsv3; 3486 WebSocketOpcode opcode; // 4 bits 3487 bool masked; 3488 ubyte lengthIndicator; // don't set this when building one to send 3489 ulong realLength; // don't use when sending 3490 ubyte[4] maskingKey; // don't set this when sending 3491 ubyte[] data; 3492 3493 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { 3494 WebSocketFrame msg; 3495 msg.fin = true; 3496 msg.opcode = opcode; 3497 msg.data = cast(ubyte[]) data; 3498 3499 return msg; 3500 } 3501 3502 private void send(scope void delegate(ubyte[]) llsend) { 3503 ubyte[64] headerScratch; 3504 int headerScratchPos = 0; 3505 3506 realLength = data.length; 3507 3508 { 3509 ubyte b1; 3510 b1 |= cast(ubyte) opcode; 3511 b1 |= rsv3 ? (1 << 4) : 0; 3512 b1 |= rsv2 ? (1 << 5) : 0; 3513 b1 |= rsv1 ? (1 << 6) : 0; 3514 b1 |= fin ? (1 << 7) : 0; 3515 3516 headerScratch[0] = b1; 3517 headerScratchPos++; 3518 } 3519 3520 { 3521 headerScratchPos++; // we'll set header[1] at the end of this 3522 auto rlc = realLength; 3523 ubyte b2; 3524 b2 |= masked ? (1 << 7) : 0; 3525 3526 assert(headerScratchPos == 2); 3527 3528 if(realLength > 65535) { 3529 // use 64 bit length 3530 b2 |= 0x7f; 3531 3532 // FIXME: double check endinaness 3533 foreach(i; 0 .. 8) { 3534 headerScratch[2 + 7 - i] = rlc & 0x0ff; 3535 rlc >>>= 8; 3536 } 3537 3538 headerScratchPos += 8; 3539 } else if(realLength > 125) { 3540 // use 16 bit length 3541 b2 |= 0x7e; 3542 3543 // FIXME: double check endinaness 3544 foreach(i; 0 .. 2) { 3545 headerScratch[2 + 1 - i] = rlc & 0x0ff; 3546 rlc >>>= 8; 3547 } 3548 3549 headerScratchPos += 2; 3550 } else { 3551 // use 7 bit length 3552 b2 |= realLength & 0b_0111_1111; 3553 } 3554 3555 headerScratch[1] = b2; 3556 } 3557 3558 //assert(!masked, "masking key not properly implemented"); 3559 if(masked) { 3560 // FIXME: randomize this 3561 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 3562 headerScratchPos += 4; 3563 3564 // we'll just mask it in place... 3565 int keyIdx = 0; 3566 foreach(i; 0 .. data.length) { 3567 data[i] = data[i] ^ maskingKey[keyIdx]; 3568 if(keyIdx == 3) 3569 keyIdx = 0; 3570 else 3571 keyIdx++; 3572 } 3573 } 3574 3575 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 3576 llsend(headerScratch[0 .. headerScratchPos]); 3577 llsend(data); 3578 } 3579 3580 static WebSocketFrame read(ref ubyte[] d) { 3581 WebSocketFrame msg; 3582 3583 auto orig = d; 3584 3585 WebSocketFrame needsMoreData() { 3586 d = orig; 3587 return WebSocketFrame.init; 3588 } 3589 3590 if(d.length < 2) 3591 return needsMoreData(); 3592 3593 ubyte b = d[0]; 3594 3595 msg.populated = true; 3596 3597 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 3598 b >>= 4; 3599 msg.rsv3 = b & 0x01; 3600 b >>= 1; 3601 msg.rsv2 = b & 0x01; 3602 b >>= 1; 3603 msg.rsv1 = b & 0x01; 3604 b >>= 1; 3605 msg.fin = b & 0x01; 3606 3607 b = d[1]; 3608 msg.masked = (b & 0b1000_0000) ? true : false; 3609 msg.lengthIndicator = b & 0b0111_1111; 3610 3611 d = d[2 .. $]; 3612 3613 if(msg.lengthIndicator == 0x7e) { 3614 // 16 bit length 3615 msg.realLength = 0; 3616 3617 if(d.length < 2) return needsMoreData(); 3618 3619 foreach(i; 0 .. 2) { 3620 msg.realLength |= d[0] << ((1-i) * 8); 3621 d = d[1 .. $]; 3622 } 3623 } else if(msg.lengthIndicator == 0x7f) { 3624 // 64 bit length 3625 msg.realLength = 0; 3626 3627 if(d.length < 8) return needsMoreData(); 3628 3629 foreach(i; 0 .. 8) { 3630 msg.realLength |= d[0] << ((7-i) * 8); 3631 d = d[1 .. $]; 3632 } 3633 } else { 3634 // 7 bit length 3635 msg.realLength = msg.lengthIndicator; 3636 } 3637 3638 if(msg.masked) { 3639 3640 if(d.length < 4) return needsMoreData(); 3641 3642 msg.maskingKey = d[0 .. 4]; 3643 d = d[4 .. $]; 3644 } 3645 3646 if(msg.realLength > d.length) { 3647 return needsMoreData(); 3648 } 3649 3650 msg.data = d[0 .. cast(size_t) msg.realLength]; 3651 d = d[cast(size_t) msg.realLength .. $]; 3652 3653 return msg; 3654 } 3655 3656 void unmaskInPlace() { 3657 if(this.masked) { 3658 int keyIdx = 0; 3659 foreach(i; 0 .. this.data.length) { 3660 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 3661 if(keyIdx == 3) 3662 keyIdx = 0; 3663 else 3664 keyIdx++; 3665 } 3666 } 3667 } 3668 3669 char[] textData() { 3670 return cast(char[]) data; 3671 } 3672 } 3673 } 3674 3675 /+ 3676 so the url params are arguments. it knows the request 3677 internally. other params are properties on the req 3678 3679 names may have different paths... those will just add ForSomething i think. 3680 3681 auto req = api.listMergeRequests 3682 req.page = 10; 3683 3684 or 3685 req.page(1) 3686 .bar("foo") 3687 3688 req.execute(); 3689 3690 3691 everything in the response is nullable access through the 3692 dynamic object, just with property getters there. need to make 3693 it static generated tho 3694 3695 other messages may be: isPresent and getDynamic 3696 3697 3698 AND/OR what about doing it like the rails objects 3699 3700 BroadcastMessage.get(4) 3701 // various properties 3702 3703 // it lists what you updated 3704 3705 BroadcastMessage.foo().bar().put(5) 3706 +/