1 // Written in the D programming language. 2 3 /** 4 Networking client functionality as provided by $(HTTP curl.haxx.se/libcurl, 5 libcurl). The libcurl library must be installed on the system in order to use 6 this module. 7 8 $(SCRIPT inhibitQuickIndex = 1;) 9 10 $(DIVC quickindex, 11 $(BOOKTABLE , 12 $(TR $(TH Category) $(TH Functions) 13 ) 14 $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get) 15 $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace) 16 $(MYREF connect) $(MYREF byLine) $(MYREF byChunk) 17 $(MYREF byLineAsync) $(MYREF byChunkAsync) ) 18 ) 19 $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF 20 SMTP) ) 21 ) 22 ) 23 ) 24 25 Note: 26 You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"]) 27 to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB). 28 29 Windows x86 note: 30 A DMD compatible libcurl static library can be downloaded from the dlang.org 31 $(LINK2 http://downloads.dlang.org/other/index.html, download archive page). 32 33 This module is not available for iOS, tvOS or watchOS. 34 35 Compared to using libcurl directly this module allows simpler client code for 36 common uses, requires no unsafe operations, and integrates better with the rest 37 of the language. Futhermore it provides $(MREF_ALTTEXT range, std,range) 38 access to protocols supported by libcurl both synchronously and asynchronously. 39 40 A high level and a low level API are available. The high level API is built 41 entirely on top of the low level one. 42 43 The high level API is for commonly used functionality such as HTTP/FTP get. The 44 $(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous 45 $(MREF_ALTTEXT range, std,range) that performs the request in another 46 thread while handling a line/chunk in the current thread. 47 48 The low level API allows for streaming and other advanced features. 49 50 $(BOOKTABLE Cheat Sheet, 51 $(TR $(TH Function Name) $(TH Description) 52 ) 53 $(LEADINGROW High level) 54 $(TR $(TDNW $(LREF download)) $(TD $(D 55 download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file")) 56 downloads file from URL to file system.) 57 ) 58 $(TR $(TDNW $(LREF upload)) $(TD $(D 59 upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");) 60 uploads file from file system to URL.) 61 ) 62 $(TR $(TDNW $(LREF get)) $(TD $(D 63 get("dlang.org")) returns a char[] containing the dlang.org web page.) 64 ) 65 $(TR $(TDNW $(LREF put)) $(TD $(D 66 put("dlang.org", "Hi")) returns a char[] containing 67 the dlang.org web page. after a HTTP PUT of "hi") 68 ) 69 $(TR $(TDNW $(LREF post)) $(TD $(D 70 post("dlang.org", "Hi")) returns a char[] containing 71 the dlang.org web page. after a HTTP POST of "hi") 72 ) 73 $(TR $(TDNW $(LREF byLine)) $(TD $(D 74 byLine("dlang.org")) returns a range of char[] containing the 75 dlang.org web page.) 76 ) 77 $(TR $(TDNW $(LREF byChunk)) $(TD $(D 78 byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the 79 dlang.org web page.) 80 ) 81 $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D 82 byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web 83 page asynchronously.) 84 ) 85 $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D 86 byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the 87 dlang.org web page asynchronously.) 88 ) 89 $(LEADINGROW Low level 90 ) 91 $(TR $(TDNW $(LREF HTTP)) $(TD `HTTP` struct for advanced usage)) 92 $(TR $(TDNW $(LREF FTP)) $(TD `FTP` struct for advanced usage)) 93 $(TR $(TDNW $(LREF SMTP)) $(TD `SMTP` struct for advanced usage)) 94 ) 95 96 97 Example: 98 --- 99 import std.net.curl, std.stdio; 100 101 // Return a char[] containing the content specified by a URL 102 auto content = get("dlang.org"); 103 104 // Post data and return a char[] containing the content specified by a URL 105 auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]); 106 107 // Get content of file from ftp server 108 auto content = get("ftp.digitalmars.com/sieve.ds"); 109 110 // Post and print out content line by line. The request is done in another thread. 111 foreach (line; byLineAsync("dlang.org", "Post data")) 112 writeln(line); 113 114 // Get using a line range and proxy settings 115 auto client = HTTP(); 116 client.proxy = "1.2.3.4"; 117 foreach (line; byLine("dlang.org", client)) 118 writeln(line); 119 --- 120 121 For more control than the high level functions provide, use the low level API: 122 123 Example: 124 --- 125 import std.net.curl, std.stdio; 126 127 // GET with custom data receivers 128 auto http = HTTP("dlang.org"); 129 http.onReceiveHeader = 130 (in char[] key, in char[] value) { writeln(key, ": ", value); }; 131 http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 132 http.perform(); 133 --- 134 135 First, an instance of the reference-counted HTTP struct is created. Then the 136 custom delegates are set. These will be called whenever the HTTP instance 137 receives a header and a data buffer, respectively. In this simple example, the 138 headers are written to stdout and the data is ignored. If the request should be 139 stopped before it has finished then return something less than data.length from 140 the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more 141 information. Finally the HTTP request is effected by calling perform(), which is 142 synchronous. 143 144 Source: $(PHOBOSSRC std/net/curl.d) 145 146 Copyright: Copyright Jonas Drewsen 2011-2012 147 License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 148 Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao. 149 150 Credits: The functionally is based on $(HTTP curl.haxx.se/libcurl, libcurl). 151 LibCurl is licensed under an MIT/X derivative license. 152 */ 153 /* 154 Copyright Jonas Drewsen 2011 - 2012. 155 Distributed under the Boost Software License, Version 1.0. 156 (See accompanying file LICENSE_1_0.txt or copy at 157 http://www.boost.org/LICENSE_1_0.txt) 158 */ 159 module std.net.curl; 160 161 public import etc.c.curl : CurlOption; 162 import core.time : dur; 163 import etc.c.curl : CURLcode; 164 import std.range.primitives; 165 import std.encoding : EncodingScheme; 166 import std.traits : isSomeChar; 167 import std.typecons : Flag, Yes, No, Tuple; 168 169 version (iOS) 170 version = iOSDerived; 171 else version (TVOS) 172 version = iOSDerived; 173 else version (WatchOS) 174 version = iOSDerived; 175 176 version (iOSDerived) {} 177 else: 178 179 version (StdUnittest) 180 { 181 import std.socket : Socket, SocketShutdown; 182 183 private struct TestServer 184 { 185 import std.concurrency : Tid; 186 187 import std.socket : Socket, TcpSocket; 188 189 string addr() { return _addr; } 190 191 void handle(void function(Socket s) dg) 192 { 193 import std.concurrency : send; 194 tid.send(dg); 195 } 196 197 private: 198 string _addr; 199 Tid tid; 200 TcpSocket sock; 201 202 static void loop(shared TcpSocket listener) 203 { 204 import std.concurrency : OwnerTerminated, receiveOnly; 205 import std.stdio : stderr; 206 207 try while (true) 208 { 209 void function(Socket) handler = void; 210 try 211 handler = receiveOnly!(typeof(handler)); 212 catch (OwnerTerminated) 213 return; 214 handler((cast() listener).accept); 215 } 216 catch (Throwable e) 217 { 218 // https://issues.dlang.org/show_bug.cgi?id=7018 219 stderr.writeln(e); 220 } 221 } 222 } 223 224 private TestServer startServer() 225 { 226 import std.concurrency : spawn; 227 import std.socket : INADDR_LOOPBACK, InternetAddress, TcpSocket; 228 229 tlsInit = true; 230 auto sock = new TcpSocket; 231 sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY)); 232 sock.listen(1); 233 auto addr = sock.localAddress.toString(); 234 auto tid = spawn(&TestServer.loop, cast(shared) sock); 235 return TestServer(addr, tid, sock); 236 } 237 238 /** Test server */ 239 __gshared TestServer server; 240 /** Thread-local storage init */ 241 bool tlsInit; 242 243 private ref TestServer testServer() 244 { 245 import std.concurrency : initOnce; 246 return initOnce!server(startServer()); 247 } 248 249 static ~this() 250 { 251 // terminate server from a thread local dtor of the thread that started it, 252 // because thread_joinall is called before shared module dtors 253 if (tlsInit && server.sock) 254 { 255 server.sock.shutdown(SocketShutdown.RECEIVE); 256 server.sock.close(); 257 } 258 } 259 260 private struct Request(T) 261 { 262 string hdrs; 263 immutable(T)[] bdy; 264 } 265 266 private Request!T recvReq(T=char)(Socket s) 267 { 268 import std.algorithm.comparison : min; 269 import std.algorithm.searching : find, canFind; 270 import std.conv : to; 271 import std.regex : ctRegex, matchFirst; 272 273 ubyte[1024] tmp=void; 274 ubyte[] buf; 275 276 while (true) 277 { 278 auto nbytes = s.receive(tmp[]); 279 assert(nbytes >= 0); 280 281 immutable beg = buf.length > 3 ? buf.length - 3 : 0; 282 buf ~= tmp[0 .. nbytes]; 283 auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n"); 284 if (bdy.empty) 285 continue; 286 287 auto hdrs = cast(string) buf[0 .. $ - bdy.length]; 288 bdy.popFrontN(4); 289 // no support for chunked transfer-encoding 290 if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i"))) 291 { 292 import std.uni : asUpperCase; 293 if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE")) 294 s.send(httpContinue); 295 296 size_t remain = m.captures[1].to!size_t - bdy.length; 297 while (remain) 298 { 299 nbytes = s.receive(tmp[0 .. min(remain, $)]); 300 assert(nbytes >= 0); 301 buf ~= tmp[0 .. nbytes]; 302 remain -= nbytes; 303 } 304 } 305 else 306 { 307 assert(bdy.empty); 308 } 309 bdy = buf[hdrs.length + 4 .. $]; 310 return typeof(return)(hdrs, cast(immutable(T)[])bdy); 311 } 312 } 313 314 private string httpOK(string msg) 315 { 316 import std.conv : to; 317 318 return "HTTP/1.1 200 OK\r\n"~ 319 "Content-Type: text/plain\r\n"~ 320 "Content-Length: "~msg.length.to!string~"\r\n"~ 321 "\r\n"~ 322 msg; 323 } 324 325 private string httpOK() 326 { 327 return "HTTP/1.1 200 OK\r\n"~ 328 "Content-Length: 0\r\n"~ 329 "\r\n"; 330 } 331 332 private string httpNotFound() 333 { 334 return "HTTP/1.1 404 Not Found\r\n"~ 335 "Content-Length: 0\r\n"~ 336 "\r\n"; 337 } 338 339 private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n"; 340 } 341 version (StdDdoc) import std.stdio; 342 343 // Default data timeout for Protocols 344 private enum _defaultDataTimeout = dur!"minutes"(2); 345 346 /** 347 Macros: 348 349 CALLBACK_PARAMS = $(TABLE , 350 $(DDOC_PARAM_ROW 351 $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal)) 352 $(DDOC_PARAM_DESC total bytes to download) 353 ) 354 $(DDOC_PARAM_ROW 355 $(DDOC_PARAM_ID $(DDOC_PARAM dlNow)) 356 $(DDOC_PARAM_DESC currently downloaded bytes) 357 ) 358 $(DDOC_PARAM_ROW 359 $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal)) 360 $(DDOC_PARAM_DESC total bytes to upload) 361 ) 362 $(DDOC_PARAM_ROW 363 $(DDOC_PARAM_ID $(DDOC_PARAM ulNow)) 364 $(DDOC_PARAM_DESC currently uploaded bytes) 365 ) 366 ) 367 */ 368 369 /** Connection type used when the URL should be used to auto detect the protocol. 370 * 371 * This struct is used as placeholder for the connection parameter when calling 372 * the high level API and the connection type (HTTP/FTP) should be guessed by 373 * inspecting the URL parameter. 374 * 375 * The rules for guessing the protocol are: 376 * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed. 377 * 2, HTTP connection otherwise. 378 * 379 * Example: 380 * --- 381 * import std.net.curl; 382 * // Two requests below will do the same. 383 * char[] content; 384 * 385 * // Explicit connection provided 386 * content = get!HTTP("dlang.org"); 387 * 388 * // Guess connection type by looking at the URL 389 * content = get!AutoProtocol("ftp://foo.com/file"); 390 * // and since AutoProtocol is default this is the same as 391 * content = get("ftp://foo.com/file"); 392 * // and will end up detecting FTP from the url and be the same as 393 * content = get!FTP("ftp://foo.com/file"); 394 * --- 395 */ 396 struct AutoProtocol { } 397 398 // Returns true if the url points to an FTP resource 399 private bool isFTPUrl(const(char)[] url) 400 { 401 import std.algorithm.searching : startsWith; 402 import std.uni : toLower; 403 404 return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0; 405 } 406 407 // Is true if the Conn type is a valid Curl Connection type. 408 private template isCurlConn(Conn) 409 { 410 enum auto isCurlConn = is(Conn : HTTP) || 411 is(Conn : FTP) || is(Conn : AutoProtocol); 412 } 413 414 /** HTTP/FTP download to local file system. 415 * 416 * Params: 417 * url = resource to download 418 * saveToPath = path to store the downloaded content on local disk 419 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 420 * guess connection type and create a new instance for this call only. 421 * 422 * Example: 423 * ---- 424 * import std.net.curl; 425 * download("https://httpbin.org/get", "/tmp/downloaded-http-file"); 426 * ---- 427 */ 428 void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn()) 429 if (isCurlConn!Conn) 430 { 431 static if (is(Conn : HTTP) || is(Conn : FTP)) 432 { 433 import std.stdio : File; 434 conn.url = url; 435 auto f = File(saveToPath, "wb"); 436 conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; }; 437 conn.perform(); 438 } 439 else 440 { 441 if (isFTPUrl(url)) 442 return download!FTP(url, saveToPath, FTP()); 443 else 444 return download!HTTP(url, saveToPath, HTTP()); 445 } 446 } 447 448 @system unittest 449 { 450 import std.algorithm.searching : canFind; 451 static import std.file; 452 453 foreach (host; [testServer.addr, "http://"~testServer.addr]) 454 { 455 testServer.handle((s) { 456 assert(s.recvReq.hdrs.canFind("GET /")); 457 s.send(httpOK("Hello world")); 458 }); 459 auto fn = std.file.deleteme; 460 scope (exit) 461 { 462 if (std.file.exists(fn)) 463 std.file.remove(fn); 464 } 465 download(host, fn); 466 assert(std.file.readText(fn) == "Hello world"); 467 } 468 } 469 470 /** Upload file from local files system using the HTTP or FTP protocol. 471 * 472 * Params: 473 * loadFromPath = path load data from local disk. 474 * url = resource to upload to 475 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 476 * guess connection type and create a new instance for this call only. 477 * 478 * Example: 479 * ---- 480 * import std.net.curl; 481 * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds"); 482 * upload("/tmp/downloaded-http-file", "https://httpbin.org/post"); 483 * ---- 484 */ 485 void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn()) 486 if (isCurlConn!Conn) 487 { 488 static if (is(Conn : HTTP)) 489 { 490 conn.url = url; 491 conn.method = HTTP.Method.put; 492 } 493 else static if (is(Conn : FTP)) 494 { 495 conn.url = url; 496 conn.handle.set(CurlOption.upload, 1L); 497 } 498 else 499 { 500 if (isFTPUrl(url)) 501 return upload!FTP(loadFromPath, url, FTP()); 502 else 503 return upload!HTTP(loadFromPath, url, HTTP()); 504 } 505 506 static if (is(Conn : HTTP) || is(Conn : FTP)) 507 { 508 import std.stdio : File; 509 auto f = File(loadFromPath, "rb"); 510 conn.onSend = buf => f.rawRead(buf).length; 511 immutable sz = f.size; 512 if (sz != ulong.max) 513 conn.contentLength = sz; 514 conn.perform(); 515 } 516 } 517 518 @system unittest 519 { 520 import std.algorithm.searching : canFind; 521 static import std.file; 522 523 foreach (host; [testServer.addr, "http://"~testServer.addr]) 524 { 525 auto fn = std.file.deleteme; 526 scope (exit) 527 { 528 if (std.file.exists(fn)) 529 std.file.remove(fn); 530 } 531 std.file.write(fn, "upload data\n"); 532 testServer.handle((s) { 533 auto req = s.recvReq; 534 assert(req.hdrs.canFind("PUT /path")); 535 assert(req.bdy.canFind("upload data")); 536 s.send(httpOK()); 537 }); 538 upload(fn, host ~ "/path"); 539 } 540 } 541 542 /** HTTP/FTP get content. 543 * 544 * Params: 545 * url = resource to get 546 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 547 * guess connection type and create a new instance for this call only. 548 * 549 * The template parameter `T` specifies the type to return. Possible values 550 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 551 * for `char`, content will be converted from the connection character set 552 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 553 * by default) to UTF-8. 554 * 555 * Example: 556 * ---- 557 * import std.net.curl; 558 * auto content = get("https://httpbin.org/get"); 559 * ---- 560 * 561 * Returns: 562 * A T[] range containing the content of the resource pointed to by the URL. 563 * 564 * Throws: 565 * 566 * `CurlException` on error. 567 * 568 * See_Also: $(LREF HTTP.Method) 569 */ 570 T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn()) 571 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 572 { 573 static if (is(Conn : HTTP)) 574 { 575 conn.method = HTTP.Method.get; 576 return _basicHTTP!(T)(url, "", conn); 577 578 } 579 else static if (is(Conn : FTP)) 580 { 581 return _basicFTP!(T)(url, "", conn); 582 } 583 else 584 { 585 if (isFTPUrl(url)) 586 return get!(FTP,T)(url, FTP()); 587 else 588 return get!(HTTP,T)(url, HTTP()); 589 } 590 } 591 592 @system unittest 593 { 594 import std.algorithm.searching : canFind; 595 596 foreach (host; [testServer.addr, "http://"~testServer.addr]) 597 { 598 testServer.handle((s) { 599 assert(s.recvReq.hdrs.canFind("GET /path")); 600 s.send(httpOK("GETRESPONSE")); 601 }); 602 auto res = get(host ~ "/path"); 603 assert(res == "GETRESPONSE"); 604 } 605 } 606 607 608 /** HTTP post content. 609 * 610 * Params: 611 * url = resource to post to 612 * postDict = data to send as the body of the request. An associative array 613 * of `string` is accepted and will be encoded using 614 * www-form-urlencoding 615 * postData = data to send as the body of the request. An array 616 * of an arbitrary type is accepted and will be cast to ubyte[] 617 * before sending it. 618 * conn = HTTP connection to use 619 * T = The template parameter `T` specifies the type to return. Possible values 620 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 621 * for `char`, content will be converted from the connection character set 622 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 623 * by default) to UTF-8. 624 * 625 * Examples: 626 * ---- 627 * import std.net.curl; 628 * 629 * auto content1 = post("https://httpbin.org/post", ["name1" : "value1", "name2" : "value2"]); 630 * auto content2 = post("https://httpbin.org/post", [1,2,3,4]); 631 * ---- 632 * 633 * Returns: 634 * A T[] range containing the content of the resource pointed to by the URL. 635 * 636 * See_Also: $(LREF HTTP.Method) 637 */ 638 T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP()) 639 if (is(T == char) || is(T == ubyte)) 640 { 641 conn.method = HTTP.Method.post; 642 return _basicHTTP!(T)(url, postData, conn); 643 } 644 645 @system unittest 646 { 647 import std.algorithm.searching : canFind; 648 649 foreach (host; [testServer.addr, "http://"~testServer.addr]) 650 { 651 testServer.handle((s) { 652 auto req = s.recvReq; 653 assert(req.hdrs.canFind("POST /path")); 654 assert(req.bdy.canFind("POSTBODY")); 655 s.send(httpOK("POSTRESPONSE")); 656 }); 657 auto res = post(host ~ "/path", "POSTBODY"); 658 assert(res == "POSTRESPONSE"); 659 } 660 } 661 662 @system unittest 663 { 664 import std.algorithm.searching : canFind; 665 666 auto data = new ubyte[](256); 667 foreach (i, ref ub; data) 668 ub = cast(ubyte) i; 669 670 testServer.handle((s) { 671 auto req = s.recvReq!ubyte; 672 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 673 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 674 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 675 }); 676 auto res = post!ubyte(testServer.addr, data); 677 assert(res == cast(ubyte[])[17, 27, 35, 41]); 678 } 679 680 /// ditto 681 T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP()) 682 if (is(T == char) || is(T == ubyte)) 683 { 684 import std.uri : urlEncode; 685 686 return post!T(url, urlEncode(postDict), conn); 687 } 688 689 @system unittest 690 { 691 import std.algorithm.searching : canFind; 692 import std.meta : AliasSeq; 693 694 static immutable expected = ["name1=value1&name2=value2", "name2=value2&name1=value1"]; 695 696 foreach (host; [testServer.addr, "http://" ~ testServer.addr]) 697 { 698 foreach (T; AliasSeq!(char, ubyte)) 699 { 700 testServer.handle((s) { 701 auto req = s.recvReq!char; 702 s.send(httpOK(req.bdy)); 703 }); 704 auto res = post!T(host ~ "/path", ["name1" : "value1", "name2" : "value2"]); 705 assert(canFind(expected, res)); 706 } 707 } 708 } 709 710 /** HTTP/FTP put content. 711 * 712 * Params: 713 * url = resource to put 714 * putData = data to send as the body of the request. An array 715 * of an arbitrary type is accepted and will be cast to ubyte[] 716 * before sending it. 717 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 718 * guess connection type and create a new instance for this call only. 719 * 720 * The template parameter `T` specifies the type to return. Possible values 721 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 722 * for `char`, content will be converted from the connection character set 723 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 724 * by default) to UTF-8. 725 * 726 * Example: 727 * ---- 728 * import std.net.curl; 729 * auto content = put("https://httpbin.org/put", 730 * "Putting this data"); 731 * ---- 732 * 733 * Returns: 734 * A T[] range containing the content of the resource pointed to by the URL. 735 * 736 * See_Also: $(LREF HTTP.Method) 737 */ 738 T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData, 739 Conn conn = Conn()) 740 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 741 { 742 static if (is(Conn : HTTP)) 743 { 744 conn.method = HTTP.Method.put; 745 return _basicHTTP!(T)(url, putData, conn); 746 } 747 else static if (is(Conn : FTP)) 748 { 749 return _basicFTP!(T)(url, putData, conn); 750 } 751 else 752 { 753 if (isFTPUrl(url)) 754 return put!(FTP,T)(url, putData, FTP()); 755 else 756 return put!(HTTP,T)(url, putData, HTTP()); 757 } 758 } 759 760 @system unittest 761 { 762 import std.algorithm.searching : canFind; 763 764 foreach (host; [testServer.addr, "http://"~testServer.addr]) 765 { 766 testServer.handle((s) { 767 auto req = s.recvReq; 768 assert(req.hdrs.canFind("PUT /path")); 769 assert(req.bdy.canFind("PUTBODY")); 770 s.send(httpOK("PUTRESPONSE")); 771 }); 772 auto res = put(host ~ "/path", "PUTBODY"); 773 assert(res == "PUTRESPONSE"); 774 } 775 } 776 777 778 /** HTTP/FTP delete content. 779 * 780 * Params: 781 * url = resource to delete 782 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 783 * guess connection type and create a new instance for this call only. 784 * 785 * Example: 786 * ---- 787 * import std.net.curl; 788 * del("https://httpbin.org/delete"); 789 * ---- 790 * 791 * See_Also: $(LREF HTTP.Method) 792 */ 793 void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn()) 794 if (isCurlConn!Conn) 795 { 796 static if (is(Conn : HTTP)) 797 { 798 conn.method = HTTP.Method.del; 799 _basicHTTP!char(url, cast(void[]) null, conn); 800 } 801 else static if (is(Conn : FTP)) 802 { 803 import std.algorithm.searching : findSplitAfter; 804 import std.conv : text; 805 import std.exception : enforce; 806 807 auto trimmed = url.findSplitAfter("ftp://")[1]; 808 auto t = trimmed.findSplitAfter("/"); 809 enum minDomainNameLength = 3; 810 enforce!CurlException(t[0].length > minDomainNameLength, 811 text("Invalid FTP URL for delete ", url)); 812 conn.url = t[0]; 813 814 enforce!CurlException(!t[1].empty, 815 text("No filename specified to delete for URL ", url)); 816 conn.addCommand("DELE " ~ t[1]); 817 conn.perform(); 818 } 819 else 820 { 821 if (isFTPUrl(url)) 822 return del!FTP(url, FTP()); 823 else 824 return del!HTTP(url, HTTP()); 825 } 826 } 827 828 @system unittest 829 { 830 import std.algorithm.searching : canFind; 831 832 foreach (host; [testServer.addr, "http://"~testServer.addr]) 833 { 834 testServer.handle((s) { 835 auto req = s.recvReq; 836 assert(req.hdrs.canFind("DELETE /path")); 837 s.send(httpOK()); 838 }); 839 del(host ~ "/path"); 840 } 841 } 842 843 844 /** HTTP options request. 845 * 846 * Params: 847 * url = resource make a option call to 848 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 849 * guess connection type and create a new instance for this call only. 850 * 851 * The template parameter `T` specifies the type to return. Possible values 852 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 853 * 854 * Example: 855 * ---- 856 * import std.net.curl; 857 * auto http = HTTP(); 858 * options("https://httpbin.org/headers", http); 859 * writeln("Allow set to " ~ http.responseHeaders["Allow"]); 860 * ---- 861 * 862 * Returns: 863 * A T[] range containing the options of the resource pointed to by the URL. 864 * 865 * See_Also: $(LREF HTTP.Method) 866 */ 867 T[] options(T = char)(const(char)[] url, HTTP conn = HTTP()) 868 if (is(T == char) || is(T == ubyte)) 869 { 870 conn.method = HTTP.Method.options; 871 return _basicHTTP!(T)(url, null, conn); 872 } 873 874 @system unittest 875 { 876 import std.algorithm.searching : canFind; 877 878 testServer.handle((s) { 879 auto req = s.recvReq; 880 assert(req.hdrs.canFind("OPTIONS /path")); 881 s.send(httpOK("OPTIONSRESPONSE")); 882 }); 883 auto res = options(testServer.addr ~ "/path"); 884 assert(res == "OPTIONSRESPONSE"); 885 } 886 887 888 /** HTTP trace request. 889 * 890 * Params: 891 * url = resource make a trace call to 892 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 893 * guess connection type and create a new instance for this call only. 894 * 895 * The template parameter `T` specifies the type to return. Possible values 896 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 897 * 898 * Example: 899 * ---- 900 * import std.net.curl; 901 * trace("https://httpbin.org/headers"); 902 * ---- 903 * 904 * Returns: 905 * A T[] range containing the trace info of the resource pointed to by the URL. 906 * 907 * See_Also: $(LREF HTTP.Method) 908 */ 909 T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP()) 910 if (is(T == char) || is(T == ubyte)) 911 { 912 conn.method = HTTP.Method.trace; 913 return _basicHTTP!(T)(url, cast(void[]) null, conn); 914 } 915 916 @system unittest 917 { 918 import std.algorithm.searching : canFind; 919 920 testServer.handle((s) { 921 auto req = s.recvReq; 922 assert(req.hdrs.canFind("TRACE /path")); 923 s.send(httpOK("TRACERESPONSE")); 924 }); 925 auto res = trace(testServer.addr ~ "/path"); 926 assert(res == "TRACERESPONSE"); 927 } 928 929 930 /** HTTP connect request. 931 * 932 * Params: 933 * url = resource make a connect to 934 * conn = HTTP connection to use 935 * 936 * The template parameter `T` specifies the type to return. Possible values 937 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 938 * 939 * Example: 940 * ---- 941 * import std.net.curl; 942 * connect("https://httpbin.org/headers"); 943 * ---- 944 * 945 * Returns: 946 * A T[] range containing the connect info of the resource pointed to by the URL. 947 * 948 * See_Also: $(LREF HTTP.Method) 949 */ 950 T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP()) 951 if (is(T == char) || is(T == ubyte)) 952 { 953 conn.method = HTTP.Method.connect; 954 return _basicHTTP!(T)(url, cast(void[]) null, conn); 955 } 956 957 @system unittest 958 { 959 import std.algorithm.searching : canFind; 960 961 testServer.handle((s) { 962 auto req = s.recvReq; 963 assert(req.hdrs.canFind("CONNECT /path")); 964 s.send(httpOK("CONNECTRESPONSE")); 965 }); 966 auto res = connect(testServer.addr ~ "/path"); 967 assert(res == "CONNECTRESPONSE"); 968 } 969 970 971 /** HTTP patch content. 972 * 973 * Params: 974 * url = resource to patch 975 * patchData = data to send as the body of the request. An array 976 * of an arbitrary type is accepted and will be cast to ubyte[] 977 * before sending it. 978 * conn = HTTP connection to use 979 * 980 * The template parameter `T` specifies the type to return. Possible values 981 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 982 * 983 * Example: 984 * ---- 985 * auto http = HTTP(); 986 * http.addRequestHeader("Content-Type", "application/json"); 987 * auto content = patch("https://httpbin.org/patch", `{"title": "Patched Title"}`, http); 988 * ---- 989 * 990 * Returns: 991 * A T[] range containing the content of the resource pointed to by the URL. 992 * 993 * See_Also: $(LREF HTTP.Method) 994 */ 995 T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData, 996 HTTP conn = HTTP()) 997 if (is(T == char) || is(T == ubyte)) 998 { 999 conn.method = HTTP.Method.patch; 1000 return _basicHTTP!(T)(url, patchData, conn); 1001 } 1002 1003 @system unittest 1004 { 1005 import std.algorithm.searching : canFind; 1006 1007 testServer.handle((s) { 1008 auto req = s.recvReq; 1009 assert(req.hdrs.canFind("PATCH /path")); 1010 assert(req.bdy.canFind("PATCHBODY")); 1011 s.send(httpOK("PATCHRESPONSE")); 1012 }); 1013 auto res = patch(testServer.addr ~ "/path", "PATCHBODY"); 1014 assert(res == "PATCHRESPONSE"); 1015 } 1016 1017 1018 /* 1019 * Helper function for the high level interface. 1020 * 1021 * It performs an HTTP request using the client which must have 1022 * been setup correctly before calling this function. 1023 */ 1024 private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client) 1025 { 1026 import std.algorithm.comparison : min; 1027 import std.format : format; 1028 import std.exception : enforce; 1029 import etc.c.curl : CurlSeek, CurlSeekPos; 1030 1031 immutable doSend = sendData !is null && 1032 (client.method == HTTP.Method.post || 1033 client.method == HTTP.Method.put || 1034 client.method == HTTP.Method.patch); 1035 1036 scope (exit) 1037 { 1038 client.onReceiveHeader = null; 1039 client.onReceiveStatusLine = null; 1040 client.onReceive = null; 1041 1042 if (doSend) 1043 { 1044 client.onSend = null; 1045 client.handle.onSeek = null; 1046 client.contentLength = 0; 1047 } 1048 } 1049 client.url = url; 1050 HTTP.StatusLine statusLine; 1051 import std.array : appender; 1052 auto content = appender!(ubyte[])(); 1053 client.onReceive = (ubyte[] data) 1054 { 1055 content ~= data; 1056 return data.length; 1057 }; 1058 1059 if (doSend) 1060 { 1061 client.contentLength = sendData.length; 1062 auto remainingData = sendData; 1063 client.onSend = delegate size_t(void[] buf) 1064 { 1065 size_t minLen = min(buf.length, remainingData.length); 1066 if (minLen == 0) return 0; 1067 buf[0 .. minLen] = remainingData[0 .. minLen]; 1068 remainingData = remainingData[minLen..$]; 1069 return minLen; 1070 }; 1071 client.handle.onSeek = delegate(long offset, CurlSeekPos mode) 1072 { 1073 switch (mode) 1074 { 1075 case CurlSeekPos.set: 1076 remainingData = sendData[cast(size_t) offset..$]; 1077 return CurlSeek.ok; 1078 default: 1079 // As of curl 7.18.0, libcurl will not pass 1080 // anything other than CurlSeekPos.set. 1081 return CurlSeek.cantseek; 1082 } 1083 }; 1084 } 1085 1086 client.onReceiveHeader = (in char[] key, 1087 in char[] value) 1088 { 1089 if (key == "content-length") 1090 { 1091 import std.conv : to; 1092 content.reserve(value.to!size_t); 1093 } 1094 }; 1095 client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; }; 1096 client.perform(); 1097 enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code, 1098 format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason))); 1099 1100 return _decodeContent!T(content.data, client.p.charset); 1101 } 1102 1103 @system unittest 1104 { 1105 import std.algorithm.searching : canFind; 1106 import std.exception : collectException; 1107 1108 testServer.handle((s) { 1109 auto req = s.recvReq; 1110 assert(req.hdrs.canFind("GET /path")); 1111 s.send(httpNotFound()); 1112 }); 1113 auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path")); 1114 assert(e.msg == "HTTP request returned status code 404 (Not Found)"); 1115 assert(e.status == 404); 1116 } 1117 1118 // Content length must be reset after post 1119 // https://issues.dlang.org/show_bug.cgi?id=14760 1120 @system unittest 1121 { 1122 import std.algorithm.searching : canFind; 1123 1124 testServer.handle((s) { 1125 auto req = s.recvReq; 1126 assert(req.hdrs.canFind("POST /")); 1127 assert(req.bdy.canFind("POSTBODY")); 1128 s.send(httpOK("POSTRESPONSE")); 1129 1130 req = s.recvReq; 1131 assert(req.hdrs.canFind("TRACE /")); 1132 assert(req.bdy.empty); 1133 s.blocking = false; 1134 ubyte[6] buf = void; 1135 assert(s.receive(buf[]) < 0); 1136 s.send(httpOK("TRACERESPONSE")); 1137 }); 1138 auto http = HTTP(); 1139 auto res = post(testServer.addr, "POSTBODY", http); 1140 assert(res == "POSTRESPONSE"); 1141 res = trace(testServer.addr, http); 1142 assert(res == "TRACERESPONSE"); 1143 } 1144 1145 @system unittest // charset detection and transcoding to T 1146 { 1147 testServer.handle((s) { 1148 s.send("HTTP/1.1 200 OK\r\n"~ 1149 "Content-Length: 4\r\n"~ 1150 "Content-Type: text/plain; charset=utf-8\r\n" ~ 1151 "\r\n" ~ 1152 "äbc"); 1153 }); 1154 auto client = HTTP(); 1155 auto result = _basicHTTP!char(testServer.addr, "", client); 1156 assert(result == "äbc"); 1157 1158 testServer.handle((s) { 1159 s.send("HTTP/1.1 200 OK\r\n"~ 1160 "Content-Length: 3\r\n"~ 1161 "Content-Type: text/plain; charset=iso-8859-1\r\n" ~ 1162 "\r\n" ~ 1163 0xE4 ~ "bc"); 1164 }); 1165 client = HTTP(); 1166 result = _basicHTTP!char(testServer.addr, "", client); 1167 assert(result == "äbc"); 1168 } 1169 1170 /* 1171 * Helper function for the high level interface. 1172 * 1173 * It performs an FTP request using the client which must have 1174 * been setup correctly before calling this function. 1175 */ 1176 private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client) 1177 { 1178 import std.algorithm.comparison : min; 1179 1180 scope (exit) 1181 { 1182 client.onReceive = null; 1183 if (!sendData.empty) 1184 client.onSend = null; 1185 } 1186 1187 ubyte[] content; 1188 1189 if (client.encoding.empty) 1190 client.encoding = "ISO-8859-1"; 1191 1192 client.url = url; 1193 client.onReceive = (ubyte[] data) 1194 { 1195 content ~= data; 1196 return data.length; 1197 }; 1198 1199 if (!sendData.empty) 1200 { 1201 client.handle.set(CurlOption.upload, 1L); 1202 client.onSend = delegate size_t(void[] buf) 1203 { 1204 size_t minLen = min(buf.length, sendData.length); 1205 if (minLen == 0) return 0; 1206 buf[0 .. minLen] = sendData[0 .. minLen]; 1207 sendData = sendData[minLen..$]; 1208 return minLen; 1209 }; 1210 } 1211 1212 client.perform(); 1213 1214 return _decodeContent!T(content, client.encoding); 1215 } 1216 1217 /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to 1218 * correct string format 1219 */ 1220 private auto _decodeContent(T)(ubyte[] content, string encoding) 1221 { 1222 static if (is(T == ubyte)) 1223 { 1224 return content; 1225 } 1226 else 1227 { 1228 import std.exception : enforce; 1229 import std.format : format; 1230 1231 // Optimally just return the utf8 encoded content 1232 if (encoding == "UTF-8") 1233 return cast(char[])(content); 1234 1235 // The content has to be re-encoded to utf8 1236 auto scheme = EncodingScheme.create(encoding); 1237 enforce!CurlException(scheme !is null, 1238 format("Unknown encoding '%s'", encoding)); 1239 1240 auto strInfo = decodeString(content, scheme); 1241 enforce!CurlException(strInfo[0] != size_t.max, 1242 format("Invalid encoding sequence for encoding '%s'", 1243 encoding)); 1244 1245 return strInfo[1]; 1246 } 1247 } 1248 1249 alias KeepTerminator = Flag!"keepTerminator"; 1250 /+ 1251 struct ByLineBuffer(Char) 1252 { 1253 bool linePresent; 1254 bool EOF; 1255 Char[] buffer; 1256 ubyte[] decodeRemainder; 1257 1258 bool append(const(ubyte)[] data) 1259 { 1260 byLineBuffer ~= data; 1261 } 1262 1263 @property bool linePresent() 1264 { 1265 return byLinePresent; 1266 } 1267 1268 Char[] get() 1269 { 1270 if (!linePresent) 1271 { 1272 // Decode ubyte[] into Char[] until a Terminator is found. 1273 // If not Terminator is found and EOF is false then raise an 1274 // exception. 1275 } 1276 return byLineBuffer; 1277 } 1278 1279 } 1280 ++/ 1281 /** HTTP/FTP fetch content as a range of lines. 1282 * 1283 * A range of lines is returned when the request is complete. If the method or 1284 * other request properties is to be customized then set the `conn` parameter 1285 * with a HTTP/FTP instance that has these properties set. 1286 * 1287 * Example: 1288 * ---- 1289 * import std.net.curl, std.stdio; 1290 * foreach (line; byLine("dlang.org")) 1291 * writeln(line); 1292 * ---- 1293 * 1294 * Params: 1295 * url = The url to receive content from 1296 * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be 1297 * returned as part of the lines in the range. 1298 * terminator = The character that terminates a line 1299 * conn = The connection to use e.g. HTTP or FTP. 1300 * 1301 * Returns: 1302 * A range of Char[] with the content of the resource pointer to by the URL 1303 */ 1304 auto byLine(Conn = AutoProtocol, Terminator = char, Char = char) 1305 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1306 Terminator terminator = '\n', Conn conn = Conn()) 1307 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1308 { 1309 static struct SyncLineInputRange 1310 { 1311 1312 private Char[] lines; 1313 private Char[] current; 1314 private bool currentValid; 1315 private bool keepTerminator; 1316 private Terminator terminator; 1317 1318 this(Char[] lines, bool kt, Terminator terminator) 1319 { 1320 this.lines = lines; 1321 this.keepTerminator = kt; 1322 this.terminator = terminator; 1323 currentValid = true; 1324 popFront(); 1325 } 1326 1327 @property @safe bool empty() 1328 { 1329 return !currentValid; 1330 } 1331 1332 @property @safe Char[] front() 1333 { 1334 import std.exception : enforce; 1335 enforce!CurlException(currentValid, "Cannot call front() on empty range"); 1336 return current; 1337 } 1338 1339 void popFront() 1340 { 1341 import std.algorithm.searching : findSplitAfter, findSplit; 1342 import std.exception : enforce; 1343 1344 enforce!CurlException(currentValid, "Cannot call popFront() on empty range"); 1345 if (lines.empty) 1346 { 1347 currentValid = false; 1348 return; 1349 } 1350 1351 if (keepTerminator) 1352 { 1353 auto r = findSplitAfter(lines, [ terminator ]); 1354 if (r[0].empty) 1355 { 1356 current = r[1]; 1357 lines = r[0]; 1358 } 1359 else 1360 { 1361 current = r[0]; 1362 lines = r[1]; 1363 } 1364 } 1365 else 1366 { 1367 auto r = findSplit(lines, [ terminator ]); 1368 current = r[0]; 1369 lines = r[2]; 1370 } 1371 } 1372 } 1373 1374 auto result = _getForRange!Char(url, conn); 1375 return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator); 1376 } 1377 1378 @system unittest 1379 { 1380 import std.algorithm.comparison : equal; 1381 1382 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1383 { 1384 testServer.handle((s) { 1385 auto req = s.recvReq; 1386 s.send(httpOK("Line1\nLine2\nLine3")); 1387 }); 1388 assert(byLine(host).equal(["Line1", "Line2", "Line3"])); 1389 } 1390 } 1391 1392 /** HTTP/FTP fetch content as a range of chunks. 1393 * 1394 * A range of chunks is returned when the request is complete. If the method or 1395 * other request properties is to be customized then set the `conn` parameter 1396 * with a HTTP/FTP instance that has these properties set. 1397 * 1398 * Example: 1399 * ---- 1400 * import std.net.curl, std.stdio; 1401 * foreach (chunk; byChunk("dlang.org", 100)) 1402 * writeln(chunk); // chunk is ubyte[100] 1403 * ---- 1404 * 1405 * Params: 1406 * url = The url to receive content from 1407 * chunkSize = The size of each chunk 1408 * conn = The connection to use e.g. HTTP or FTP. 1409 * 1410 * Returns: 1411 * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL 1412 */ 1413 auto byChunk(Conn = AutoProtocol) 1414 (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn()) 1415 if (isCurlConn!(Conn)) 1416 { 1417 static struct SyncChunkInputRange 1418 { 1419 private size_t chunkSize; 1420 private ubyte[] _bytes; 1421 private size_t offset; 1422 1423 this(ubyte[] bytes, size_t chunkSize) 1424 { 1425 this._bytes = bytes; 1426 this.chunkSize = chunkSize; 1427 } 1428 1429 @property @safe auto empty() 1430 { 1431 return offset == _bytes.length; 1432 } 1433 1434 @property ubyte[] front() 1435 { 1436 size_t nextOffset = offset + chunkSize; 1437 if (nextOffset > _bytes.length) nextOffset = _bytes.length; 1438 return _bytes[offset .. nextOffset]; 1439 } 1440 1441 @safe void popFront() 1442 { 1443 offset += chunkSize; 1444 if (offset > _bytes.length) offset = _bytes.length; 1445 } 1446 } 1447 1448 auto result = _getForRange!ubyte(url, conn); 1449 return SyncChunkInputRange(result, chunkSize); 1450 } 1451 1452 @system unittest 1453 { 1454 import std.algorithm.comparison : equal; 1455 1456 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1457 { 1458 testServer.handle((s) { 1459 auto req = s.recvReq; 1460 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1461 }); 1462 assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1463 } 1464 } 1465 1466 private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn) 1467 { 1468 static if (is(Conn : HTTP)) 1469 { 1470 conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method; 1471 return _basicHTTP!(T)(url, null, conn); 1472 } 1473 else static if (is(Conn : FTP)) 1474 { 1475 return _basicFTP!(T)(url, null, conn); 1476 } 1477 else 1478 { 1479 if (isFTPUrl(url)) 1480 return get!(FTP,T)(url, FTP()); 1481 else 1482 return get!(HTTP,T)(url, HTTP()); 1483 } 1484 } 1485 1486 /* 1487 Main thread part of the message passing protocol used for all async 1488 curl protocols. 1489 */ 1490 private mixin template WorkerThreadProtocol(Unit, alias units) 1491 { 1492 import core.time : Duration; 1493 1494 @property bool empty() 1495 { 1496 tryEnsureUnits(); 1497 return state == State.done; 1498 } 1499 1500 @property Unit[] front() 1501 { 1502 import std.format : format; 1503 tryEnsureUnits(); 1504 assert(state == State.gotUnits, 1505 format("Expected %s but got $s", 1506 State.gotUnits, state)); 1507 return units; 1508 } 1509 1510 void popFront() 1511 { 1512 import std.concurrency : send; 1513 import std.format : format; 1514 1515 tryEnsureUnits(); 1516 assert(state == State.gotUnits, 1517 format("Expected %s but got $s", 1518 State.gotUnits, state)); 1519 state = State.needUnits; 1520 // Send to worker thread for buffer reuse 1521 workerTid.send(cast(immutable(Unit)[]) units); 1522 units = null; 1523 } 1524 1525 /** Wait for duration or until data is available and return true if data is 1526 available 1527 */ 1528 bool wait(Duration d) 1529 { 1530 import core.time : dur; 1531 import std.datetime.stopwatch : StopWatch; 1532 import std.concurrency : receiveTimeout; 1533 1534 if (state == State.gotUnits) 1535 return true; 1536 1537 enum noDur = dur!"hnsecs"(0); 1538 StopWatch sw; 1539 sw.start(); 1540 while (state != State.gotUnits && d > noDur) 1541 { 1542 final switch (state) 1543 { 1544 case State.needUnits: 1545 receiveTimeout(d, 1546 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1547 { 1548 if (origin != workerTid) 1549 return false; 1550 units = cast(Unit[]) _data.data; 1551 state = State.gotUnits; 1552 return true; 1553 }, 1554 (Tid origin, CurlMessage!bool f) 1555 { 1556 if (origin != workerTid) 1557 return false; 1558 state = state.done; 1559 return true; 1560 } 1561 ); 1562 break; 1563 case State.gotUnits: return true; 1564 case State.done: 1565 return false; 1566 } 1567 d -= sw.peek(); 1568 sw.reset(); 1569 } 1570 return state == State.gotUnits; 1571 } 1572 1573 enum State 1574 { 1575 needUnits, 1576 gotUnits, 1577 done 1578 } 1579 State state; 1580 1581 void tryEnsureUnits() 1582 { 1583 import std.concurrency : receive; 1584 while (true) 1585 { 1586 final switch (state) 1587 { 1588 case State.needUnits: 1589 receive( 1590 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1591 { 1592 if (origin != workerTid) 1593 return false; 1594 units = cast(Unit[]) _data.data; 1595 state = State.gotUnits; 1596 return true; 1597 }, 1598 (Tid origin, CurlMessage!bool f) 1599 { 1600 if (origin != workerTid) 1601 return false; 1602 state = state.done; 1603 return true; 1604 } 1605 ); 1606 break; 1607 case State.gotUnits: return; 1608 case State.done: 1609 return; 1610 } 1611 } 1612 } 1613 } 1614 1615 /** HTTP/FTP fetch content as a range of lines asynchronously. 1616 * 1617 * A range of lines is returned immediately and the request that fetches the 1618 * lines is performed in another thread. If the method or other request 1619 * properties is to be customized then set the `conn` parameter with a 1620 * HTTP/FTP instance that has these properties set. 1621 * 1622 * If `postData` is non-_null the method will be set to `post` for HTTP 1623 * requests. 1624 * 1625 * The background thread will buffer up to transmitBuffers number of lines 1626 * before it stops receiving data from network. When the main thread reads the 1627 * lines from the range it frees up buffers and allows for the background thread 1628 * to receive more data from the network. 1629 * 1630 * If no data is available and the main thread accesses the range it will block 1631 * until data becomes available. An exception to this is the `wait(Duration)` method on 1632 * the $(LREF LineInputRange). This method will wait at maximum for the 1633 * specified duration and return true if data is available. 1634 * 1635 * Example: 1636 * ---- 1637 * import std.net.curl, std.stdio; 1638 * // Get some pages in the background 1639 * auto range1 = byLineAsync("www.google.com"); 1640 * auto range2 = byLineAsync("www.wikipedia.org"); 1641 * foreach (line; byLineAsync("dlang.org")) 1642 * writeln(line); 1643 * 1644 * // Lines already fetched in the background and ready 1645 * foreach (line; range1) writeln(line); 1646 * foreach (line; range2) writeln(line); 1647 * ---- 1648 * 1649 * ---- 1650 * import std.net.curl, std.stdio; 1651 * // Get a line in a background thread and wait in 1652 * // main thread for 2 seconds for it to arrive. 1653 * auto range3 = byLineAsync("dlang.com"); 1654 * if (range3.wait(dur!"seconds"(2))) 1655 * writeln(range3.front); 1656 * else 1657 * writeln("No line received after 2 seconds!"); 1658 * ---- 1659 * 1660 * Params: 1661 * url = The url to receive content from 1662 * postData = Data to HTTP Post 1663 * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be 1664 * returned as part of the lines in the range. 1665 * terminator = The character that terminates a line 1666 * transmitBuffers = The number of lines buffered asynchronously 1667 * conn = The connection to use e.g. HTTP or FTP. 1668 * 1669 * Returns: 1670 * A range of Char[] with the content of the resource pointer to by the 1671 * URL. 1672 */ 1673 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit) 1674 (const(char)[] url, const(PostUnit)[] postData, 1675 KeepTerminator keepTerminator = No.keepTerminator, 1676 Terminator terminator = '\n', 1677 size_t transmitBuffers = 10, Conn conn = Conn()) 1678 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1679 { 1680 static if (is(Conn : AutoProtocol)) 1681 { 1682 if (isFTPUrl(url)) 1683 return byLineAsync(url, postData, keepTerminator, 1684 terminator, transmitBuffers, FTP()); 1685 else 1686 return byLineAsync(url, postData, keepTerminator, 1687 terminator, transmitBuffers, HTTP()); 1688 } 1689 else 1690 { 1691 import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; 1692 // 50 is just an arbitrary number for now 1693 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1694 auto tid = spawn(&_async!().spawn!(Conn, Char, Terminator)); 1695 tid.send(thisTid); 1696 tid.send(terminator); 1697 tid.send(keepTerminator == Yes.keepTerminator); 1698 1699 _async!().duplicateConnection(url, conn, postData, tid); 1700 1701 return _async!().LineInputRange!Char(tid, transmitBuffers, 1702 Conn.defaultAsyncStringBufferSize); 1703 } 1704 } 1705 1706 /// ditto 1707 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char) 1708 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1709 Terminator terminator = '\n', 1710 size_t transmitBuffers = 10, Conn conn = Conn()) 1711 { 1712 static if (is(Conn : AutoProtocol)) 1713 { 1714 if (isFTPUrl(url)) 1715 return byLineAsync(url, cast(void[]) null, keepTerminator, 1716 terminator, transmitBuffers, FTP()); 1717 else 1718 return byLineAsync(url, cast(void[]) null, keepTerminator, 1719 terminator, transmitBuffers, HTTP()); 1720 } 1721 else 1722 { 1723 return byLineAsync(url, cast(void[]) null, keepTerminator, 1724 terminator, transmitBuffers, conn); 1725 } 1726 } 1727 1728 @system unittest 1729 { 1730 import std.algorithm.comparison : equal; 1731 1732 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1733 { 1734 testServer.handle((s) { 1735 auto req = s.recvReq; 1736 s.send(httpOK("Line1\nLine2\nLine3")); 1737 }); 1738 assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"])); 1739 } 1740 } 1741 1742 /** HTTP/FTP fetch content as a range of chunks asynchronously. 1743 * 1744 * A range of chunks is returned immediately and the request that fetches the 1745 * chunks is performed in another thread. If the method or other request 1746 * properties is to be customized then set the `conn` parameter with a 1747 * HTTP/FTP instance that has these properties set. 1748 * 1749 * If `postData` is non-_null the method will be set to `post` for HTTP 1750 * requests. 1751 * 1752 * The background thread will buffer up to transmitBuffers number of chunks 1753 * before is stops receiving data from network. When the main thread reads the 1754 * chunks from the range it frees up buffers and allows for the background 1755 * thread to receive more data from the network. 1756 * 1757 * If no data is available and the main thread access the range it will block 1758 * until data becomes available. An exception to this is the `wait(Duration)` 1759 * method on the $(LREF ChunkInputRange). This method will wait at maximum for the specified 1760 * duration and return true if data is available. 1761 * 1762 * Example: 1763 * ---- 1764 * import std.net.curl, std.stdio; 1765 * // Get some pages in the background 1766 * auto range1 = byChunkAsync("www.google.com", 100); 1767 * auto range2 = byChunkAsync("www.wikipedia.org"); 1768 * foreach (chunk; byChunkAsync("dlang.org")) 1769 * writeln(chunk); // chunk is ubyte[100] 1770 * 1771 * // Chunks already fetched in the background and ready 1772 * foreach (chunk; range1) writeln(chunk); 1773 * foreach (chunk; range2) writeln(chunk); 1774 * ---- 1775 * 1776 * ---- 1777 * import std.net.curl, std.stdio; 1778 * // Get a line in a background thread and wait in 1779 * // main thread for 2 seconds for it to arrive. 1780 * auto range3 = byChunkAsync("dlang.com", 10); 1781 * if (range3.wait(dur!"seconds"(2))) 1782 * writeln(range3.front); 1783 * else 1784 * writeln("No chunk received after 2 seconds!"); 1785 * ---- 1786 * 1787 * Params: 1788 * url = The url to receive content from 1789 * postData = Data to HTTP Post 1790 * chunkSize = The size of the chunks 1791 * transmitBuffers = The number of chunks buffered asynchronously 1792 * conn = The connection to use e.g. HTTP or FTP. 1793 * 1794 * Returns: 1795 * A range of ubyte[chunkSize] with the content of the resource pointer to by 1796 * the URL. 1797 */ 1798 auto byChunkAsync(Conn = AutoProtocol, PostUnit) 1799 (const(char)[] url, const(PostUnit)[] postData, 1800 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1801 Conn conn = Conn()) 1802 if (isCurlConn!(Conn)) 1803 { 1804 static if (is(Conn : AutoProtocol)) 1805 { 1806 if (isFTPUrl(url)) 1807 return byChunkAsync(url, postData, chunkSize, 1808 transmitBuffers, FTP()); 1809 else 1810 return byChunkAsync(url, postData, chunkSize, 1811 transmitBuffers, HTTP()); 1812 } 1813 else 1814 { 1815 import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; 1816 // 50 is just an arbitrary number for now 1817 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1818 auto tid = spawn(&_async!().spawn!(Conn, ubyte)); 1819 tid.send(thisTid); 1820 1821 _async!().duplicateConnection(url, conn, postData, tid); 1822 1823 return _async!().ChunkInputRange(tid, transmitBuffers, chunkSize); 1824 } 1825 } 1826 1827 /// ditto 1828 auto byChunkAsync(Conn = AutoProtocol) 1829 (const(char)[] url, 1830 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1831 Conn conn = Conn()) 1832 if (isCurlConn!(Conn)) 1833 { 1834 static if (is(Conn : AutoProtocol)) 1835 { 1836 if (isFTPUrl(url)) 1837 return byChunkAsync(url, cast(void[]) null, chunkSize, 1838 transmitBuffers, FTP()); 1839 else 1840 return byChunkAsync(url, cast(void[]) null, chunkSize, 1841 transmitBuffers, HTTP()); 1842 } 1843 else 1844 { 1845 return byChunkAsync(url, cast(void[]) null, chunkSize, 1846 transmitBuffers, conn); 1847 } 1848 } 1849 1850 @system unittest 1851 { 1852 import std.algorithm.comparison : equal; 1853 1854 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1855 { 1856 testServer.handle((s) { 1857 auto req = s.recvReq; 1858 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1859 }); 1860 assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1861 } 1862 } 1863 1864 1865 /* 1866 Mixin template for all supported curl protocols. This is the commom 1867 functionallity such as timeouts and network interface settings. This should 1868 really be in the HTTP/FTP/SMTP structs but the documentation tool does not 1869 support a mixin to put its doc strings where a mixin is done. Therefore docs 1870 in this template is copied into each of HTTP/FTP/SMTP below. 1871 */ 1872 private mixin template Protocol() 1873 { 1874 import etc.c.curl : CurlReadFunc, RawCurlProxy = CurlProxy; 1875 import core.time : Duration; 1876 import std.socket : InternetAddress; 1877 1878 /// Value to return from `onSend`/`onReceive` delegates in order to 1879 /// pause a request 1880 alias requestPause = CurlReadFunc.pause; 1881 1882 /// Value to return from onSend delegate in order to abort a request 1883 alias requestAbort = CurlReadFunc.abort; 1884 1885 static uint defaultAsyncStringBufferSize = 100; 1886 1887 /** 1888 The curl handle used by this connection. 1889 */ 1890 @property ref Curl handle() return 1891 { 1892 return p.curl; 1893 } 1894 1895 /** 1896 True if the instance is stopped. A stopped instance is not usable. 1897 */ 1898 @property bool isStopped() 1899 { 1900 return p.curl.stopped; 1901 } 1902 1903 /// Stop and invalidate this instance. 1904 void shutdown() 1905 { 1906 p.curl.shutdown(); 1907 } 1908 1909 /** Set verbose. 1910 This will print request information to stderr. 1911 */ 1912 @property void verbose(bool on) 1913 { 1914 p.curl.set(CurlOption.verbose, on ? 1L : 0L); 1915 } 1916 1917 // Connection settings 1918 1919 /// Set timeout for activity on connection. 1920 @property void dataTimeout(Duration d) 1921 { 1922 p.curl.set(CurlOption.low_speed_limit, 1); 1923 p.curl.set(CurlOption.low_speed_time, d.total!"seconds"); 1924 } 1925 1926 /** Set maximum time an operation is allowed to take. 1927 This includes dns resolution, connecting, data transfer, etc. 1928 */ 1929 @property void operationTimeout(Duration d) 1930 { 1931 p.curl.set(CurlOption.timeout_ms, d.total!"msecs"); 1932 } 1933 1934 /// Set timeout for connecting. 1935 @property void connectTimeout(Duration d) 1936 { 1937 p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs"); 1938 } 1939 1940 // Network settings 1941 1942 /** Proxy 1943 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 1944 */ 1945 @property void proxy(const(char)[] host) 1946 { 1947 p.curl.set(CurlOption.proxy, host); 1948 } 1949 1950 /** Proxy port 1951 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 1952 */ 1953 @property void proxyPort(ushort port) 1954 { 1955 p.curl.set(CurlOption.proxyport, cast(long) port); 1956 } 1957 1958 /// Type of proxy 1959 alias CurlProxy = RawCurlProxy; 1960 1961 /** Proxy type 1962 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 1963 */ 1964 @property void proxyType(CurlProxy type) 1965 { 1966 p.curl.set(CurlOption.proxytype, cast(long) type); 1967 } 1968 1969 /// DNS lookup timeout. 1970 @property void dnsTimeout(Duration d) 1971 { 1972 p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs"); 1973 } 1974 1975 /** 1976 * The network interface to use in form of the the IP of the interface. 1977 * 1978 * Example: 1979 * ---- 1980 * theprotocol.netInterface = "192.168.1.32"; 1981 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 1982 * ---- 1983 * 1984 * See: $(REF InternetAddress, std,socket) 1985 */ 1986 @property void netInterface(const(char)[] i) 1987 { 1988 p.curl.set(CurlOption.intrface, i); 1989 } 1990 1991 /// ditto 1992 @property void netInterface(const(ubyte)[4] i) 1993 { 1994 import std.format : format; 1995 const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]); 1996 netInterface = str; 1997 } 1998 1999 /// ditto 2000 @property void netInterface(InternetAddress i) 2001 { 2002 netInterface = i.toAddrString(); 2003 } 2004 2005 /** 2006 Set the local outgoing port to use. 2007 Params: 2008 port = the first outgoing port number to try and use 2009 */ 2010 @property void localPort(ushort port) 2011 { 2012 p.curl.set(CurlOption.localport, cast(long) port); 2013 } 2014 2015 /** 2016 Set the no proxy flag for the specified host names. 2017 Params: 2018 test = a list of comma host names that do not require 2019 proxy to get reached 2020 */ 2021 void setNoProxy(string hosts) 2022 { 2023 p.curl.set(CurlOption.noproxy, hosts); 2024 } 2025 2026 /** 2027 Set the local outgoing port range to use. 2028 This can be used together with the localPort property. 2029 Params: 2030 range = if the first port is occupied then try this many 2031 port number forwards 2032 */ 2033 @property void localPortRange(ushort range) 2034 { 2035 p.curl.set(CurlOption.localportrange, cast(long) range); 2036 } 2037 2038 /** Set the tcp no-delay socket option on or off. 2039 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2040 */ 2041 @property void tcpNoDelay(bool on) 2042 { 2043 p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) ); 2044 } 2045 2046 /** Sets whether SSL peer certificates should be verified. 2047 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer) 2048 */ 2049 @property void verifyPeer(bool on) 2050 { 2051 p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0); 2052 } 2053 2054 /** Sets whether the host within an SSL certificate should be verified. 2055 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer) 2056 */ 2057 @property void verifyHost(bool on) 2058 { 2059 p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0); 2060 } 2061 2062 // Authentication settings 2063 2064 /** 2065 Set the user name, password and optionally domain for authentication 2066 purposes. 2067 2068 Some protocols may need authentication in some cases. Use this 2069 function to provide credentials. 2070 2071 Params: 2072 username = the username 2073 password = the password 2074 domain = used for NTLM authentication only and is set to the NTLM domain 2075 name 2076 */ 2077 void setAuthentication(const(char)[] username, const(char)[] password, 2078 const(char)[] domain = "") 2079 { 2080 import std.format : format; 2081 if (!domain.empty) 2082 username = format("%s/%s", domain, username); 2083 p.curl.set(CurlOption.userpwd, format("%s:%s", username, password)); 2084 } 2085 2086 @system unittest 2087 { 2088 import std.algorithm.searching : canFind; 2089 2090 testServer.handle((s) { 2091 auto req = s.recvReq; 2092 assert(req.hdrs.canFind("GET /")); 2093 assert(req.hdrs.canFind("Basic dXNlcjpwYXNz")); 2094 s.send(httpOK()); 2095 }); 2096 2097 auto http = HTTP(testServer.addr); 2098 http.onReceive = (ubyte[] data) { return data.length; }; 2099 http.setAuthentication("user", "pass"); 2100 http.perform(); 2101 2102 // https://issues.dlang.org/show_bug.cgi?id=17540 2103 http.setNoProxy("www.example.com"); 2104 } 2105 2106 /** 2107 Set the user name and password for proxy authentication. 2108 2109 Params: 2110 username = the username 2111 password = the password 2112 */ 2113 void setProxyAuthentication(const(char)[] username, const(char)[] password) 2114 { 2115 import std.array : replace; 2116 import std.format : format; 2117 2118 p.curl.set(CurlOption.proxyuserpwd, 2119 format("%s:%s", 2120 username.replace(":", "%3A"), 2121 password.replace(":", "%3A")) 2122 ); 2123 } 2124 2125 /** 2126 * The event handler that gets called when data is needed for sending. The 2127 * length of the `void[]` specifies the maximum number of bytes that can 2128 * be sent. 2129 * 2130 * Returns: 2131 * The callback returns the number of elements in the buffer that have been 2132 * filled and are ready to send. 2133 * The special value `.abortRequest` can be returned in order to abort the 2134 * current request. 2135 * The special value `.pauseRequest` can be returned in order to pause the 2136 * current request. 2137 * 2138 * Example: 2139 * ---- 2140 * import std.net.curl; 2141 * string msg = "Hello world"; 2142 * auto client = HTTP("dlang.org"); 2143 * client.onSend = delegate size_t(void[] data) 2144 * { 2145 * auto m = cast(void[]) msg; 2146 * size_t length = m.length > data.length ? data.length : m.length; 2147 * if (length == 0) return 0; 2148 * data[0 .. length] = m[0 .. length]; 2149 * msg = msg[length..$]; 2150 * return length; 2151 * }; 2152 * client.perform(); 2153 * ---- 2154 */ 2155 @property void onSend(size_t delegate(void[]) callback) 2156 { 2157 p.curl.clear(CurlOption.postfields); // cannot specify data when using callback 2158 p.curl.onSend = callback; 2159 } 2160 2161 /** 2162 * The event handler that receives incoming data. Be sure to copy the 2163 * incoming ubyte[] since it is not guaranteed to be valid after the 2164 * callback returns. 2165 * 2166 * Returns: 2167 * The callback returns the number of incoming bytes read. If the entire array is 2168 * not read the request will abort. 2169 * The special value .pauseRequest can be returned in order to pause the 2170 * current request. 2171 * 2172 * Example: 2173 * ---- 2174 * import std.net.curl, std.stdio; 2175 * auto client = HTTP("dlang.org"); 2176 * client.onReceive = (ubyte[] data) 2177 * { 2178 * writeln("Got data", to!(const(char)[])(data)); 2179 * return data.length; 2180 * }; 2181 * client.perform(); 2182 * ---- 2183 */ 2184 @property void onReceive(size_t delegate(ubyte[]) callback) 2185 { 2186 p.curl.onReceive = callback; 2187 } 2188 2189 /** 2190 * The event handler that gets called to inform of upload/download progress. 2191 * 2192 * Params: 2193 * dlTotal = total bytes to download 2194 * dlNow = currently downloaded bytes 2195 * ulTotal = total bytes to upload 2196 * ulNow = currently uploaded bytes 2197 * 2198 * Returns: 2199 * Return 0 from the callback to signal success, return non-zero to abort 2200 * transfer 2201 * 2202 * Example: 2203 * ---- 2204 * import std.net.curl, std.stdio; 2205 * auto client = HTTP("dlang.org"); 2206 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) 2207 * { 2208 * writeln("Progress: downloaded ", dln, " of ", dl); 2209 * writeln("Progress: uploaded ", uln, " of ", ul); 2210 * return 0; 2211 * }; 2212 * client.perform(); 2213 * ---- 2214 */ 2215 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2216 size_t ulTotal, size_t ulNow) callback) 2217 { 2218 p.curl.onProgress = callback; 2219 } 2220 } 2221 2222 /* 2223 Decode `ubyte[]` array using the provided EncodingScheme up to maxChars 2224 Returns: Tuple of ubytes read and the `Char[]` characters decoded. 2225 Not all ubytes are guaranteed to be read in case of decoding error. 2226 */ 2227 private Tuple!(size_t,Char[]) 2228 decodeString(Char = char)(const(ubyte)[] data, 2229 EncodingScheme scheme, 2230 size_t maxChars = size_t.max) 2231 { 2232 import std.encoding : INVALID_SEQUENCE; 2233 Char[] res; 2234 immutable startLen = data.length; 2235 size_t charsDecoded = 0; 2236 while (data.length && charsDecoded < maxChars) 2237 { 2238 immutable dchar dc = scheme.safeDecode(data); 2239 if (dc == INVALID_SEQUENCE) 2240 { 2241 return typeof(return)(size_t.max, cast(Char[]) null); 2242 } 2243 charsDecoded++; 2244 res ~= dc; 2245 } 2246 return typeof(return)(startLen-data.length, res); 2247 } 2248 2249 /* 2250 Decode `ubyte[]` array using the provided `EncodingScheme` until a the 2251 line terminator specified is found. The basesrc parameter is effectively 2252 prepended to src as the first thing. 2253 2254 This function is used for decoding as much of the src buffer as 2255 possible until either the terminator is found or decoding fails. If 2256 it fails as the last data in the src it may mean that the src buffer 2257 were missing some bytes in order to represent a correct code 2258 point. Upon the next call to this function more bytes have been 2259 received from net and the failing bytes should be given as the 2260 basesrc parameter. It is done this way to minimize data copying. 2261 2262 Returns: true if a terminator was found 2263 Not all ubytes are guaranteed to be read in case of decoding error. 2264 any decoded chars will be inserted into dst. 2265 */ 2266 private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc, 2267 ref const(ubyte)[] src, 2268 ref Char[] dst, 2269 EncodingScheme scheme, 2270 Terminator terminator) 2271 { 2272 import std.algorithm.searching : endsWith; 2273 import std.encoding : INVALID_SEQUENCE; 2274 import std.exception : enforce; 2275 2276 // if there is anything in the basesrc then try to decode that 2277 // first. 2278 if (basesrc.length != 0) 2279 { 2280 // Try to ensure 4 entries in the basesrc by copying from src. 2281 immutable blen = basesrc.length; 2282 immutable len = (basesrc.length + src.length) >= 4 ? 2283 4 : basesrc.length + src.length; 2284 basesrc.length = len; 2285 2286 immutable dchar dc = scheme.safeDecode(basesrc); 2287 if (dc == INVALID_SEQUENCE) 2288 { 2289 enforce!CurlException(len != 4, "Invalid code sequence"); 2290 return false; 2291 } 2292 dst ~= dc; 2293 src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src 2294 basesrc.length = 0; 2295 } 2296 2297 while (src.length) 2298 { 2299 const lsrc = src; 2300 dchar dc = scheme.safeDecode(src); 2301 if (dc == INVALID_SEQUENCE) 2302 { 2303 if (src.empty) 2304 { 2305 // The invalid sequence was in the end of the src. Maybe there 2306 // just need to be more bytes available so these last bytes are 2307 // put back to src for later use. 2308 src = lsrc; 2309 return false; 2310 } 2311 dc = '?'; 2312 } 2313 dst ~= dc; 2314 2315 if (dst.endsWith(terminator)) 2316 return true; 2317 } 2318 return false; // no terminator found 2319 } 2320 2321 /** 2322 * HTTP client functionality. 2323 * 2324 * Example: 2325 * 2326 * Get with custom data receivers: 2327 * 2328 * --- 2329 * import std.net.curl, std.stdio; 2330 * 2331 * auto http = HTTP("https://dlang.org"); 2332 * http.onReceiveHeader = 2333 * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); }; 2334 * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 2335 * http.perform(); 2336 * --- 2337 * 2338 */ 2339 2340 /** 2341 * Put with data senders: 2342 * 2343 * --- 2344 * import std.net.curl, std.stdio; 2345 * 2346 * auto http = HTTP("https://dlang.org"); 2347 * auto msg = "Hello world"; 2348 * http.contentLength = msg.length; 2349 * http.onSend = (void[] data) 2350 * { 2351 * auto m = cast(void[]) msg; 2352 * size_t len = m.length > data.length ? data.length : m.length; 2353 * if (len == 0) return len; 2354 * data[0 .. len] = m[0 .. len]; 2355 * msg = msg[len..$]; 2356 * return len; 2357 * }; 2358 * http.perform(); 2359 * --- 2360 * 2361 */ 2362 2363 /** 2364 * Tracking progress: 2365 * 2366 * --- 2367 * import std.net.curl, std.stdio; 2368 * 2369 * auto http = HTTP(); 2370 * http.method = HTTP.Method.get; 2371 * http.url = "http://upload.wikimedia.org/wikipedia/commons/" ~ 2372 * "5/53/Wikipedia-logo-en-big.png"; 2373 * http.onReceive = (ubyte[] data) { return data.length; }; 2374 * http.onProgress = (size_t dltotal, size_t dlnow, 2375 * size_t ultotal, size_t ulnow) 2376 * { 2377 * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); 2378 * return 0; 2379 * }; 2380 * http.perform(); 2381 * --- 2382 * 2383 * See_Also: $(LINK2 http://www.ietf.org/rfc/rfc2616.txt, RFC2616) 2384 * 2385 */ 2386 struct HTTP 2387 { 2388 mixin Protocol; 2389 2390 import std.datetime.systime : SysTime; 2391 import std.typecons : RefCounted; 2392 import etc.c.curl : CurlAuth, CurlInfo, curl_slist, CURLVERSION_NOW, curl_off_t; 2393 2394 /// Authentication method equal to $(REF CurlAuth, etc,c,curl) 2395 alias AuthMethod = CurlAuth; 2396 2397 static private uint defaultMaxRedirects = 10; 2398 2399 private struct Impl 2400 { 2401 ~this() 2402 { 2403 if (headersOut !is null) 2404 Curl.curl.slist_free_all(headersOut); 2405 if (curl.handle !is null) // work around RefCounted/emplace bug 2406 curl.shutdown(); 2407 } 2408 Curl curl; 2409 curl_slist* headersOut; 2410 string[string] headersIn; 2411 string charset; 2412 2413 /// The status line of the final sub-request in a request. 2414 StatusLine status; 2415 private void delegate(StatusLine) onReceiveStatusLine; 2416 2417 /// The HTTP method to use. 2418 Method method = Method.undefined; 2419 2420 @system @property void onReceiveHeader(void delegate(in char[] key, 2421 in char[] value) callback) 2422 { 2423 import std.algorithm.searching : startsWith; 2424 import std.regex : regex, match; 2425 import std.uni : toLower; 2426 2427 // Wrap incoming callback in order to separate http status line from 2428 // http headers. On redirected requests there may be several such 2429 // status lines. The last one is the one recorded. 2430 auto dg = (in char[] header) 2431 { 2432 import std.utf : UTFException; 2433 try 2434 { 2435 if (header.empty) 2436 { 2437 // header delimiter 2438 return; 2439 } 2440 if (header.startsWith("HTTP/")) 2441 { 2442 headersIn.clear(); 2443 if (parseStatusLine(header, status)) 2444 { 2445 if (onReceiveStatusLine != null) 2446 onReceiveStatusLine(status); 2447 } 2448 return; 2449 } 2450 2451 // Normal http header 2452 auto m = match(cast(char[]) header, regex("(.*?): (.*)$")); 2453 2454 auto fieldName = m.captures[1].toLower().idup; 2455 if (fieldName == "content-type") 2456 { 2457 auto mct = match(cast(char[]) m.captures[2], 2458 regex("charset=([^;]*)", "i")); 2459 if (!mct.empty && mct.captures.length > 1) 2460 charset = mct.captures[1].idup; 2461 } 2462 2463 if (!m.empty && callback !is null) 2464 callback(fieldName, m.captures[2]); 2465 headersIn[fieldName] = m.captures[2].idup; 2466 } 2467 catch (UTFException e) 2468 { 2469 //munch it - a header should be all ASCII, any "wrong UTF" is broken header 2470 } 2471 }; 2472 2473 curl.onReceiveHeader = dg; 2474 } 2475 } 2476 2477 private RefCounted!Impl p; 2478 import etc.c.curl : CurlTimeCond; 2479 2480 /// Parse status line, as received from / generated by cURL. 2481 private static bool parseStatusLine(const char[] header, out StatusLine status) @safe 2482 { 2483 import std.conv : to; 2484 import std.regex : regex, match; 2485 2486 const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$")); 2487 if (m.empty) 2488 return false; // Invalid status line 2489 else 2490 { 2491 status.majorVersion = to!ushort(m.captures[1]); 2492 status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0; 2493 status.code = to!ushort(m.captures[3]); 2494 status.reason = m.captures[4].idup; 2495 return true; 2496 } 2497 } 2498 2499 @safe unittest 2500 { 2501 StatusLine status; 2502 assert(parseStatusLine("HTTP/1.1 200 OK", status) 2503 && status == StatusLine(1, 1, 200, "OK")); 2504 assert(parseStatusLine("HTTP/1.0 304 Not Modified", status) 2505 && status == StatusLine(1, 0, 304, "Not Modified")); 2506 // The HTTP2 protocol is binary; cURL generates this fake text header. 2507 assert(parseStatusLine("HTTP/2 200", status) 2508 && status == StatusLine(2, 0, 200, null)); 2509 } 2510 2511 /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl) 2512 2513 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 2514 */ 2515 alias TimeCond = CurlTimeCond; 2516 2517 /** 2518 Constructor taking the url as parameter. 2519 */ 2520 static HTTP opCall(const(char)[] url) 2521 { 2522 HTTP http; 2523 http.initialize(); 2524 http.url = url; 2525 return http; 2526 } 2527 2528 /// 2529 static HTTP opCall() 2530 { 2531 HTTP http; 2532 http.initialize(); 2533 return http; 2534 } 2535 2536 /// 2537 HTTP dup() 2538 { 2539 HTTP copy; 2540 copy.initialize(); 2541 copy.p.method = p.method; 2542 curl_slist* cur = p.headersOut; 2543 curl_slist* newlist = null; 2544 while (cur) 2545 { 2546 newlist = Curl.curl.slist_append(newlist, cur.data); 2547 cur = cur.next; 2548 } 2549 copy.p.headersOut = newlist; 2550 copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut); 2551 copy.p.curl = p.curl.dup(); 2552 copy.dataTimeout = _defaultDataTimeout; 2553 copy.onReceiveHeader = null; 2554 return copy; 2555 } 2556 2557 private void initialize() 2558 { 2559 p.curl.initialize(); 2560 maxRedirects = HTTP.defaultMaxRedirects; 2561 p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC 2562 p.method = Method.undefined; 2563 setUserAgent(HTTP.defaultUserAgent); 2564 dataTimeout = _defaultDataTimeout; 2565 onReceiveHeader = null; 2566 verifyPeer = true; 2567 verifyHost = true; 2568 } 2569 2570 /** 2571 Perform a http request. 2572 2573 After the HTTP client has been setup and possibly assigned callbacks the 2574 `perform()` method will start performing the request towards the 2575 specified server. 2576 2577 Params: 2578 throwOnError = whether to throw an exception or return a CurlCode on error 2579 */ 2580 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 2581 { 2582 p.status.reset(); 2583 2584 CurlOption opt; 2585 final switch (p.method) 2586 { 2587 case Method.head: 2588 p.curl.set(CurlOption.nobody, 1L); 2589 opt = CurlOption.nobody; 2590 break; 2591 case Method.undefined: 2592 case Method.get: 2593 p.curl.set(CurlOption.httpget, 1L); 2594 opt = CurlOption.httpget; 2595 break; 2596 case Method.post: 2597 p.curl.set(CurlOption.post, 1L); 2598 opt = CurlOption.post; 2599 break; 2600 case Method.put: 2601 p.curl.set(CurlOption.upload, 1L); 2602 opt = CurlOption.upload; 2603 break; 2604 case Method.del: 2605 p.curl.set(CurlOption.customrequest, "DELETE"); 2606 opt = CurlOption.customrequest; 2607 break; 2608 case Method.options: 2609 p.curl.set(CurlOption.customrequest, "OPTIONS"); 2610 opt = CurlOption.customrequest; 2611 break; 2612 case Method.trace: 2613 p.curl.set(CurlOption.customrequest, "TRACE"); 2614 opt = CurlOption.customrequest; 2615 break; 2616 case Method.connect: 2617 p.curl.set(CurlOption.customrequest, "CONNECT"); 2618 opt = CurlOption.customrequest; 2619 break; 2620 case Method.patch: 2621 p.curl.set(CurlOption.customrequest, "PATCH"); 2622 opt = CurlOption.customrequest; 2623 break; 2624 } 2625 2626 scope (exit) p.curl.clear(opt); 2627 return p.curl.perform(throwOnError); 2628 } 2629 2630 /// The URL to specify the location of the resource. 2631 @property void url(const(char)[] url) 2632 { 2633 import std.algorithm.searching : startsWith; 2634 import std.uni : toLower; 2635 if (!startsWith(url.toLower(), "http://", "https://")) 2636 url = "http://" ~ url; 2637 p.curl.set(CurlOption.url, url); 2638 } 2639 2640 /// Set the CA certificate bundle file to use for SSL peer verification 2641 @property void caInfo(const(char)[] caFile) 2642 { 2643 p.curl.set(CurlOption.cainfo, caFile); 2644 } 2645 2646 // This is a workaround for mixed in content not having its 2647 // docs mixed in. 2648 version (StdDdoc) 2649 { 2650 static import etc.c.curl; 2651 2652 /// Value to return from `onSend`/`onReceive` delegates in order to 2653 /// pause a request 2654 alias requestPause = CurlReadFunc.pause; 2655 2656 /// Value to return from onSend delegate in order to abort a request 2657 alias requestAbort = CurlReadFunc.abort; 2658 2659 /** 2660 True if the instance is stopped. A stopped instance is not usable. 2661 */ 2662 @property bool isStopped(); 2663 2664 /// Stop and invalidate this instance. 2665 void shutdown(); 2666 2667 /** Set verbose. 2668 This will print request information to stderr. 2669 */ 2670 @property void verbose(bool on); 2671 2672 // Connection settings 2673 2674 /// Set timeout for activity on connection. 2675 @property void dataTimeout(Duration d); 2676 2677 /** Set maximum time an operation is allowed to take. 2678 This includes dns resolution, connecting, data transfer, etc. 2679 */ 2680 @property void operationTimeout(Duration d); 2681 2682 /// Set timeout for connecting. 2683 @property void connectTimeout(Duration d); 2684 2685 // Network settings 2686 2687 /** Proxy 2688 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 2689 */ 2690 @property void proxy(const(char)[] host); 2691 2692 /** Proxy port 2693 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 2694 */ 2695 @property void proxyPort(ushort port); 2696 2697 /// Type of proxy 2698 alias CurlProxy = etc.c.curl.CurlProxy; 2699 2700 /** Proxy type 2701 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 2702 */ 2703 @property void proxyType(CurlProxy type); 2704 2705 /// DNS lookup timeout. 2706 @property void dnsTimeout(Duration d); 2707 2708 /** 2709 * The network interface to use in form of the the IP of the interface. 2710 * 2711 * Example: 2712 * ---- 2713 * theprotocol.netInterface = "192.168.1.32"; 2714 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 2715 * ---- 2716 * 2717 * See: $(REF InternetAddress, std,socket) 2718 */ 2719 @property void netInterface(const(char)[] i); 2720 2721 /// ditto 2722 @property void netInterface(const(ubyte)[4] i); 2723 2724 /// ditto 2725 @property void netInterface(InternetAddress i); 2726 2727 /** 2728 Set the local outgoing port to use. 2729 Params: 2730 port = the first outgoing port number to try and use 2731 */ 2732 @property void localPort(ushort port); 2733 2734 /** 2735 Set the local outgoing port range to use. 2736 This can be used together with the localPort property. 2737 Params: 2738 range = if the first port is occupied then try this many 2739 port number forwards 2740 */ 2741 @property void localPortRange(ushort range); 2742 2743 /** Set the tcp no-delay socket option on or off. 2744 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2745 */ 2746 @property void tcpNoDelay(bool on); 2747 2748 // Authentication settings 2749 2750 /** 2751 Set the user name, password and optionally domain for authentication 2752 purposes. 2753 2754 Some protocols may need authentication in some cases. Use this 2755 function to provide credentials. 2756 2757 Params: 2758 username = the username 2759 password = the password 2760 domain = used for NTLM authentication only and is set to the NTLM domain 2761 name 2762 */ 2763 void setAuthentication(const(char)[] username, const(char)[] password, 2764 const(char)[] domain = ""); 2765 2766 /** 2767 Set the user name and password for proxy authentication. 2768 2769 Params: 2770 username = the username 2771 password = the password 2772 */ 2773 void setProxyAuthentication(const(char)[] username, const(char)[] password); 2774 2775 /** 2776 * The event handler that gets called when data is needed for sending. The 2777 * length of the `void[]` specifies the maximum number of bytes that can 2778 * be sent. 2779 * 2780 * Returns: 2781 * The callback returns the number of elements in the buffer that have been 2782 * filled and are ready to send. 2783 * The special value `.abortRequest` can be returned in order to abort the 2784 * current request. 2785 * The special value `.pauseRequest` can be returned in order to pause the 2786 * current request. 2787 * 2788 * Example: 2789 * ---- 2790 * import std.net.curl; 2791 * string msg = "Hello world"; 2792 * auto client = HTTP("dlang.org"); 2793 * client.onSend = delegate size_t(void[] data) 2794 * { 2795 * auto m = cast(void[]) msg; 2796 * size_t length = m.length > data.length ? data.length : m.length; 2797 * if (length == 0) return 0; 2798 * data[0 .. length] = m[0 .. length]; 2799 * msg = msg[length..$]; 2800 * return length; 2801 * }; 2802 * client.perform(); 2803 * ---- 2804 */ 2805 @property void onSend(size_t delegate(void[]) callback); 2806 2807 /** 2808 * The event handler that receives incoming data. Be sure to copy the 2809 * incoming ubyte[] since it is not guaranteed to be valid after the 2810 * callback returns. 2811 * 2812 * Returns: 2813 * The callback returns the incoming bytes read. If not the entire array is 2814 * the request will abort. 2815 * The special value .pauseRequest can be returned in order to pause the 2816 * current request. 2817 * 2818 * Example: 2819 * ---- 2820 * import std.net.curl, std.stdio; 2821 * auto client = HTTP("dlang.org"); 2822 * client.onReceive = (ubyte[] data) 2823 * { 2824 * writeln("Got data", to!(const(char)[])(data)); 2825 * return data.length; 2826 * }; 2827 * client.perform(); 2828 * ---- 2829 */ 2830 @property void onReceive(size_t delegate(ubyte[]) callback); 2831 2832 /** 2833 * Register an event handler that gets called to inform of 2834 * upload/download progress. 2835 * 2836 * Callback_parameters: 2837 * $(CALLBACK_PARAMS) 2838 * 2839 * Callback_returns: Return 0 to signal success, return non-zero to 2840 * abort transfer. 2841 * 2842 * Example: 2843 * ---- 2844 * import std.net.curl, std.stdio; 2845 * auto client = HTTP("dlang.org"); 2846 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) 2847 * { 2848 * writeln("Progress: downloaded ", dln, " of ", dl); 2849 * writeln("Progress: uploaded ", uln, " of ", ul); 2850 * return 0; 2851 * }; 2852 * client.perform(); 2853 * ---- 2854 */ 2855 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2856 size_t ulTotal, size_t ulNow) callback); 2857 } 2858 2859 /** Clear all outgoing headers. 2860 */ 2861 void clearRequestHeaders() 2862 { 2863 if (p.headersOut !is null) 2864 Curl.curl.slist_free_all(p.headersOut); 2865 p.headersOut = null; 2866 p.curl.clear(CurlOption.httpheader); 2867 } 2868 2869 /** Add a header e.g. "X-CustomField: Something is fishy". 2870 * 2871 * There is no remove header functionality. Do a $(LREF clearRequestHeaders) 2872 * and set the needed headers instead. 2873 * 2874 * Example: 2875 * --- 2876 * import std.net.curl; 2877 * auto client = HTTP(); 2878 * client.addRequestHeader("X-Custom-ABC", "This is the custom value"); 2879 * auto content = get("dlang.org", client); 2880 * --- 2881 */ 2882 void addRequestHeader(const(char)[] name, const(char)[] value) 2883 { 2884 import std.format : format; 2885 import std.internal.cstring : tempCString; 2886 import std.uni : icmp; 2887 2888 if (icmp(name, "User-Agent") == 0) 2889 return setUserAgent(value); 2890 string nv = format("%s: %s", name, value); 2891 p.headersOut = Curl.curl.slist_append(p.headersOut, 2892 nv.tempCString().buffPtr); 2893 p.curl.set(CurlOption.httpheader, p.headersOut); 2894 } 2895 2896 /** 2897 * The default "User-Agent" value send with a request. 2898 * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))" 2899 */ 2900 static string defaultUserAgent() @property 2901 { 2902 import std.compiler : version_major, version_minor; 2903 import std.format : format, sformat; 2904 2905 // http://curl.haxx.se/docs/versions.html 2906 enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)"; 2907 enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3; 2908 2909 static char[maxLen] buf = void; 2910 static string userAgent; 2911 2912 if (!userAgent.length) 2913 { 2914 auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num; 2915 userAgent = cast(immutable) sformat( 2916 buf, fmt, version_major, version_minor, 2917 curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF); 2918 } 2919 return userAgent; 2920 } 2921 2922 /** Set the value of the user agent request header field. 2923 * 2924 * By default a request has it's "User-Agent" field set to $(LREF 2925 * defaultUserAgent) even if `setUserAgent` was never called. Pass 2926 * an empty string to suppress the "User-Agent" field altogether. 2927 */ 2928 void setUserAgent(const(char)[] userAgent) 2929 { 2930 p.curl.set(CurlOption.useragent, userAgent); 2931 } 2932 2933 /** 2934 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 2935 * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. 2936 * 2937 * Params: 2938 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 2939 * The values are: 2940 * `etc.c.curl.CurlInfo.namelookup_time`, 2941 * `etc.c.curl.CurlInfo.connect_time`, 2942 * `etc.c.curl.CurlInfo.pretransfer_time`, 2943 * `etc.c.curl.CurlInfo.starttransfer_time`, 2944 * `etc.c.curl.CurlInfo.redirect_time`, 2945 * `etc.c.curl.CurlInfo.appconnect_time`, 2946 * `etc.c.curl.CurlInfo.total_time`. 2947 * val = the actual value of the inquired timing. 2948 * 2949 * Returns: 2950 * The return code of the operation. The value stored in val 2951 * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. 2952 * 2953 * Example: 2954 * --- 2955 * import std.net.curl; 2956 * import etc.c.curl : CurlError, CurlInfo; 2957 * 2958 * auto client = HTTP("dlang.org"); 2959 * client.perform(); 2960 * 2961 * double val; 2962 * CurlCode code; 2963 * 2964 * code = client.getTiming(CurlInfo.namelookup_time, val); 2965 * assert(code == CurlError.ok); 2966 * --- 2967 */ 2968 CurlCode getTiming(CurlInfo timing, ref double val) 2969 { 2970 return p.curl.getTiming(timing, val); 2971 } 2972 2973 /** The headers read from a successful response. 2974 * 2975 */ 2976 @property string[string] responseHeaders() 2977 { 2978 return p.headersIn; 2979 } 2980 2981 /// HTTP method used. 2982 @property void method(Method m) 2983 { 2984 p.method = m; 2985 } 2986 2987 /// ditto 2988 @property Method method() 2989 { 2990 return p.method; 2991 } 2992 2993 /** 2994 HTTP status line of last response. One call to perform may 2995 result in several requests because of redirection. 2996 */ 2997 @property StatusLine statusLine() 2998 { 2999 return p.status; 3000 } 3001 3002 /// Set the active cookie string e.g. "name1=value1;name2=value2" 3003 void setCookie(const(char)[] cookie) 3004 { 3005 p.curl.set(CurlOption.cookie, cookie); 3006 } 3007 3008 /// Set a file path to where a cookie jar should be read/stored. 3009 void setCookieJar(const(char)[] path) 3010 { 3011 p.curl.set(CurlOption.cookiefile, path); 3012 if (path.length) 3013 p.curl.set(CurlOption.cookiejar, path); 3014 } 3015 3016 /// Flush cookie jar to disk. 3017 void flushCookieJar() 3018 { 3019 p.curl.set(CurlOption.cookielist, "FLUSH"); 3020 } 3021 3022 /// Clear session cookies. 3023 void clearSessionCookies() 3024 { 3025 p.curl.set(CurlOption.cookielist, "SESS"); 3026 } 3027 3028 /// Clear all cookies. 3029 void clearAllCookies() 3030 { 3031 p.curl.set(CurlOption.cookielist, "ALL"); 3032 } 3033 3034 /** 3035 Set time condition on the request. 3036 3037 Params: 3038 cond = `CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}` 3039 timestamp = Timestamp for the condition 3040 3041 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 3042 */ 3043 void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp) 3044 { 3045 p.curl.set(CurlOption.timecondition, cond); 3046 p.curl.set(CurlOption.timevalue, timestamp.toUnixTime()); 3047 } 3048 3049 /** Specifying data to post when not using the onSend callback. 3050 * 3051 * The data is NOT copied by the library. Content-Type will default to 3052 * application/octet-stream. Data is not converted or encoded by this 3053 * method. 3054 * 3055 * Example: 3056 * ---- 3057 * import std.net.curl, std.stdio; 3058 * auto http = HTTP("http://www.mydomain.com"); 3059 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3060 * http.postData = [1,2,3,4,5]; 3061 * http.perform(); 3062 * ---- 3063 */ 3064 @property void postData(const(void)[] data) 3065 { 3066 setPostData(data, "application/octet-stream"); 3067 } 3068 3069 /** Specifying data to post when not using the onSend callback. 3070 * 3071 * The data is NOT copied by the library. Content-Type will default to 3072 * text/plain. Data is not converted or encoded by this method. 3073 * 3074 * Example: 3075 * ---- 3076 * import std.net.curl, std.stdio; 3077 * auto http = HTTP("http://www.mydomain.com"); 3078 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3079 * http.postData = "The quick...."; 3080 * http.perform(); 3081 * ---- 3082 */ 3083 @property void postData(const(char)[] data) 3084 { 3085 setPostData(data, "text/plain"); 3086 } 3087 3088 /** 3089 * Specify data to post when not using the onSend callback, with 3090 * user-specified Content-Type. 3091 * Params: 3092 * data = Data to post. 3093 * contentType = MIME type of the data, for example, "text/plain" or 3094 * "application/octet-stream". See also: 3095 * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type, 3096 * Internet media type) on Wikipedia. 3097 * ----- 3098 * import std.net.curl; 3099 * auto http = HTTP("http://onlineform.example.com"); 3100 * auto data = "app=login&username=bob&password=s00perS3kret"; 3101 * http.setPostData(data, "application/x-www-form-urlencoded"); 3102 * http.onReceive = (ubyte[] data) { return data.length; }; 3103 * http.perform(); 3104 * ----- 3105 */ 3106 void setPostData(const(void)[] data, string contentType) 3107 { 3108 // cannot use callback when specifying data directly so it is disabled here. 3109 p.curl.clear(CurlOption.readfunction); 3110 addRequestHeader("Content-Type", contentType); 3111 p.curl.set(CurlOption.postfields, cast(void*) data.ptr); 3112 p.curl.set(CurlOption.postfieldsize, data.length); 3113 if (method == Method.undefined) 3114 method = Method.post; 3115 } 3116 3117 @system unittest 3118 { 3119 import std.algorithm.searching : canFind; 3120 3121 testServer.handle((s) { 3122 auto req = s.recvReq!ubyte; 3123 assert(req.hdrs.canFind("POST /path")); 3124 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 3125 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 3126 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 3127 }); 3128 auto data = new ubyte[](256); 3129 foreach (i, ref ub; data) 3130 ub = cast(ubyte) i; 3131 3132 auto http = HTTP(testServer.addr~"/path"); 3133 http.postData = data; 3134 ubyte[] res; 3135 http.onReceive = (data) { res ~= data; return data.length; }; 3136 http.perform(); 3137 assert(res == cast(ubyte[])[17, 27, 35, 41]); 3138 } 3139 3140 /** 3141 * Set the event handler that receives incoming headers. 3142 * 3143 * The callback will receive a header field key, value as parameter. The 3144 * `const(char)[]` arrays are not valid after the delegate has returned. 3145 * 3146 * Example: 3147 * ---- 3148 * import std.net.curl, std.stdio; 3149 * auto http = HTTP("dlang.org"); 3150 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3151 * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); }; 3152 * http.perform(); 3153 * ---- 3154 */ 3155 @property void onReceiveHeader(void delegate(in char[] key, 3156 in char[] value) callback) 3157 { 3158 p.onReceiveHeader = callback; 3159 } 3160 3161 /** 3162 Callback for each received StatusLine. 3163 3164 Notice that several callbacks can be done for each call to 3165 `perform()` due to redirections. 3166 3167 See_Also: $(LREF StatusLine) 3168 */ 3169 @property void onReceiveStatusLine(void delegate(StatusLine) callback) 3170 { 3171 p.onReceiveStatusLine = callback; 3172 } 3173 3174 /** 3175 The content length in bytes when using request that has content 3176 e.g. POST/PUT and not using chunked transfer. Is set as the 3177 "Content-Length" header. Set to ulong.max to reset to chunked transfer. 3178 */ 3179 @property void contentLength(ulong len) 3180 { 3181 import std.conv : to; 3182 3183 CurlOption lenOpt; 3184 3185 // Force post if necessary 3186 if (p.method != Method.put && p.method != Method.post && 3187 p.method != Method.patch) 3188 p.method = Method.post; 3189 3190 if (p.method == Method.post || p.method == Method.patch) 3191 lenOpt = CurlOption.postfieldsize_large; 3192 else 3193 lenOpt = CurlOption.infilesize_large; 3194 3195 if (size_t.max != ulong.max && len == size_t.max) 3196 len = ulong.max; // check size_t.max for backwards compat, turn into error 3197 3198 if (len == ulong.max) 3199 { 3200 // HTTP 1.1 supports requests with no length header set. 3201 addRequestHeader("Transfer-Encoding", "chunked"); 3202 addRequestHeader("Expect", "100-continue"); 3203 } 3204 else 3205 { 3206 p.curl.set(lenOpt, to!curl_off_t(len)); 3207 } 3208 } 3209 3210 /** 3211 Authentication method as specified in $(LREF AuthMethod). 3212 */ 3213 @property void authenticationMethod(AuthMethod authMethod) 3214 { 3215 p.curl.set(CurlOption.httpauth, cast(long) authMethod); 3216 } 3217 3218 /** 3219 Set max allowed redirections using the location header. 3220 uint.max for infinite. 3221 */ 3222 @property void maxRedirects(uint maxRedirs) 3223 { 3224 if (maxRedirs == uint.max) 3225 { 3226 // Disable 3227 p.curl.set(CurlOption.followlocation, 0); 3228 } 3229 else 3230 { 3231 p.curl.set(CurlOption.followlocation, 1); 3232 p.curl.set(CurlOption.maxredirs, maxRedirs); 3233 } 3234 } 3235 3236 /** <a name="HTTP.Method"/>The standard HTTP methods : 3237 * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1) 3238 */ 3239 enum Method 3240 { 3241 undefined, 3242 head, /// 3243 get, /// 3244 post, /// 3245 put, /// 3246 del, /// 3247 options, /// 3248 trace, /// 3249 connect, /// 3250 patch, /// 3251 } 3252 3253 /** 3254 HTTP status line ie. the first line returned in an HTTP response. 3255 3256 If authentication or redirections are done then the status will be for 3257 the last response received. 3258 */ 3259 struct StatusLine 3260 { 3261 ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0. 3262 ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0. 3263 ushort code; /// HTTP status line code e.g. 200. 3264 string reason; /// HTTP status line reason string. 3265 3266 /// Reset this status line 3267 @safe void reset() 3268 { 3269 majorVersion = 0; 3270 minorVersion = 0; 3271 code = 0; 3272 reason = ""; 3273 } 3274 3275 /// 3276 string toString() const 3277 { 3278 import std.format : format; 3279 return format("%s %s (%s.%s)", 3280 code, reason, majorVersion, minorVersion); 3281 } 3282 } 3283 3284 } // HTTP 3285 3286 @system unittest // charset/Charset/CHARSET/... 3287 { 3288 import etc.c.curl; 3289 3290 static foreach (c; ["charset", "Charset", "CHARSET", "CharSet", "charSet", 3291 "ChArSeT", "cHaRsEt"]) 3292 {{ 3293 testServer.handle((s) { 3294 s.send("HTTP/1.1 200 OK\r\n"~ 3295 "Content-Length: 0\r\n"~ 3296 "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~ 3297 "\r\n"); 3298 }); 3299 3300 auto http = HTTP(testServer.addr); 3301 http.perform(); 3302 assert(http.p.charset == "foo"); 3303 3304 // https://issues.dlang.org/show_bug.cgi?id=16736 3305 double val; 3306 CurlCode code; 3307 3308 code = http.getTiming(CurlInfo.total_time, val); 3309 assert(code == CurlError.ok); 3310 code = http.getTiming(CurlInfo.namelookup_time, val); 3311 assert(code == CurlError.ok); 3312 code = http.getTiming(CurlInfo.connect_time, val); 3313 assert(code == CurlError.ok); 3314 code = http.getTiming(CurlInfo.pretransfer_time, val); 3315 assert(code == CurlError.ok); 3316 code = http.getTiming(CurlInfo.starttransfer_time, val); 3317 assert(code == CurlError.ok); 3318 code = http.getTiming(CurlInfo.redirect_time, val); 3319 assert(code == CurlError.ok); 3320 code = http.getTiming(CurlInfo.appconnect_time, val); 3321 assert(code == CurlError.ok); 3322 }} 3323 } 3324 3325 /** 3326 FTP client functionality. 3327 3328 See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959) 3329 */ 3330 struct FTP 3331 { 3332 3333 mixin Protocol; 3334 3335 import std.typecons : RefCounted; 3336 import etc.c.curl : CurlError, CurlInfo, curl_off_t, curl_slist; 3337 3338 private struct Impl 3339 { 3340 ~this() 3341 { 3342 if (commands !is null) 3343 Curl.curl.slist_free_all(commands); 3344 if (curl.handle !is null) // work around RefCounted/emplace bug 3345 curl.shutdown(); 3346 } 3347 curl_slist* commands; 3348 Curl curl; 3349 string encoding; 3350 } 3351 3352 private RefCounted!Impl p; 3353 3354 /** 3355 FTP access to the specified url. 3356 */ 3357 static FTP opCall(const(char)[] url) 3358 { 3359 FTP ftp; 3360 ftp.initialize(); 3361 ftp.url = url; 3362 return ftp; 3363 } 3364 3365 /// 3366 static FTP opCall() 3367 { 3368 FTP ftp; 3369 ftp.initialize(); 3370 return ftp; 3371 } 3372 3373 /// 3374 FTP dup() 3375 { 3376 FTP copy = FTP(); 3377 copy.initialize(); 3378 copy.p.encoding = p.encoding; 3379 copy.p.curl = p.curl.dup(); 3380 curl_slist* cur = p.commands; 3381 curl_slist* newlist = null; 3382 while (cur) 3383 { 3384 newlist = Curl.curl.slist_append(newlist, cur.data); 3385 cur = cur.next; 3386 } 3387 copy.p.commands = newlist; 3388 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3389 copy.dataTimeout = _defaultDataTimeout; 3390 return copy; 3391 } 3392 3393 private void initialize() 3394 { 3395 p.curl.initialize(); 3396 p.encoding = "ISO-8859-1"; 3397 dataTimeout = _defaultDataTimeout; 3398 } 3399 3400 /** 3401 Performs the ftp request as it has been configured. 3402 3403 After a FTP client has been setup and possibly assigned callbacks the $(D 3404 perform()) method will start performing the actual communication with the 3405 server. 3406 3407 Params: 3408 throwOnError = whether to throw an exception or return a CurlCode on error 3409 */ 3410 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3411 { 3412 return p.curl.perform(throwOnError); 3413 } 3414 3415 /// The URL to specify the location of the resource. 3416 @property void url(const(char)[] url) 3417 { 3418 import std.algorithm.searching : startsWith; 3419 import std.uni : toLower; 3420 3421 if (!startsWith(url.toLower(), "ftp://", "ftps://")) 3422 url = "ftp://" ~ url; 3423 p.curl.set(CurlOption.url, url); 3424 } 3425 3426 // This is a workaround for mixed in content not having its 3427 // docs mixed in. 3428 version (StdDdoc) 3429 { 3430 static import etc.c.curl; 3431 3432 /// Value to return from `onSend`/`onReceive` delegates in order to 3433 /// pause a request 3434 alias requestPause = CurlReadFunc.pause; 3435 3436 /// Value to return from onSend delegate in order to abort a request 3437 alias requestAbort = CurlReadFunc.abort; 3438 3439 /** 3440 True if the instance is stopped. A stopped instance is not usable. 3441 */ 3442 @property bool isStopped(); 3443 3444 /// Stop and invalidate this instance. 3445 void shutdown(); 3446 3447 /** Set verbose. 3448 This will print request information to stderr. 3449 */ 3450 @property void verbose(bool on); 3451 3452 // Connection settings 3453 3454 /// Set timeout for activity on connection. 3455 @property void dataTimeout(Duration d); 3456 3457 /** Set maximum time an operation is allowed to take. 3458 This includes dns resolution, connecting, data transfer, etc. 3459 */ 3460 @property void operationTimeout(Duration d); 3461 3462 /// Set timeout for connecting. 3463 @property void connectTimeout(Duration d); 3464 3465 // Network settings 3466 3467 /** Proxy 3468 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3469 */ 3470 @property void proxy(const(char)[] host); 3471 3472 /** Proxy port 3473 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3474 */ 3475 @property void proxyPort(ushort port); 3476 3477 /// Type of proxy 3478 alias CurlProxy = etc.c.curl.CurlProxy; 3479 3480 /** Proxy type 3481 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3482 */ 3483 @property void proxyType(CurlProxy type); 3484 3485 /// DNS lookup timeout. 3486 @property void dnsTimeout(Duration d); 3487 3488 /** 3489 * The network interface to use in form of the the IP of the interface. 3490 * 3491 * Example: 3492 * ---- 3493 * theprotocol.netInterface = "192.168.1.32"; 3494 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3495 * ---- 3496 * 3497 * See: $(REF InternetAddress, std,socket) 3498 */ 3499 @property void netInterface(const(char)[] i); 3500 3501 /// ditto 3502 @property void netInterface(const(ubyte)[4] i); 3503 3504 /// ditto 3505 @property void netInterface(InternetAddress i); 3506 3507 /** 3508 Set the local outgoing port to use. 3509 Params: 3510 port = the first outgoing port number to try and use 3511 */ 3512 @property void localPort(ushort port); 3513 3514 /** 3515 Set the local outgoing port range to use. 3516 This can be used together with the localPort property. 3517 Params: 3518 range = if the first port is occupied then try this many 3519 port number forwards 3520 */ 3521 @property void localPortRange(ushort range); 3522 3523 /** Set the tcp no-delay socket option on or off. 3524 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3525 */ 3526 @property void tcpNoDelay(bool on); 3527 3528 // Authentication settings 3529 3530 /** 3531 Set the user name, password and optionally domain for authentication 3532 purposes. 3533 3534 Some protocols may need authentication in some cases. Use this 3535 function to provide credentials. 3536 3537 Params: 3538 username = the username 3539 password = the password 3540 domain = used for NTLM authentication only and is set to the NTLM domain 3541 name 3542 */ 3543 void setAuthentication(const(char)[] username, const(char)[] password, 3544 const(char)[] domain = ""); 3545 3546 /** 3547 Set the user name and password for proxy authentication. 3548 3549 Params: 3550 username = the username 3551 password = the password 3552 */ 3553 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3554 3555 /** 3556 * The event handler that gets called when data is needed for sending. The 3557 * length of the `void[]` specifies the maximum number of bytes that can 3558 * be sent. 3559 * 3560 * Returns: 3561 * The callback returns the number of elements in the buffer that have been 3562 * filled and are ready to send. 3563 * The special value `.abortRequest` can be returned in order to abort the 3564 * current request. 3565 * The special value `.pauseRequest` can be returned in order to pause the 3566 * current request. 3567 * 3568 */ 3569 @property void onSend(size_t delegate(void[]) callback); 3570 3571 /** 3572 * The event handler that receives incoming data. Be sure to copy the 3573 * incoming ubyte[] since it is not guaranteed to be valid after the 3574 * callback returns. 3575 * 3576 * Returns: 3577 * The callback returns the incoming bytes read. If not the entire array is 3578 * the request will abort. 3579 * The special value .pauseRequest can be returned in order to pause the 3580 * current request. 3581 * 3582 */ 3583 @property void onReceive(size_t delegate(ubyte[]) callback); 3584 3585 /** 3586 * The event handler that gets called to inform of upload/download progress. 3587 * 3588 * Callback_parameters: 3589 * $(CALLBACK_PARAMS) 3590 * 3591 * Callback_returns: 3592 * Return 0 from the callback to signal success, return non-zero to 3593 * abort transfer. 3594 */ 3595 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 3596 size_t ulTotal, size_t ulNow) callback); 3597 } 3598 3599 /** Clear all commands send to ftp server. 3600 */ 3601 void clearCommands() 3602 { 3603 if (p.commands !is null) 3604 Curl.curl.slist_free_all(p.commands); 3605 p.commands = null; 3606 p.curl.clear(CurlOption.postquote); 3607 } 3608 3609 /** Add a command to send to ftp server. 3610 * 3611 * There is no remove command functionality. Do a $(LREF clearCommands) and 3612 * set the needed commands instead. 3613 * 3614 * Example: 3615 * --- 3616 * import std.net.curl; 3617 * auto client = FTP(); 3618 * client.addCommand("RNFR my_file.txt"); 3619 * client.addCommand("RNTO my_renamed_file.txt"); 3620 * upload("my_file.txt", "ftp.digitalmars.com", client); 3621 * --- 3622 */ 3623 void addCommand(const(char)[] command) 3624 { 3625 import std.internal.cstring : tempCString; 3626 p.commands = Curl.curl.slist_append(p.commands, 3627 command.tempCString().buffPtr); 3628 p.curl.set(CurlOption.postquote, p.commands); 3629 } 3630 3631 /// Connection encoding. Defaults to ISO-8859-1. 3632 @property void encoding(string name) 3633 { 3634 p.encoding = name; 3635 } 3636 3637 /// ditto 3638 @property string encoding() 3639 { 3640 return p.encoding; 3641 } 3642 3643 /** 3644 The content length in bytes of the ftp data. 3645 */ 3646 @property void contentLength(ulong len) 3647 { 3648 import std.conv : to; 3649 p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len)); 3650 } 3651 3652 /** 3653 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 3654 * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. 3655 * 3656 * Params: 3657 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 3658 * The values are: 3659 * `etc.c.curl.CurlInfo.namelookup_time`, 3660 * `etc.c.curl.CurlInfo.connect_time`, 3661 * `etc.c.curl.CurlInfo.pretransfer_time`, 3662 * `etc.c.curl.CurlInfo.starttransfer_time`, 3663 * `etc.c.curl.CurlInfo.redirect_time`, 3664 * `etc.c.curl.CurlInfo.appconnect_time`, 3665 * `etc.c.curl.CurlInfo.total_time`. 3666 * val = the actual value of the inquired timing. 3667 * 3668 * Returns: 3669 * The return code of the operation. The value stored in val 3670 * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. 3671 * 3672 * Example: 3673 * --- 3674 * import std.net.curl; 3675 * import etc.c.curl : CurlError, CurlInfo; 3676 * 3677 * auto client = FTP(); 3678 * client.addCommand("RNFR my_file.txt"); 3679 * client.addCommand("RNTO my_renamed_file.txt"); 3680 * upload("my_file.txt", "ftp.digitalmars.com", client); 3681 * 3682 * double val; 3683 * CurlCode code; 3684 * 3685 * code = client.getTiming(CurlInfo.namelookup_time, val); 3686 * assert(code == CurlError.ok); 3687 * --- 3688 */ 3689 CurlCode getTiming(CurlInfo timing, ref double val) 3690 { 3691 return p.curl.getTiming(timing, val); 3692 } 3693 3694 @system unittest 3695 { 3696 auto client = FTP(); 3697 3698 double val; 3699 CurlCode code; 3700 3701 code = client.getTiming(CurlInfo.total_time, val); 3702 assert(code == CurlError.ok); 3703 code = client.getTiming(CurlInfo.namelookup_time, val); 3704 assert(code == CurlError.ok); 3705 code = client.getTiming(CurlInfo.connect_time, val); 3706 assert(code == CurlError.ok); 3707 code = client.getTiming(CurlInfo.pretransfer_time, val); 3708 assert(code == CurlError.ok); 3709 code = client.getTiming(CurlInfo.starttransfer_time, val); 3710 assert(code == CurlError.ok); 3711 code = client.getTiming(CurlInfo.redirect_time, val); 3712 assert(code == CurlError.ok); 3713 code = client.getTiming(CurlInfo.appconnect_time, val); 3714 assert(code == CurlError.ok); 3715 } 3716 } 3717 3718 /** 3719 * Basic SMTP protocol support. 3720 * 3721 * Example: 3722 * --- 3723 * import std.net.curl; 3724 * 3725 * // Send an email with SMTPS 3726 * auto smtp = SMTP("smtps://smtp.gmail.com"); 3727 * smtp.setAuthentication("from.addr@gmail.com", "password"); 3728 * smtp.mailTo = ["<to.addr@gmail.com>"]; 3729 * smtp.mailFrom = "<from.addr@gmail.com>"; 3730 * smtp.message = "Example Message"; 3731 * smtp.perform(); 3732 * --- 3733 * 3734 * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821) 3735 */ 3736 struct SMTP 3737 { 3738 mixin Protocol; 3739 import std.typecons : RefCounted; 3740 import etc.c.curl : CurlUseSSL, curl_slist; 3741 3742 private struct Impl 3743 { 3744 ~this() 3745 { 3746 if (curl.handle !is null) // work around RefCounted/emplace bug 3747 curl.shutdown(); 3748 } 3749 Curl curl; 3750 3751 @property void message(string msg) 3752 { 3753 import std.algorithm.comparison : min; 3754 3755 auto _message = msg; 3756 /** 3757 This delegate reads the message text and copies it. 3758 */ 3759 curl.onSend = delegate size_t(void[] data) 3760 { 3761 if (!msg.length) return 0; 3762 size_t to_copy = min(data.length, _message.length); 3763 data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy]; 3764 _message = _message[to_copy..$]; 3765 return to_copy; 3766 }; 3767 } 3768 } 3769 3770 private RefCounted!Impl p; 3771 3772 /** 3773 Sets to the URL of the SMTP server. 3774 */ 3775 static SMTP opCall(const(char)[] url) 3776 { 3777 SMTP smtp; 3778 smtp.initialize(); 3779 smtp.url = url; 3780 return smtp; 3781 } 3782 3783 /// 3784 static SMTP opCall() 3785 { 3786 SMTP smtp; 3787 smtp.initialize(); 3788 return smtp; 3789 } 3790 3791 /+ TODO: The other structs have this function. 3792 SMTP dup() 3793 { 3794 SMTP copy = SMTP(); 3795 copy.initialize(); 3796 copy.p.encoding = p.encoding; 3797 copy.p.curl = p.curl.dup(); 3798 curl_slist* cur = p.commands; 3799 curl_slist* newlist = null; 3800 while (cur) 3801 { 3802 newlist = Curl.curl.slist_append(newlist, cur.data); 3803 cur = cur.next; 3804 } 3805 copy.p.commands = newlist; 3806 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3807 copy.dataTimeout = _defaultDataTimeout; 3808 return copy; 3809 } 3810 +/ 3811 3812 /** 3813 Performs the request as configured. 3814 Params: 3815 throwOnError = whether to throw an exception or return a CurlCode on error 3816 */ 3817 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3818 { 3819 return p.curl.perform(throwOnError); 3820 } 3821 3822 /// The URL to specify the location of the resource. 3823 @property void url(const(char)[] url) 3824 { 3825 import std.algorithm.searching : startsWith; 3826 import std.exception : enforce; 3827 import std.uni : toLower; 3828 3829 auto lowered = url.toLower(); 3830 3831 if (lowered.startsWith("smtps://")) 3832 { 3833 p.curl.set(CurlOption.use_ssl, CurlUseSSL.all); 3834 } 3835 else 3836 { 3837 enforce!CurlException(lowered.startsWith("smtp://"), 3838 "The url must be for the smtp protocol."); 3839 } 3840 p.curl.set(CurlOption.url, url); 3841 } 3842 3843 private void initialize() 3844 { 3845 p.curl.initialize(); 3846 p.curl.set(CurlOption.upload, 1L); 3847 dataTimeout = _defaultDataTimeout; 3848 verifyPeer = true; 3849 verifyHost = true; 3850 } 3851 3852 // This is a workaround for mixed in content not having its 3853 // docs mixed in. 3854 version (StdDdoc) 3855 { 3856 static import etc.c.curl; 3857 3858 /// Value to return from `onSend`/`onReceive` delegates in order to 3859 /// pause a request 3860 alias requestPause = CurlReadFunc.pause; 3861 3862 /// Value to return from onSend delegate in order to abort a request 3863 alias requestAbort = CurlReadFunc.abort; 3864 3865 /** 3866 True if the instance is stopped. A stopped instance is not usable. 3867 */ 3868 @property bool isStopped(); 3869 3870 /// Stop and invalidate this instance. 3871 void shutdown(); 3872 3873 /** Set verbose. 3874 This will print request information to stderr. 3875 */ 3876 @property void verbose(bool on); 3877 3878 // Connection settings 3879 3880 /// Set timeout for activity on connection. 3881 @property void dataTimeout(Duration d); 3882 3883 /** Set maximum time an operation is allowed to take. 3884 This includes dns resolution, connecting, data transfer, etc. 3885 */ 3886 @property void operationTimeout(Duration d); 3887 3888 /// Set timeout for connecting. 3889 @property void connectTimeout(Duration d); 3890 3891 // Network settings 3892 3893 /** Proxy 3894 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3895 */ 3896 @property void proxy(const(char)[] host); 3897 3898 /** Proxy port 3899 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3900 */ 3901 @property void proxyPort(ushort port); 3902 3903 /// Type of proxy 3904 alias CurlProxy = etc.c.curl.CurlProxy; 3905 3906 /** Proxy type 3907 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3908 */ 3909 @property void proxyType(CurlProxy type); 3910 3911 /// DNS lookup timeout. 3912 @property void dnsTimeout(Duration d); 3913 3914 /** 3915 * The network interface to use in form of the the IP of the interface. 3916 * 3917 * Example: 3918 * ---- 3919 * theprotocol.netInterface = "192.168.1.32"; 3920 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3921 * ---- 3922 * 3923 * See: $(REF InternetAddress, std,socket) 3924 */ 3925 @property void netInterface(const(char)[] i); 3926 3927 /// ditto 3928 @property void netInterface(const(ubyte)[4] i); 3929 3930 /// ditto 3931 @property void netInterface(InternetAddress i); 3932 3933 /** 3934 Set the local outgoing port to use. 3935 Params: 3936 port = the first outgoing port number to try and use 3937 */ 3938 @property void localPort(ushort port); 3939 3940 /** 3941 Set the local outgoing port range to use. 3942 This can be used together with the localPort property. 3943 Params: 3944 range = if the first port is occupied then try this many 3945 port number forwards 3946 */ 3947 @property void localPortRange(ushort range); 3948 3949 /** Set the tcp no-delay socket option on or off. 3950 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3951 */ 3952 @property void tcpNoDelay(bool on); 3953 3954 // Authentication settings 3955 3956 /** 3957 Set the user name, password and optionally domain for authentication 3958 purposes. 3959 3960 Some protocols may need authentication in some cases. Use this 3961 function to provide credentials. 3962 3963 Params: 3964 username = the username 3965 password = the password 3966 domain = used for NTLM authentication only and is set to the NTLM domain 3967 name 3968 */ 3969 void setAuthentication(const(char)[] username, const(char)[] password, 3970 const(char)[] domain = ""); 3971 3972 /** 3973 Set the user name and password for proxy authentication. 3974 3975 Params: 3976 username = the username 3977 password = the password 3978 */ 3979 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3980 3981 /** 3982 * The event handler that gets called when data is needed for sending. The 3983 * length of the `void[]` specifies the maximum number of bytes that can 3984 * be sent. 3985 * 3986 * Returns: 3987 * The callback returns the number of elements in the buffer that have been 3988 * filled and are ready to send. 3989 * The special value `.abortRequest` can be returned in order to abort the 3990 * current request. 3991 * The special value `.pauseRequest` can be returned in order to pause the 3992 * current request. 3993 */ 3994 @property void onSend(size_t delegate(void[]) callback); 3995 3996 /** 3997 * The event handler that receives incoming data. Be sure to copy the 3998 * incoming ubyte[] since it is not guaranteed to be valid after the 3999 * callback returns. 4000 * 4001 * Returns: 4002 * The callback returns the incoming bytes read. If not the entire array is 4003 * the request will abort. 4004 * The special value .pauseRequest can be returned in order to pause the 4005 * current request. 4006 */ 4007 @property void onReceive(size_t delegate(ubyte[]) callback); 4008 4009 /** 4010 * The event handler that gets called to inform of upload/download progress. 4011 * 4012 * Callback_parameters: 4013 * $(CALLBACK_PARAMS) 4014 * 4015 * Callback_returns: 4016 * Return 0 from the callback to signal success, return non-zero to 4017 * abort transfer. 4018 */ 4019 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 4020 size_t ulTotal, size_t ulNow) callback); 4021 } 4022 4023 /** 4024 Setter for the sender's email address. 4025 */ 4026 @property void mailFrom()(const(char)[] sender) 4027 { 4028 assert(!sender.empty, "Sender must not be empty"); 4029 p.curl.set(CurlOption.mail_from, sender); 4030 } 4031 4032 /** 4033 Setter for the recipient email addresses. 4034 */ 4035 void mailTo()(const(char)[][] recipients...) 4036 { 4037 import std.internal.cstring : tempCString; 4038 assert(!recipients.empty, "Recipient must not be empty"); 4039 curl_slist* recipients_list = null; 4040 foreach (recipient; recipients) 4041 { 4042 recipients_list = 4043 Curl.curl.slist_append(recipients_list, 4044 recipient.tempCString().buffPtr); 4045 } 4046 p.curl.set(CurlOption.mail_rcpt, recipients_list); 4047 } 4048 4049 /** 4050 Sets the message body text. 4051 */ 4052 4053 @property void message(string msg) 4054 { 4055 p.message = msg; 4056 } 4057 } 4058 4059 @system unittest 4060 { 4061 import std.net.curl; 4062 4063 // Send an email with SMTPS 4064 auto smtp = SMTP("smtps://smtp.gmail.com"); 4065 smtp.setAuthentication("from.addr@gmail.com", "password"); 4066 smtp.mailTo = ["<to.addr@gmail.com>"]; 4067 smtp.mailFrom = "<from.addr@gmail.com>"; 4068 smtp.message = "Example Message"; 4069 //smtp.perform(); 4070 } 4071 4072 4073 /++ 4074 Exception thrown on errors in std.net.curl functions. 4075 +/ 4076 class CurlException : Exception 4077 { 4078 /++ 4079 Params: 4080 msg = The message for the exception. 4081 file = The file where the exception occurred. 4082 line = The line number where the exception occurred. 4083 next = The previous exception in the chain of exceptions, if any. 4084 +/ 4085 @safe pure nothrow 4086 this(string msg, 4087 string file = __FILE__, 4088 size_t line = __LINE__, 4089 Throwable next = null) 4090 { 4091 super(msg, file, line, next); 4092 } 4093 } 4094 4095 /++ 4096 Exception thrown on timeout errors in std.net.curl functions. 4097 +/ 4098 class CurlTimeoutException : CurlException 4099 { 4100 /++ 4101 Params: 4102 msg = The message for the exception. 4103 file = The file where the exception occurred. 4104 line = The line number where the exception occurred. 4105 next = The previous exception in the chain of exceptions, if any. 4106 +/ 4107 @safe pure nothrow 4108 this(string msg, 4109 string file = __FILE__, 4110 size_t line = __LINE__, 4111 Throwable next = null) 4112 { 4113 super(msg, file, line, next); 4114 } 4115 } 4116 4117 /++ 4118 Exception thrown on HTTP request failures, e.g. 404 Not Found. 4119 +/ 4120 class HTTPStatusException : CurlException 4121 { 4122 /++ 4123 Params: 4124 status = The HTTP status code. 4125 msg = The message for the exception. 4126 file = The file where the exception occurred. 4127 line = The line number where the exception occurred. 4128 next = The previous exception in the chain of exceptions, if any. 4129 +/ 4130 @safe pure nothrow 4131 this(int status, 4132 string msg, 4133 string file = __FILE__, 4134 size_t line = __LINE__, 4135 Throwable next = null) 4136 { 4137 super(msg, file, line, next); 4138 this.status = status; 4139 } 4140 4141 immutable int status; /// The HTTP status code 4142 } 4143 4144 /// Equal to $(REF CURLcode, etc,c,curl) 4145 alias CurlCode = CURLcode; 4146 4147 /// Flag to specify whether or not an exception is thrown on error. 4148 alias ThrowOnError = Flag!"throwOnError"; 4149 4150 private struct CurlAPI 4151 { 4152 import etc.c.curl : CurlGlobal; 4153 static struct API 4154 { 4155 import etc.c.curl : curl_version_info, curl_version_info_data, 4156 CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist; 4157 extern(C): 4158 import core.stdc.config : c_long; 4159 CURLcode function(c_long flags) global_init; 4160 void function() global_cleanup; 4161 curl_version_info_data * function(CURLversion) version_info; 4162 CURL* function() easy_init; 4163 CURLcode function(CURL *curl, CURLoption option,...) easy_setopt; 4164 CURLcode function(CURL *curl) easy_perform; 4165 CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo; 4166 CURL* function(CURL *curl) easy_duphandle; 4167 char* function(CURLcode) easy_strerror; 4168 CURLcode function(CURL *handle, int bitmask) easy_pause; 4169 void function(CURL *curl) easy_cleanup; 4170 curl_slist* function(curl_slist *, char *) slist_append; 4171 void function(curl_slist *) slist_free_all; 4172 } 4173 __gshared API _api; 4174 __gshared void* _handle; 4175 4176 static ref API instance() @property 4177 { 4178 import std.concurrency : initOnce; 4179 initOnce!_handle(loadAPI()); 4180 return _api; 4181 } 4182 4183 static void* loadAPI() 4184 { 4185 import std.exception : enforce; 4186 4187 version (Posix) 4188 { 4189 import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY; 4190 alias loadSym = dlsym; 4191 } 4192 else version (Windows) 4193 { 4194 import core.sys.windows.winbase : GetProcAddress, GetModuleHandleA, 4195 LoadLibraryA; 4196 alias loadSym = GetProcAddress; 4197 } 4198 else 4199 static assert(0, "unimplemented"); 4200 4201 void* handle; 4202 version (Posix) 4203 handle = dlopen(null, RTLD_LAZY); 4204 else version (Windows) 4205 handle = GetModuleHandleA(null); 4206 assert(handle !is null); 4207 4208 // try to load curl from the executable to allow static linking 4209 if (loadSym(handle, "curl_global_init") is null) 4210 { 4211 import std.format : format; 4212 version (Posix) 4213 dlclose(handle); 4214 4215 version (LibcurlPath) 4216 { 4217 import std..string : strip; 4218 static immutable names = [strip(import("LibcurlPathFile"))]; 4219 } 4220 else version (OSX) 4221 static immutable names = ["libcurl.4.dylib"]; 4222 else version (Posix) 4223 { 4224 static immutable names = ["libcurl.so", "libcurl.so.4", 4225 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"]; 4226 } 4227 else version (Windows) 4228 static immutable names = ["libcurl.dll", "curl.dll"]; 4229 4230 foreach (name; names) 4231 { 4232 version (Posix) 4233 handle = dlopen(name.ptr, RTLD_LAZY); 4234 else version (Windows) 4235 handle = LoadLibraryA(name.ptr); 4236 if (handle !is null) break; 4237 } 4238 4239 enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); 4240 } 4241 4242 foreach (i, FP; typeof(API.tupleof)) 4243 { 4244 enum name = __traits(identifier, _api.tupleof[i]); 4245 auto p = enforce!CurlException(loadSym(handle, "curl_"~name), 4246 "Couldn't load curl_"~name~" from libcurl."); 4247 _api.tupleof[i] = cast(FP) p; 4248 } 4249 4250 enforce!CurlException(!_api.global_init(CurlGlobal.all), 4251 "Failed to initialize libcurl"); 4252 4253 static extern(C) void cleanup() 4254 { 4255 if (_handle is null) return; 4256 _api.global_cleanup(); 4257 version (Posix) 4258 { 4259 import core.sys.posix.dlfcn : dlclose; 4260 dlclose(_handle); 4261 } 4262 else version (Windows) 4263 { 4264 import core.sys.windows.winbase : FreeLibrary; 4265 FreeLibrary(_handle); 4266 } 4267 else 4268 static assert(0, "unimplemented"); 4269 _api = API.init; 4270 _handle = null; 4271 } 4272 4273 import core.stdc.stdlib : atexit; 4274 atexit(&cleanup); 4275 4276 return handle; 4277 } 4278 } 4279 4280 /** 4281 Wrapper to provide a better interface to libcurl than using the plain C API. 4282 It is recommended to use the `HTTP`/`FTP` etc. structs instead unless 4283 raw access to libcurl is needed. 4284 4285 Warning: This struct uses interior pointers for callbacks. Only allocate it 4286 on the stack if you never move or copy it. This also means passing by reference 4287 when passing Curl to other functions. Otherwise always allocate on 4288 the heap. 4289 */ 4290 struct Curl 4291 { 4292 import etc.c.curl : CURL, CurlError, CurlPause, CurlSeek, CurlSeekPos, 4293 curl_socket_t, CurlSockType, 4294 CurlReadFunc, CurlInfo, curlsocktype, curl_off_t, 4295 LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR, LIBCURL_VERSION_PATCH; 4296 4297 alias OutData = void[]; 4298 alias InData = ubyte[]; 4299 private bool _stopped; 4300 4301 private static auto ref curl() @property { return CurlAPI.instance; } 4302 4303 // A handle should not be used by two threads simultaneously 4304 private CURL* handle; 4305 4306 // May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE` 4307 private size_t delegate(OutData) _onSend; 4308 private size_t delegate(InData) _onReceive; 4309 private void delegate(in char[]) _onReceiveHeader; 4310 private CurlSeek delegate(long,CurlSeekPos) _onSeek; 4311 private int delegate(curl_socket_t,CurlSockType) _onSocketOption; 4312 private int delegate(size_t dltotal, size_t dlnow, 4313 size_t ultotal, size_t ulnow) _onProgress; 4314 4315 alias requestPause = CurlReadFunc.pause; 4316 alias requestAbort = CurlReadFunc.abort; 4317 4318 /** 4319 Initialize the instance by creating a working curl handle. 4320 */ 4321 void initialize() 4322 { 4323 import std.exception : enforce; 4324 enforce!CurlException(!handle, "Curl instance already initialized"); 4325 handle = curl.easy_init(); 4326 enforce!CurlException(handle, "Curl instance couldn't be initialized"); 4327 _stopped = false; 4328 set(CurlOption.nosignal, 1); 4329 } 4330 4331 /// 4332 @property bool stopped() const 4333 { 4334 return _stopped; 4335 } 4336 4337 /** 4338 Duplicate this handle. 4339 4340 The new handle will have all options set as the one it was duplicated 4341 from. An exception to this is that all options that cannot be shared 4342 across threads are reset thereby making it safe to use the duplicate 4343 in a new thread. 4344 */ 4345 Curl dup() 4346 { 4347 import std.meta : AliasSeq; 4348 Curl copy; 4349 copy.handle = curl.easy_duphandle(handle); 4350 copy._stopped = false; 4351 4352 with (CurlOption) { 4353 auto tt = AliasSeq!(file, writefunction, writeheader, 4354 headerfunction, infile, readfunction, ioctldata, ioctlfunction, 4355 seekdata, seekfunction, sockoptdata, sockoptfunction, 4356 opensocketdata, opensocketfunction, progressdata, 4357 progressfunction, debugdata, debugfunction, interleavedata, 4358 interleavefunction, chunk_data, chunk_bgn_function, 4359 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields); 4360 4361 foreach (option; tt) 4362 copy.clear(option); 4363 } 4364 4365 // The options are only supported by libcurl when it has been built 4366 // against certain versions of OpenSSL - if your libcurl uses an old 4367 // OpenSSL, or uses an entirely different SSL engine, attempting to 4368 // clear these normally will raise an exception 4369 copy.clearIfSupported(CurlOption.ssl_ctx_function); 4370 copy.clearIfSupported(CurlOption.ssh_keydata); 4371 4372 // Enable for curl version > 7.21.7 4373 static if (LIBCURL_VERSION_MAJOR >= 7 && 4374 LIBCURL_VERSION_MINOR >= 21 && 4375 LIBCURL_VERSION_PATCH >= 7) 4376 { 4377 copy.clear(CurlOption.closesocketdata); 4378 copy.clear(CurlOption.closesocketfunction); 4379 } 4380 4381 copy.set(CurlOption.nosignal, 1); 4382 4383 // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared 4384 // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared 4385 4386 /* 4387 Allow sharing of conv functions 4388 copy.clear(CurlOption.conv_to_network_function); 4389 copy.clear(CurlOption.conv_from_network_function); 4390 copy.clear(CurlOption.conv_from_utf8_function); 4391 */ 4392 4393 return copy; 4394 } 4395 4396 private void _check(CurlCode code) 4397 { 4398 import std.exception : enforce; 4399 enforce!CurlTimeoutException(code != CurlError.operation_timedout, 4400 errorString(code)); 4401 4402 enforce!CurlException(code == CurlError.ok, 4403 errorString(code)); 4404 } 4405 4406 private string errorString(CurlCode code) 4407 { 4408 import core.stdc..string : strlen; 4409 import std.format : format; 4410 4411 auto msgZ = curl.easy_strerror(code); 4412 // doing the following (instead of just using std.conv.to!string) avoids 1 allocation 4413 return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle); 4414 } 4415 4416 private void throwOnStopped(string message = null) 4417 { 4418 import std.exception : enforce; 4419 auto def = "Curl instance called after being cleaned up"; 4420 enforce!CurlException(!stopped, 4421 message == null ? def : message); 4422 } 4423 4424 /** 4425 Stop and invalidate this curl instance. 4426 Warning: Do not call this from inside a callback handler e.g. `onReceive`. 4427 */ 4428 void shutdown() 4429 { 4430 throwOnStopped(); 4431 _stopped = true; 4432 curl.easy_cleanup(this.handle); 4433 this.handle = null; 4434 } 4435 4436 /** 4437 Pausing and continuing transfers. 4438 */ 4439 void pause(bool sendingPaused, bool receivingPaused) 4440 { 4441 throwOnStopped(); 4442 _check(curl.easy_pause(this.handle, 4443 (sendingPaused ? CurlPause.send_cont : CurlPause.send) | 4444 (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); 4445 } 4446 4447 /** 4448 Set a string curl option. 4449 Params: 4450 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4451 value = The string 4452 */ 4453 void set(CurlOption option, const(char)[] value) 4454 { 4455 import std.internal.cstring : tempCString; 4456 throwOnStopped(); 4457 _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr)); 4458 } 4459 4460 /** 4461 Set a long curl option. 4462 Params: 4463 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4464 value = The long 4465 */ 4466 void set(CurlOption option, long value) 4467 { 4468 throwOnStopped(); 4469 _check(curl.easy_setopt(this.handle, option, value)); 4470 } 4471 4472 /** 4473 Set a void* curl option. 4474 Params: 4475 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4476 value = The pointer 4477 */ 4478 void set(CurlOption option, void* value) 4479 { 4480 throwOnStopped(); 4481 _check(curl.easy_setopt(this.handle, option, value)); 4482 } 4483 4484 /** 4485 Clear a pointer option. 4486 Params: 4487 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4488 */ 4489 void clear(CurlOption option) 4490 { 4491 throwOnStopped(); 4492 _check(curl.easy_setopt(this.handle, option, null)); 4493 } 4494 4495 /** 4496 Clear a pointer option. Does not raise an exception if the underlying 4497 libcurl does not support the option. Use sparingly. 4498 Params: 4499 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4500 */ 4501 void clearIfSupported(CurlOption option) 4502 { 4503 throwOnStopped(); 4504 auto rval = curl.easy_setopt(this.handle, option, null); 4505 if (rval != CurlError.unknown_option && rval != CurlError.not_built_in) 4506 _check(rval); 4507 } 4508 4509 /** 4510 perform the curl request by doing the HTTP,FTP etc. as it has 4511 been setup beforehand. 4512 4513 Params: 4514 throwOnError = whether to throw an exception or return a CurlCode on error 4515 */ 4516 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 4517 { 4518 throwOnStopped(); 4519 CurlCode code = curl.easy_perform(this.handle); 4520 if (throwOnError) 4521 _check(code); 4522 return code; 4523 } 4524 4525 /** 4526 Get the various timings like name lookup time, total time, connect time etc. 4527 The timed category is passed through the timing parameter while the timing 4528 value is stored at val. The value is usable only if res is equal to 4529 `etc.c.curl.CurlError.ok`. 4530 */ 4531 CurlCode getTiming(CurlInfo timing, ref double val) 4532 { 4533 CurlCode code; 4534 code = curl.easy_getinfo(handle, timing, &val); 4535 return code; 4536 } 4537 4538 /** 4539 * The event handler that receives incoming data. 4540 * 4541 * Params: 4542 * callback = the callback that receives the `ubyte[]` data. 4543 * Be sure to copy the incoming data and not store 4544 * a slice. 4545 * 4546 * Returns: 4547 * The callback returns the incoming bytes read. If not the entire array is 4548 * the request will abort. 4549 * The special value HTTP.pauseRequest can be returned in order to pause the 4550 * current request. 4551 * 4552 * Example: 4553 * ---- 4554 * import std.net.curl, std.stdio; 4555 * Curl curl; 4556 * curl.initialize(); 4557 * curl.set(CurlOption.url, "http://dlang.org"); 4558 * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; 4559 * curl.perform(); 4560 * ---- 4561 */ 4562 @property void onReceive(size_t delegate(InData) callback) 4563 { 4564 _onReceive = (InData id) 4565 { 4566 throwOnStopped("Receive callback called on cleaned up Curl instance"); 4567 return callback(id); 4568 }; 4569 set(CurlOption.file, cast(void*) &this); 4570 set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback); 4571 } 4572 4573 /** 4574 * The event handler that receives incoming headers for protocols 4575 * that uses headers. 4576 * 4577 * Params: 4578 * callback = the callback that receives the header string. 4579 * Make sure the callback copies the incoming params if 4580 * it needs to store it because they are references into 4581 * the backend and may very likely change. 4582 * 4583 * Example: 4584 * ---- 4585 * import std.net.curl, std.stdio; 4586 * Curl curl; 4587 * curl.initialize(); 4588 * curl.set(CurlOption.url, "http://dlang.org"); 4589 * curl.onReceiveHeader = (in char[] header) { writeln(header); }; 4590 * curl.perform(); 4591 * ---- 4592 */ 4593 @property void onReceiveHeader(void delegate(in char[]) callback) 4594 { 4595 _onReceiveHeader = (in char[] od) 4596 { 4597 throwOnStopped("Receive header callback called on "~ 4598 "cleaned up Curl instance"); 4599 callback(od); 4600 }; 4601 set(CurlOption.writeheader, cast(void*) &this); 4602 set(CurlOption.headerfunction, 4603 cast(void*) &Curl._receiveHeaderCallback); 4604 } 4605 4606 /** 4607 * The event handler that gets called when data is needed for sending. 4608 * 4609 * Params: 4610 * callback = the callback that has a `void[]` buffer to be filled 4611 * 4612 * Returns: 4613 * The callback returns the number of elements in the buffer that have been 4614 * filled and are ready to send. 4615 * The special value `Curl.abortRequest` can be returned in 4616 * order to abort the current request. 4617 * The special value `Curl.pauseRequest` can be returned in order to 4618 * pause the current request. 4619 * 4620 * Example: 4621 * ---- 4622 * import std.net.curl; 4623 * Curl curl; 4624 * curl.initialize(); 4625 * curl.set(CurlOption.url, "http://dlang.org"); 4626 * 4627 * string msg = "Hello world"; 4628 * curl.onSend = (void[] data) 4629 * { 4630 * auto m = cast(void[]) msg; 4631 * size_t length = m.length > data.length ? data.length : m.length; 4632 * if (length == 0) return 0; 4633 * data[0 .. length] = m[0 .. length]; 4634 * msg = msg[length..$]; 4635 * return length; 4636 * }; 4637 * curl.perform(); 4638 * ---- 4639 */ 4640 @property void onSend(size_t delegate(OutData) callback) 4641 { 4642 _onSend = (OutData od) 4643 { 4644 throwOnStopped("Send callback called on cleaned up Curl instance"); 4645 return callback(od); 4646 }; 4647 set(CurlOption.infile, cast(void*) &this); 4648 set(CurlOption.readfunction, cast(void*) &Curl._sendCallback); 4649 } 4650 4651 /** 4652 * The event handler that gets called when the curl backend needs to seek 4653 * the data to be sent. 4654 * 4655 * Params: 4656 * callback = the callback that receives a seek offset and a seek position 4657 * $(REF CurlSeekPos, etc,c,curl) 4658 * 4659 * Returns: 4660 * The callback returns the success state of the seeking 4661 * $(REF CurlSeek, etc,c,curl) 4662 * 4663 * Example: 4664 * ---- 4665 * import std.net.curl; 4666 * Curl curl; 4667 * curl.initialize(); 4668 * curl.set(CurlOption.url, "http://dlang.org"); 4669 * curl.onSeek = (long p, CurlSeekPos sp) 4670 * { 4671 * return CurlSeek.cantseek; 4672 * }; 4673 * curl.perform(); 4674 * ---- 4675 */ 4676 @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback) 4677 { 4678 _onSeek = (long ofs, CurlSeekPos sp) 4679 { 4680 throwOnStopped("Seek callback called on cleaned up Curl instance"); 4681 return callback(ofs, sp); 4682 }; 4683 set(CurlOption.seekdata, cast(void*) &this); 4684 set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback); 4685 } 4686 4687 /** 4688 * The event handler that gets called when the net socket has been created 4689 * but a `connect()` call has not yet been done. This makes it possible to set 4690 * misc. socket options. 4691 * 4692 * Params: 4693 * callback = the callback that receives the socket and socket type 4694 * $(REF CurlSockType, etc,c,curl) 4695 * 4696 * Returns: 4697 * Return 0 from the callback to signal success, return 1 to signal error 4698 * and make curl close the socket 4699 * 4700 * Example: 4701 * ---- 4702 * import std.net.curl; 4703 * Curl curl; 4704 * curl.initialize(); 4705 * curl.set(CurlOption.url, "http://dlang.org"); 4706 * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; 4707 * curl.perform(); 4708 * ---- 4709 */ 4710 @property void onSocketOption(int delegate(curl_socket_t, 4711 CurlSockType) callback) 4712 { 4713 _onSocketOption = (curl_socket_t sock, CurlSockType st) 4714 { 4715 throwOnStopped("Socket option callback called on "~ 4716 "cleaned up Curl instance"); 4717 return callback(sock, st); 4718 }; 4719 set(CurlOption.sockoptdata, cast(void*) &this); 4720 set(CurlOption.sockoptfunction, 4721 cast(void*) &Curl._socketOptionCallback); 4722 } 4723 4724 /** 4725 * The event handler that gets called to inform of upload/download progress. 4726 * 4727 * Params: 4728 * callback = the callback that receives the (total bytes to download, 4729 * currently downloaded bytes, total bytes to upload, currently uploaded 4730 * bytes). 4731 * 4732 * Returns: 4733 * Return 0 from the callback to signal success, return non-zero to abort 4734 * transfer 4735 * 4736 * Example: 4737 * ---- 4738 * import std.net.curl, std.stdio; 4739 * Curl curl; 4740 * curl.initialize(); 4741 * curl.set(CurlOption.url, "http://dlang.org"); 4742 * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) 4743 * { 4744 * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); 4745 * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); 4746 * return 0; 4747 * }; 4748 * curl.perform(); 4749 * ---- 4750 */ 4751 @property void onProgress(int delegate(size_t dlTotal, 4752 size_t dlNow, 4753 size_t ulTotal, 4754 size_t ulNow) callback) 4755 { 4756 _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) 4757 { 4758 throwOnStopped("Progress callback called on cleaned "~ 4759 "up Curl instance"); 4760 return callback(dlt, dln, ult, uln); 4761 }; 4762 set(CurlOption.noprogress, 0); 4763 set(CurlOption.progressdata, cast(void*) &this); 4764 set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback); 4765 } 4766 4767 // Internal C callbacks to register with libcurl 4768 extern (C) private static 4769 size_t _receiveCallback(const char* str, 4770 size_t size, size_t nmemb, void* ptr) 4771 { 4772 auto b = cast(Curl*) ptr; 4773 if (b._onReceive != null) 4774 return b._onReceive(cast(InData)(str[0 .. size*nmemb])); 4775 return size*nmemb; 4776 } 4777 4778 extern (C) private static 4779 size_t _receiveHeaderCallback(const char* str, 4780 size_t size, size_t nmemb, void* ptr) 4781 { 4782 import std..string : chomp; 4783 4784 auto b = cast(Curl*) ptr; 4785 auto s = str[0 .. size*nmemb].chomp(); 4786 if (b._onReceiveHeader != null) 4787 b._onReceiveHeader(s); 4788 4789 return size*nmemb; 4790 } 4791 4792 extern (C) private static 4793 size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr) 4794 { 4795 Curl* b = cast(Curl*) ptr; 4796 auto a = cast(void[]) str[0 .. size*nmemb]; 4797 if (b._onSend == null) 4798 return 0; 4799 return b._onSend(a); 4800 } 4801 4802 extern (C) private static 4803 int _seekCallback(void *ptr, curl_off_t offset, int origin) 4804 { 4805 auto b = cast(Curl*) ptr; 4806 if (b._onSeek == null) 4807 return CurlSeek.cantseek; 4808 4809 // origin: CurlSeekPos.set/current/end 4810 // return: CurlSeek.ok/fail/cantseek 4811 return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin); 4812 } 4813 4814 extern (C) private static 4815 int _socketOptionCallback(void *ptr, 4816 curl_socket_t curlfd, curlsocktype purpose) 4817 { 4818 auto b = cast(Curl*) ptr; 4819 if (b._onSocketOption == null) 4820 return 0; 4821 4822 // return: 0 ok, 1 fail 4823 return b._onSocketOption(curlfd, cast(CurlSockType) purpose); 4824 } 4825 4826 extern (C) private static 4827 int _progressCallback(void *ptr, 4828 double dltotal, double dlnow, 4829 double ultotal, double ulnow) 4830 { 4831 auto b = cast(Curl*) ptr; 4832 if (b._onProgress == null) 4833 return 0; 4834 4835 // return: 0 ok, 1 fail 4836 return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow, 4837 cast(size_t) ultotal, cast(size_t) ulnow); 4838 } 4839 4840 } 4841 4842 // Internal messages send between threads. 4843 // The data is wrapped in this struct in order to ensure that 4844 // other std.concurrency.receive calls does not pick up our messages 4845 // by accident. 4846 private struct CurlMessage(T) 4847 { 4848 public T data; 4849 } 4850 4851 private static CurlMessage!T curlMessage(T)(T data) 4852 { 4853 return CurlMessage!T(data); 4854 } 4855 4856 // Pool of to be used for reusing buffers 4857 private struct Pool(Data) 4858 { 4859 private struct Entry 4860 { 4861 Data data; 4862 Entry* next; 4863 } 4864 private Entry* root; 4865 private Entry* freeList; 4866 4867 @safe @property bool empty() 4868 { 4869 return root == null; 4870 } 4871 4872 @safe nothrow void push(Data d) 4873 { 4874 if (freeList == null) 4875 { 4876 // Allocate new Entry since there is no one 4877 // available in the freeList 4878 freeList = new Entry; 4879 } 4880 freeList.data = d; 4881 Entry* oldroot = root; 4882 root = freeList; 4883 freeList = freeList.next; 4884 root.next = oldroot; 4885 } 4886 4887 @safe Data pop() 4888 { 4889 import std.exception : enforce; 4890 enforce!Exception(root != null, "pop() called on empty pool"); 4891 auto d = root.data; 4892 auto n = root.next; 4893 root.next = freeList; 4894 freeList = root; 4895 root = n; 4896 return d; 4897 } 4898 } 4899 4900 // Lazily-instantiated namespace to avoid importing std.concurrency until needed. 4901 private struct _async() 4902 { 4903 static: 4904 // https://issues.dlang.org/show_bug.cgi?id=15831 4905 // this should be inside byLineAsync 4906 // Range that reads one chunk at a time asynchronously. 4907 private struct ChunkInputRange 4908 { 4909 import std.concurrency : Tid, send; 4910 4911 private ubyte[] chunk; 4912 mixin WorkerThreadProtocol!(ubyte, chunk); 4913 4914 private Tid workerTid; 4915 private State running; 4916 4917 private this(Tid tid, size_t transmitBuffers, size_t chunkSize) 4918 { 4919 workerTid = tid; 4920 state = State.needUnits; 4921 4922 // Send buffers to other thread for it to use. Since no mechanism is in 4923 // place for moving ownership a cast to shared is done here and a cast 4924 // back to non-shared in the receiving end. 4925 foreach (i ; 0 .. transmitBuffers) 4926 { 4927 ubyte[] arr = new ubyte[](chunkSize); 4928 workerTid.send(cast(immutable(ubyte[]))arr); 4929 } 4930 } 4931 } 4932 4933 // https://issues.dlang.org/show_bug.cgi?id=15831 4934 // this should be inside byLineAsync 4935 // Range that reads one line at a time asynchronously. 4936 private static struct LineInputRange(Char) 4937 { 4938 private Char[] line; 4939 mixin WorkerThreadProtocol!(Char, line); 4940 4941 private Tid workerTid; 4942 private State running; 4943 4944 private this(Tid tid, size_t transmitBuffers, size_t bufferSize) 4945 { 4946 import std.concurrency : send; 4947 4948 workerTid = tid; 4949 state = State.needUnits; 4950 4951 // Send buffers to other thread for it to use. Since no mechanism is in 4952 // place for moving ownership a cast to shared is done here and casted 4953 // back to non-shared in the receiving end. 4954 foreach (i ; 0 .. transmitBuffers) 4955 { 4956 auto arr = new Char[](bufferSize); 4957 workerTid.send(cast(immutable(Char[]))arr); 4958 } 4959 } 4960 } 4961 4962 import std.concurrency : Tid; 4963 4964 // Shared function for reading incoming chunks of data and 4965 // sending the to a parent thread 4966 private size_t receiveChunks(ubyte[] data, ref ubyte[] outdata, 4967 Pool!(ubyte[]) freeBuffers, 4968 ref ubyte[] buffer, Tid fromTid, 4969 ref bool aborted) 4970 { 4971 import std.concurrency : receive, send, thisTid; 4972 4973 immutable datalen = data.length; 4974 4975 // Copy data to fill active buffer 4976 while (!data.empty) 4977 { 4978 4979 // Make sure a buffer is present 4980 while ( outdata.empty && freeBuffers.empty) 4981 { 4982 // Active buffer is invalid and there are no 4983 // available buffers in the pool. Wait for buffers 4984 // to return from main thread in order to reuse 4985 // them. 4986 receive((immutable(ubyte)[] buf) 4987 { 4988 buffer = cast(ubyte[]) buf; 4989 outdata = buffer[]; 4990 }, 4991 (bool flag) { aborted = true; } 4992 ); 4993 if (aborted) return cast(size_t) 0; 4994 } 4995 if (outdata.empty) 4996 { 4997 buffer = freeBuffers.pop(); 4998 outdata = buffer[]; 4999 } 5000 5001 // Copy data 5002 auto copyBytes = outdata.length < data.length ? 5003 outdata.length : data.length; 5004 5005 outdata[0 .. copyBytes] = data[0 .. copyBytes]; 5006 outdata = outdata[copyBytes..$]; 5007 data = data[copyBytes..$]; 5008 5009 if (outdata.empty) 5010 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 5011 } 5012 5013 return datalen; 5014 } 5015 5016 // ditto 5017 private void finalizeChunks(ubyte[] outdata, ref ubyte[] buffer, 5018 Tid fromTid) 5019 { 5020 import std.concurrency : send, thisTid; 5021 if (!outdata.empty) 5022 { 5023 // Resize the last buffer 5024 buffer.length = buffer.length - outdata.length; 5025 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 5026 } 5027 } 5028 5029 5030 // Shared function for reading incoming lines of data and sending the to a 5031 // parent thread 5032 private static size_t receiveLines(Terminator, Unit) 5033 (const(ubyte)[] data, ref EncodingScheme encodingScheme, 5034 bool keepTerminator, Terminator terminator, 5035 ref const(ubyte)[] leftOverBytes, ref bool bufferValid, 5036 ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer, 5037 Tid fromTid, ref bool aborted) 5038 { 5039 import std.concurrency : prioritySend, receive, send, thisTid; 5040 import std.exception : enforce; 5041 import std.format : format; 5042 import std.traits : isArray; 5043 5044 immutable datalen = data.length; 5045 5046 // Terminator is specified and buffers should be resized as determined by 5047 // the terminator 5048 5049 // Copy data to active buffer until terminator is found. 5050 5051 // Decode as many lines as possible 5052 while (true) 5053 { 5054 5055 // Make sure a buffer is present 5056 while (!bufferValid && freeBuffers.empty) 5057 { 5058 // Active buffer is invalid and there are no available buffers in 5059 // the pool. Wait for buffers to return from main thread in order to 5060 // reuse them. 5061 receive((immutable(Unit)[] buf) 5062 { 5063 buffer = cast(Unit[]) buf; 5064 buffer.length = 0; 5065 buffer.assumeSafeAppend(); 5066 bufferValid = true; 5067 }, 5068 (bool flag) { aborted = true; } 5069 ); 5070 if (aborted) return cast(size_t) 0; 5071 } 5072 if (!bufferValid) 5073 { 5074 buffer = freeBuffers.pop(); 5075 bufferValid = true; 5076 } 5077 5078 // Try to read a line from left over bytes from last onReceive plus the 5079 // newly received bytes. 5080 try 5081 { 5082 if (decodeLineInto(leftOverBytes, data, buffer, 5083 encodingScheme, terminator)) 5084 { 5085 if (keepTerminator) 5086 { 5087 fromTid.send(thisTid, 5088 curlMessage(cast(immutable(Unit)[])buffer)); 5089 } 5090 else 5091 { 5092 static if (isArray!Terminator) 5093 fromTid.send(thisTid, 5094 curlMessage(cast(immutable(Unit)[]) 5095 buffer[0..$-terminator.length])); 5096 else 5097 fromTid.send(thisTid, 5098 curlMessage(cast(immutable(Unit)[]) 5099 buffer[0..$-1])); 5100 } 5101 bufferValid = false; 5102 } 5103 else 5104 { 5105 // Could not decode an entire line. Save 5106 // bytes left in data for next call to 5107 // onReceive. Can be up to a max of 4 bytes. 5108 enforce!CurlException(data.length <= 4, 5109 format( 5110 "Too many bytes left not decoded %s"~ 5111 " > 4. Maybe the charset specified in"~ 5112 " headers does not match "~ 5113 "the actual content downloaded?", 5114 data.length)); 5115 leftOverBytes ~= data; 5116 break; 5117 } 5118 } 5119 catch (CurlException ex) 5120 { 5121 prioritySend(fromTid, cast(immutable(CurlException))ex); 5122 return cast(size_t) 0; 5123 } 5124 } 5125 return datalen; 5126 } 5127 5128 // ditto 5129 private static 5130 void finalizeLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid) 5131 { 5132 import std.concurrency : send, thisTid; 5133 if (bufferValid && buffer.length != 0) 5134 fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$])); 5135 } 5136 5137 /* Used by byLineAsync/byChunkAsync to duplicate an existing connection 5138 * that can be used exclusively in a spawned thread. 5139 */ 5140 private void duplicateConnection(Conn, PostData) 5141 (const(char)[] url, Conn conn, PostData postData, Tid tid) 5142 { 5143 import std.concurrency : send; 5144 import std.exception : enforce; 5145 5146 // no move semantic available in std.concurrency ie. must use casting. 5147 auto connDup = conn.dup(); 5148 connDup.url = url; 5149 5150 static if ( is(Conn : HTTP) ) 5151 { 5152 connDup.p.headersOut = null; 5153 connDup.method = conn.method == HTTP.Method.undefined ? 5154 HTTP.Method.get : conn.method; 5155 if (postData !is null) 5156 { 5157 if (connDup.method == HTTP.Method.put) 5158 { 5159 connDup.handle.set(CurlOption.infilesize_large, 5160 postData.length); 5161 } 5162 else 5163 { 5164 // post 5165 connDup.method = HTTP.Method.post; 5166 connDup.handle.set(CurlOption.postfieldsize_large, 5167 postData.length); 5168 } 5169 connDup.handle.set(CurlOption.copypostfields, 5170 cast(void*) postData.ptr); 5171 } 5172 tid.send(cast(ulong) connDup.handle.handle); 5173 tid.send(connDup.method); 5174 } 5175 else 5176 { 5177 enforce!CurlException(postData is null, 5178 "Cannot put ftp data using byLineAsync()"); 5179 tid.send(cast(ulong) connDup.handle.handle); 5180 tid.send(HTTP.Method.undefined); 5181 } 5182 connDup.p.curl.handle = null; // make sure handle is not freed 5183 } 5184 5185 // Spawn a thread for handling the reading of incoming data in the 5186 // background while the delegate is executing. This will optimize 5187 // throughput by allowing simultaneous input (this struct) and 5188 // output (e.g. AsyncHTTPLineOutputRange). 5189 private static void spawn(Conn, Unit, Terminator = void)() 5190 { 5191 import std.concurrency : Tid, prioritySend, receiveOnly, send, thisTid; 5192 import etc.c.curl : CURL, CurlError; 5193 Tid fromTid = receiveOnly!Tid(); 5194 5195 // Get buffer to read into 5196 Pool!(Unit[]) freeBuffers; // Free list of buffer objects 5197 5198 // Number of bytes filled into active buffer 5199 Unit[] buffer; 5200 bool aborted = false; 5201 5202 EncodingScheme encodingScheme; 5203 static if ( !is(Terminator == void)) 5204 { 5205 // Only lines reading will receive a terminator 5206 const terminator = receiveOnly!Terminator(); 5207 const keepTerminator = receiveOnly!bool(); 5208 5209 // max number of bytes to carry over from an onReceive 5210 // callback. This is 4 because it is the max code units to 5211 // decode a code point in the supported encodings. 5212 auto leftOverBytes = new const(ubyte)[4]; 5213 leftOverBytes.length = 0; 5214 auto bufferValid = false; 5215 } 5216 else 5217 { 5218 Unit[] outdata; 5219 } 5220 5221 // no move semantic available in std.concurrency ie. must use casting. 5222 auto connDup = cast(CURL*) receiveOnly!ulong(); 5223 auto client = Conn(); 5224 client.p.curl.handle = connDup; 5225 5226 // receive a method for both ftp and http but just use it for http 5227 auto method = receiveOnly!(HTTP.Method)(); 5228 5229 client.onReceive = (ubyte[] data) 5230 { 5231 // If no terminator is specified the chunk size is fixed. 5232 static if ( is(Terminator == void) ) 5233 return receiveChunks(data, outdata, freeBuffers, buffer, 5234 fromTid, aborted); 5235 else 5236 return receiveLines(data, encodingScheme, 5237 keepTerminator, terminator, leftOverBytes, 5238 bufferValid, freeBuffers, buffer, 5239 fromTid, aborted); 5240 }; 5241 5242 static if ( is(Conn == HTTP) ) 5243 { 5244 client.method = method; 5245 // register dummy header handler 5246 client.onReceiveHeader = (in char[] key, in char[] value) 5247 { 5248 if (key == "content-type") 5249 encodingScheme = EncodingScheme.create(client.p.charset); 5250 }; 5251 } 5252 else 5253 { 5254 encodingScheme = EncodingScheme.create(client.encoding); 5255 } 5256 5257 // Start the request 5258 CurlCode code; 5259 try 5260 { 5261 code = client.perform(No.throwOnError); 5262 } 5263 catch (Exception ex) 5264 { 5265 prioritySend(fromTid, cast(immutable(Exception)) ex); 5266 fromTid.send(thisTid, curlMessage(true)); // signal done 5267 return; 5268 } 5269 5270 if (code != CurlError.ok) 5271 { 5272 if (aborted && (code == CurlError.aborted_by_callback || 5273 code == CurlError.write_error)) 5274 { 5275 fromTid.send(thisTid, curlMessage(true)); // signal done 5276 return; 5277 } 5278 prioritySend(fromTid, cast(immutable(CurlException)) 5279 new CurlException(client.p.curl.errorString(code))); 5280 5281 fromTid.send(thisTid, curlMessage(true)); // signal done 5282 return; 5283 } 5284 5285 // Send remaining data that is not a full chunk size 5286 static if ( is(Terminator == void) ) 5287 finalizeChunks(outdata, buffer, fromTid); 5288 else 5289 finalizeLines(bufferValid, buffer, fromTid); 5290 5291 fromTid.send(thisTid, curlMessage(true)); // signal done 5292 } 5293 }