1 // Written in the D programming language.
2 
3 /**
4 Networking client functionality as provided by $(HTTP curl.haxx.se/libcurl,
5 libcurl). The libcurl library must be installed on the system in order to use
6 this module.
7 
8 $(SCRIPT inhibitQuickIndex = 1;)
9 
10 $(DIVC quickindex,
11 $(BOOKTABLE ,
12 $(TR $(TH Category) $(TH Functions)
13 )
14 $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get)
15 $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace)
16 $(MYREF connect) $(MYREF byLine) $(MYREF byChunk)
17 $(MYREF byLineAsync) $(MYREF byChunkAsync) )
18 )
19 $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF
20 SMTP) )
21 )
22 )
23 )
24 
25 Note:
26 You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"])
27 to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB).
28 
29 Windows x86 note:
30 A DMD compatible libcurl static library can be downloaded from the dlang.org
31 $(LINK2 http://downloads.dlang.org/other/index.html, download archive page).
32 
33 This module is not available for iOS, tvOS or watchOS.
34 
35 Compared to using libcurl directly this module allows simpler client code for
36 common uses, requires no unsafe operations, and integrates better with the rest
37 of the language. Futhermore it provides $(MREF_ALTTEXT range, std,range)
38 access to protocols supported by libcurl both synchronously and asynchronously.
39 
40 A high level and a low level API are available. The high level API is built
41 entirely on top of the low level one.
42 
43 The high level API is for commonly used functionality such as HTTP/FTP get. The
44 $(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous
45 $(MREF_ALTTEXT range, std,range) that performs the request in another
46 thread while handling a line/chunk in the current thread.
47 
48 The low level API allows for streaming and other advanced features.
49 
50 $(BOOKTABLE Cheat Sheet,
51 $(TR $(TH Function Name) $(TH Description)
52 )
53 $(LEADINGROW High level)
54 $(TR $(TDNW $(LREF download)) $(TD $(D
55 download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file"))
56 downloads file from URL to file system.)
57 )
58 $(TR $(TDNW $(LREF upload)) $(TD $(D
59 upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");)
60 uploads file from file system to URL.)
61 )
62 $(TR $(TDNW $(LREF get)) $(TD $(D
63 get("dlang.org")) returns a char[] containing the dlang.org web page.)
64 )
65 $(TR $(TDNW $(LREF put)) $(TD $(D
66 put("dlang.org", "Hi")) returns a char[] containing
67 the dlang.org web page. after a HTTP PUT of "hi")
68 )
69 $(TR $(TDNW $(LREF post)) $(TD $(D
70 post("dlang.org", "Hi")) returns a char[] containing
71 the dlang.org web page. after a HTTP POST of "hi")
72 )
73 $(TR $(TDNW $(LREF byLine)) $(TD $(D
74 byLine("dlang.org")) returns a range of char[] containing the
75 dlang.org web page.)
76 )
77 $(TR $(TDNW $(LREF byChunk)) $(TD $(D
78 byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the
79 dlang.org web page.)
80 )
81 $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D
82 byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web
83  page asynchronously.)
84 )
85 $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D
86 byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the
87 dlang.org web page asynchronously.)
88 )
89 $(LEADINGROW Low level
90 )
91 $(TR $(TDNW $(LREF HTTP)) $(TD `HTTP` struct for advanced usage))
92 $(TR $(TDNW $(LREF FTP)) $(TD `FTP` struct for advanced usage))
93 $(TR $(TDNW $(LREF SMTP)) $(TD `SMTP` struct for advanced usage))
94 )
95 
96 
97 Example:
98 ---
99 import std.net.curl, std.stdio;
100 
101 // Return a char[] containing the content specified by a URL
102 auto content = get("dlang.org");
103 
104 // Post data and return a char[] containing the content specified by a URL
105 auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]);
106 
107 // Get content of file from ftp server
108 auto content = get("ftp.digitalmars.com/sieve.ds");
109 
110 // Post and print out content line by line. The request is done in another thread.
111 foreach (line; byLineAsync("dlang.org", "Post data"))
112     writeln(line);
113 
114 // Get using a line range and proxy settings
115 auto client = HTTP();
116 client.proxy = "1.2.3.4";
117 foreach (line; byLine("dlang.org", client))
118     writeln(line);
119 ---
120 
121 For more control than the high level functions provide, use the low level API:
122 
123 Example:
124 ---
125 import std.net.curl, std.stdio;
126 
127 // GET with custom data receivers
128 auto http = HTTP("dlang.org");
129 http.onReceiveHeader =
130     (in char[] key, in char[] value) { writeln(key, ": ", value); };
131 http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
132 http.perform();
133 ---
134 
135 First, an instance of the reference-counted HTTP struct is created. Then the
136 custom delegates are set. These will be called whenever the HTTP instance
137 receives a header and a data buffer, respectively. In this simple example, the
138 headers are written to stdout and the data is ignored. If the request should be
139 stopped before it has finished then return something less than data.length from
140 the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more
141 information. Finally the HTTP request is effected by calling perform(), which is
142 synchronous.
143 
144 Source: $(PHOBOSSRC std/net/curl.d)
145 
146 Copyright: Copyright Jonas Drewsen 2011-2012
147 License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
148 Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao.
149 
150 Credits: The functionally is based on $(HTTP curl.haxx.se/libcurl, libcurl).
151          LibCurl is licensed under an MIT/X derivative license.
152 */
153 /*
154          Copyright Jonas Drewsen 2011 - 2012.
155 Distributed under the Boost Software License, Version 1.0.
156    (See accompanying file LICENSE_1_0.txt or copy at
157          http://www.boost.org/LICENSE_1_0.txt)
158 */
159 module std.net.curl;
160 
161 public import etc.c.curl : CurlOption;
162 import core.time : dur;
163 import etc.c.curl : CURLcode;
164 import std.range.primitives;
165 import std.encoding : EncodingScheme;
166 import std.traits : isSomeChar;
167 import std.typecons : Flag, Yes, No, Tuple;
168 
169 version (iOS)
170     version = iOSDerived;
171 else version (TVOS)
172     version = iOSDerived;
173 else version (WatchOS)
174     version = iOSDerived;
175 
176 version (iOSDerived) {}
177 else:
178 
179 version (StdUnittest)
180 {
181     import std.socket : Socket, SocketShutdown;
182 
183     private struct TestServer
184     {
185         import std.concurrency : Tid;
186 
187         import std.socket : Socket, TcpSocket;
188 
189         string addr() { return _addr; }
190 
191         void handle(void function(Socket s) dg)
192         {
193             import std.concurrency : send;
194             tid.send(dg);
195         }
196 
197     private:
198         string _addr;
199         Tid tid;
200         TcpSocket sock;
201 
202         static void loop(shared TcpSocket listener)
203         {
204             import std.concurrency : OwnerTerminated, receiveOnly;
205             import std.stdio : stderr;
206 
207             try while (true)
208             {
209                 void function(Socket) handler = void;
210                 try
211                     handler = receiveOnly!(typeof(handler));
212                 catch (OwnerTerminated)
213                     return;
214                 handler((cast() listener).accept);
215             }
216             catch (Throwable e)
217             {
218                 // https://issues.dlang.org/show_bug.cgi?id=7018
219                 stderr.writeln(e);
220             }
221         }
222     }
223 
224     private TestServer startServer()
225     {
226         import std.concurrency : spawn;
227         import std.socket : INADDR_LOOPBACK, InternetAddress, TcpSocket;
228 
229         tlsInit = true;
230         auto sock = new TcpSocket;
231         sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY));
232         sock.listen(1);
233         auto addr = sock.localAddress.toString();
234         auto tid = spawn(&TestServer.loop, cast(shared) sock);
235         return TestServer(addr, tid, sock);
236     }
237 
238     /** Test server */
239     __gshared TestServer server;
240     /** Thread-local storage init */
241     bool tlsInit;
242 
243     private ref TestServer testServer()
244     {
245         import std.concurrency : initOnce;
246         return initOnce!server(startServer());
247     }
248 
249     static ~this()
250     {
251         // terminate server from a thread local dtor of the thread that started it,
252         //  because thread_joinall is called before shared module dtors
253         if (tlsInit && server.sock)
254         {
255             server.sock.shutdown(SocketShutdown.RECEIVE);
256             server.sock.close();
257         }
258     }
259 
260     private struct Request(T)
261     {
262         string hdrs;
263         immutable(T)[] bdy;
264     }
265 
266     private Request!T recvReq(T=char)(Socket s)
267     {
268         import std.algorithm.comparison : min;
269         import std.algorithm.searching : find, canFind;
270         import std.conv : to;
271         import std.regex : ctRegex, matchFirst;
272 
273         ubyte[1024] tmp=void;
274         ubyte[] buf;
275 
276         while (true)
277         {
278             auto nbytes = s.receive(tmp[]);
279             assert(nbytes >= 0);
280 
281             immutable beg = buf.length > 3 ? buf.length - 3 : 0;
282             buf ~= tmp[0 .. nbytes];
283             auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n");
284             if (bdy.empty)
285                 continue;
286 
287             auto hdrs = cast(string) buf[0 .. $ - bdy.length];
288             bdy.popFrontN(4);
289             // no support for chunked transfer-encoding
290             if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i")))
291             {
292                 import std.uni : asUpperCase;
293                 if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE"))
294                     s.send(httpContinue);
295 
296                 size_t remain = m.captures[1].to!size_t - bdy.length;
297                 while (remain)
298                 {
299                     nbytes = s.receive(tmp[0 .. min(remain, $)]);
300                     assert(nbytes >= 0);
301                     buf ~= tmp[0 .. nbytes];
302                     remain -= nbytes;
303                 }
304             }
305             else
306             {
307                 assert(bdy.empty);
308             }
309             bdy = buf[hdrs.length + 4 .. $];
310             return typeof(return)(hdrs, cast(immutable(T)[])bdy);
311         }
312     }
313 
314     private string httpOK(string msg)
315     {
316         import std.conv : to;
317 
318         return "HTTP/1.1 200 OK\r\n"~
319             "Content-Type: text/plain\r\n"~
320             "Content-Length: "~msg.length.to!string~"\r\n"~
321             "\r\n"~
322             msg;
323     }
324 
325     private string httpOK()
326     {
327         return "HTTP/1.1 200 OK\r\n"~
328             "Content-Length: 0\r\n"~
329             "\r\n";
330     }
331 
332     private string httpNotFound()
333     {
334         return "HTTP/1.1 404 Not Found\r\n"~
335             "Content-Length: 0\r\n"~
336             "\r\n";
337     }
338 
339     private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n";
340 }
341 version (StdDdoc) import std.stdio;
342 
343 // Default data timeout for Protocols
344 private enum _defaultDataTimeout = dur!"minutes"(2);
345 
346 /**
347 Macros:
348 
349 CALLBACK_PARAMS = $(TABLE ,
350     $(DDOC_PARAM_ROW
351         $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal))
352         $(DDOC_PARAM_DESC total bytes to download)
353         )
354     $(DDOC_PARAM_ROW
355         $(DDOC_PARAM_ID $(DDOC_PARAM dlNow))
356         $(DDOC_PARAM_DESC currently downloaded bytes)
357         )
358     $(DDOC_PARAM_ROW
359         $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal))
360         $(DDOC_PARAM_DESC total bytes to upload)
361         )
362     $(DDOC_PARAM_ROW
363         $(DDOC_PARAM_ID $(DDOC_PARAM ulNow))
364         $(DDOC_PARAM_DESC currently uploaded bytes)
365         )
366 )
367 */
368 
369 /** Connection type used when the URL should be used to auto detect the protocol.
370   *
371   * This struct is used as placeholder for the connection parameter when calling
372   * the high level API and the connection type (HTTP/FTP) should be guessed by
373   * inspecting the URL parameter.
374   *
375   * The rules for guessing the protocol are:
376   * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed.
377   * 2, HTTP connection otherwise.
378   *
379   * Example:
380   * ---
381   * import std.net.curl;
382   * // Two requests below will do the same.
383   * char[] content;
384   *
385   * // Explicit connection provided
386   * content = get!HTTP("dlang.org");
387   *
388   * // Guess connection type by looking at the URL
389   * content = get!AutoProtocol("ftp://foo.com/file");
390   * // and since AutoProtocol is default this is the same as
391   * content = get("ftp://foo.com/file");
392   * // and will end up detecting FTP from the url and be the same as
393   * content = get!FTP("ftp://foo.com/file");
394   * ---
395   */
396 struct AutoProtocol { }
397 
398 // Returns true if the url points to an FTP resource
399 private bool isFTPUrl(const(char)[] url)
400 {
401     import std.algorithm.searching : startsWith;
402     import std.uni : toLower;
403 
404     return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0;
405 }
406 
407 // Is true if the Conn type is a valid Curl Connection type.
408 private template isCurlConn(Conn)
409 {
410     enum auto isCurlConn = is(Conn : HTTP) ||
411         is(Conn : FTP) || is(Conn : AutoProtocol);
412 }
413 
414 /** HTTP/FTP download to local file system.
415  *
416  * Params:
417  * url = resource to download
418  * saveToPath = path to store the downloaded content on local disk
419  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
420  *        guess connection type and create a new instance for this call only.
421  *
422  * Example:
423  * ----
424  * import std.net.curl;
425  * download("https://httpbin.org/get", "/tmp/downloaded-http-file");
426  * ----
427  */
428 void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn())
429 if (isCurlConn!Conn)
430 {
431     static if (is(Conn : HTTP) || is(Conn : FTP))
432     {
433         import std.stdio : File;
434         conn.url = url;
435         auto f = File(saveToPath, "wb");
436         conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; };
437         conn.perform();
438     }
439     else
440     {
441         if (isFTPUrl(url))
442             return download!FTP(url, saveToPath, FTP());
443         else
444             return download!HTTP(url, saveToPath, HTTP());
445     }
446 }
447 
448 @system unittest
449 {
450     import std.algorithm.searching : canFind;
451     static import std.file;
452 
453     foreach (host; [testServer.addr, "http://"~testServer.addr])
454     {
455         testServer.handle((s) {
456             assert(s.recvReq.hdrs.canFind("GET /"));
457             s.send(httpOK("Hello world"));
458         });
459         auto fn = std.file.deleteme;
460         scope (exit)
461         {
462             if (std.file.exists(fn))
463                 std.file.remove(fn);
464         }
465         download(host, fn);
466         assert(std.file.readText(fn) == "Hello world");
467     }
468 }
469 
470 /** Upload file from local files system using the HTTP or FTP protocol.
471  *
472  * Params:
473  * loadFromPath = path load data from local disk.
474  * url = resource to upload to
475  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
476  *        guess connection type and create a new instance for this call only.
477  *
478  * Example:
479  * ----
480  * import std.net.curl;
481  * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");
482  * upload("/tmp/downloaded-http-file", "https://httpbin.org/post");
483  * ----
484  */
485 void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn())
486 if (isCurlConn!Conn)
487 {
488     static if (is(Conn : HTTP))
489     {
490         conn.url = url;
491         conn.method = HTTP.Method.put;
492     }
493     else static if (is(Conn : FTP))
494     {
495         conn.url = url;
496         conn.handle.set(CurlOption.upload, 1L);
497     }
498     else
499     {
500         if (isFTPUrl(url))
501             return upload!FTP(loadFromPath, url, FTP());
502         else
503             return upload!HTTP(loadFromPath, url, HTTP());
504     }
505 
506     static if (is(Conn : HTTP) || is(Conn : FTP))
507     {
508         import std.stdio : File;
509         auto f = File(loadFromPath, "rb");
510         conn.onSend = buf => f.rawRead(buf).length;
511         immutable sz = f.size;
512         if (sz != ulong.max)
513             conn.contentLength = sz;
514         conn.perform();
515     }
516 }
517 
518 @system unittest
519 {
520     import std.algorithm.searching : canFind;
521     static import std.file;
522 
523     foreach (host; [testServer.addr, "http://"~testServer.addr])
524     {
525         auto fn = std.file.deleteme;
526         scope (exit)
527         {
528             if (std.file.exists(fn))
529                 std.file.remove(fn);
530         }
531         std.file.write(fn, "upload data\n");
532         testServer.handle((s) {
533             auto req = s.recvReq;
534             assert(req.hdrs.canFind("PUT /path"));
535             assert(req.bdy.canFind("upload data"));
536             s.send(httpOK());
537         });
538         upload(fn, host ~ "/path");
539     }
540 }
541 
542 /** HTTP/FTP get content.
543  *
544  * Params:
545  * url = resource to get
546  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
547  *        guess connection type and create a new instance for this call only.
548  *
549  * The template parameter `T` specifies the type to return. Possible values
550  * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking
551  * for `char`, content will be converted from the connection character set
552  * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
553  * by default) to UTF-8.
554  *
555  * Example:
556  * ----
557  * import std.net.curl;
558  * auto content = get("https://httpbin.org/get");
559  * ----
560  *
561  * Returns:
562  * A T[] range containing the content of the resource pointed to by the URL.
563  *
564  * Throws:
565  *
566  * `CurlException` on error.
567  *
568  * See_Also: $(LREF HTTP.Method)
569  */
570 T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn())
571 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
572 {
573     static if (is(Conn : HTTP))
574     {
575         conn.method = HTTP.Method.get;
576         return _basicHTTP!(T)(url, "", conn);
577 
578     }
579     else static if (is(Conn : FTP))
580     {
581         return _basicFTP!(T)(url, "", conn);
582     }
583     else
584     {
585         if (isFTPUrl(url))
586             return get!(FTP,T)(url, FTP());
587         else
588             return get!(HTTP,T)(url, HTTP());
589     }
590 }
591 
592 @system unittest
593 {
594     import std.algorithm.searching : canFind;
595 
596     foreach (host; [testServer.addr, "http://"~testServer.addr])
597     {
598         testServer.handle((s) {
599             assert(s.recvReq.hdrs.canFind("GET /path"));
600             s.send(httpOK("GETRESPONSE"));
601         });
602         auto res = get(host ~ "/path");
603         assert(res == "GETRESPONSE");
604     }
605 }
606 
607 
608 /** HTTP post content.
609  *
610  * Params:
611  *     url = resource to post to
612  *     postDict = data to send as the body of the request. An associative array
613  *                of `string` is accepted and will be encoded using
614  *                www-form-urlencoding
615  *     postData = data to send as the body of the request. An array
616  *                of an arbitrary type is accepted and will be cast to ubyte[]
617  *                before sending it.
618  *     conn = HTTP connection to use
619  *     T    = The template parameter `T` specifies the type to return. Possible values
620  *            are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking
621  *            for `char`, content will be converted from the connection character set
622  *            (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
623  *            by default) to UTF-8.
624  *
625  * Examples:
626  * ----
627  * import std.net.curl;
628  *
629  * auto content1 = post("https://httpbin.org/post", ["name1" : "value1", "name2" : "value2"]);
630  * auto content2 = post("https://httpbin.org/post", [1,2,3,4]);
631  * ----
632  *
633  * Returns:
634  * A T[] range containing the content of the resource pointed to by the URL.
635  *
636  * See_Also: $(LREF HTTP.Method)
637  */
638 T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP())
639 if (is(T == char) || is(T == ubyte))
640 {
641     conn.method = HTTP.Method.post;
642     return _basicHTTP!(T)(url, postData, conn);
643 }
644 
645 @system unittest
646 {
647     import std.algorithm.searching : canFind;
648 
649     foreach (host; [testServer.addr, "http://"~testServer.addr])
650     {
651         testServer.handle((s) {
652             auto req = s.recvReq;
653             assert(req.hdrs.canFind("POST /path"));
654             assert(req.bdy.canFind("POSTBODY"));
655             s.send(httpOK("POSTRESPONSE"));
656         });
657         auto res = post(host ~ "/path", "POSTBODY");
658         assert(res == "POSTRESPONSE");
659     }
660 }
661 
662 @system unittest
663 {
664     import std.algorithm.searching : canFind;
665 
666     auto data = new ubyte[](256);
667     foreach (i, ref ub; data)
668         ub = cast(ubyte) i;
669 
670     testServer.handle((s) {
671         auto req = s.recvReq!ubyte;
672         assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
673         assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
674         s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
675     });
676     auto res = post!ubyte(testServer.addr, data);
677     assert(res == cast(ubyte[])[17, 27, 35, 41]);
678 }
679 
680 /// ditto
681 T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP())
682 if (is(T == char) || is(T == ubyte))
683 {
684     import std.uri : urlEncode;
685 
686     return post!T(url, urlEncode(postDict), conn);
687 }
688 
689 @system unittest
690 {
691     import std.algorithm.searching : canFind;
692     import std.meta : AliasSeq;
693 
694     static immutable expected = ["name1=value1&name2=value2", "name2=value2&name1=value1"];
695 
696     foreach (host; [testServer.addr, "http://" ~ testServer.addr])
697     {
698         foreach (T; AliasSeq!(char, ubyte))
699         {
700             testServer.handle((s) {
701                 auto req = s.recvReq!char;
702                 s.send(httpOK(req.bdy));
703             });
704             auto res = post!T(host ~ "/path", ["name1" : "value1", "name2" : "value2"]);
705             assert(canFind(expected, res));
706         }
707     }
708 }
709 
710 /** HTTP/FTP put content.
711  *
712  * Params:
713  * url = resource to put
714  * putData = data to send as the body of the request. An array
715  *           of an arbitrary type is accepted and will be cast to ubyte[]
716  *           before sending it.
717  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
718  *        guess connection type and create a new instance for this call only.
719  *
720  * The template parameter `T` specifies the type to return. Possible values
721  * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking
722  * for `char`, content will be converted from the connection character set
723  * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
724  * by default) to UTF-8.
725  *
726  * Example:
727  * ----
728  * import std.net.curl;
729  * auto content = put("https://httpbin.org/put",
730  *                      "Putting this data");
731  * ----
732  *
733  * Returns:
734  * A T[] range containing the content of the resource pointed to by the URL.
735  *
736  * See_Also: $(LREF HTTP.Method)
737  */
738 T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData,
739                                                   Conn conn = Conn())
740 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
741 {
742     static if (is(Conn : HTTP))
743     {
744         conn.method = HTTP.Method.put;
745         return _basicHTTP!(T)(url, putData, conn);
746     }
747     else static if (is(Conn : FTP))
748     {
749         return _basicFTP!(T)(url, putData, conn);
750     }
751     else
752     {
753         if (isFTPUrl(url))
754             return put!(FTP,T)(url, putData, FTP());
755         else
756             return put!(HTTP,T)(url, putData, HTTP());
757     }
758 }
759 
760 @system unittest
761 {
762     import std.algorithm.searching : canFind;
763 
764     foreach (host; [testServer.addr, "http://"~testServer.addr])
765     {
766         testServer.handle((s) {
767             auto req = s.recvReq;
768             assert(req.hdrs.canFind("PUT /path"));
769             assert(req.bdy.canFind("PUTBODY"));
770             s.send(httpOK("PUTRESPONSE"));
771         });
772         auto res = put(host ~ "/path", "PUTBODY");
773         assert(res == "PUTRESPONSE");
774     }
775 }
776 
777 
778 /** HTTP/FTP delete content.
779  *
780  * Params:
781  * url = resource to delete
782  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
783  *        guess connection type and create a new instance for this call only.
784  *
785  * Example:
786  * ----
787  * import std.net.curl;
788  * del("https://httpbin.org/delete");
789  * ----
790  *
791  * See_Also: $(LREF HTTP.Method)
792  */
793 void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn())
794 if (isCurlConn!Conn)
795 {
796     static if (is(Conn : HTTP))
797     {
798         conn.method = HTTP.Method.del;
799         _basicHTTP!char(url, cast(void[]) null, conn);
800     }
801     else static if (is(Conn : FTP))
802     {
803         import std.algorithm.searching : findSplitAfter;
804         import std.conv : text;
805         import std.exception : enforce;
806 
807         auto trimmed = url.findSplitAfter("ftp://")[1];
808         auto t = trimmed.findSplitAfter("/");
809         enum minDomainNameLength = 3;
810         enforce!CurlException(t[0].length > minDomainNameLength,
811                                 text("Invalid FTP URL for delete ", url));
812         conn.url = t[0];
813 
814         enforce!CurlException(!t[1].empty,
815                                 text("No filename specified to delete for URL ", url));
816         conn.addCommand("DELE " ~ t[1]);
817         conn.perform();
818     }
819     else
820     {
821         if (isFTPUrl(url))
822             return del!FTP(url, FTP());
823         else
824             return del!HTTP(url, HTTP());
825     }
826 }
827 
828 @system unittest
829 {
830     import std.algorithm.searching : canFind;
831 
832     foreach (host; [testServer.addr, "http://"~testServer.addr])
833     {
834         testServer.handle((s) {
835             auto req = s.recvReq;
836             assert(req.hdrs.canFind("DELETE /path"));
837             s.send(httpOK());
838         });
839         del(host ~ "/path");
840     }
841 }
842 
843 
844 /** HTTP options request.
845  *
846  * Params:
847  * url = resource make a option call to
848  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
849  *        guess connection type and create a new instance for this call only.
850  *
851  * The template parameter `T` specifies the type to return. Possible values
852  * are `char` and `ubyte` to return `char[]` or `ubyte[]`.
853  *
854  * Example:
855  * ----
856  * import std.net.curl;
857  * auto http = HTTP();
858  * options("https://httpbin.org/headers", http);
859  * writeln("Allow set to " ~ http.responseHeaders["Allow"]);
860  * ----
861  *
862  * Returns:
863  * A T[] range containing the options of the resource pointed to by the URL.
864  *
865  * See_Also: $(LREF HTTP.Method)
866  */
867 T[] options(T = char)(const(char)[] url, HTTP conn = HTTP())
868 if (is(T == char) || is(T == ubyte))
869 {
870     conn.method = HTTP.Method.options;
871     return _basicHTTP!(T)(url, null, conn);
872 }
873 
874 @system unittest
875 {
876     import std.algorithm.searching : canFind;
877 
878     testServer.handle((s) {
879         auto req = s.recvReq;
880         assert(req.hdrs.canFind("OPTIONS /path"));
881         s.send(httpOK("OPTIONSRESPONSE"));
882     });
883     auto res = options(testServer.addr ~ "/path");
884     assert(res == "OPTIONSRESPONSE");
885 }
886 
887 
888 /** HTTP trace request.
889  *
890  * Params:
891  * url = resource make a trace call to
892  * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
893  *        guess connection type and create a new instance for this call only.
894  *
895  * The template parameter `T` specifies the type to return. Possible values
896  * are `char` and `ubyte` to return `char[]` or `ubyte[]`.
897  *
898  * Example:
899  * ----
900  * import std.net.curl;
901  * trace("https://httpbin.org/headers");
902  * ----
903  *
904  * Returns:
905  * A T[] range containing the trace info of the resource pointed to by the URL.
906  *
907  * See_Also: $(LREF HTTP.Method)
908  */
909 T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP())
910 if (is(T == char) || is(T == ubyte))
911 {
912     conn.method = HTTP.Method.trace;
913     return _basicHTTP!(T)(url, cast(void[]) null, conn);
914 }
915 
916 @system unittest
917 {
918     import std.algorithm.searching : canFind;
919 
920     testServer.handle((s) {
921         auto req = s.recvReq;
922         assert(req.hdrs.canFind("TRACE /path"));
923         s.send(httpOK("TRACERESPONSE"));
924     });
925     auto res = trace(testServer.addr ~ "/path");
926     assert(res == "TRACERESPONSE");
927 }
928 
929 
930 /** HTTP connect request.
931  *
932  * Params:
933  * url = resource make a connect to
934  * conn = HTTP connection to use
935  *
936  * The template parameter `T` specifies the type to return. Possible values
937  * are `char` and `ubyte` to return `char[]` or `ubyte[]`.
938  *
939  * Example:
940  * ----
941  * import std.net.curl;
942  * connect("https://httpbin.org/headers");
943  * ----
944  *
945  * Returns:
946  * A T[] range containing the connect info of the resource pointed to by the URL.
947  *
948  * See_Also: $(LREF HTTP.Method)
949  */
950 T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP())
951 if (is(T == char) || is(T == ubyte))
952 {
953     conn.method = HTTP.Method.connect;
954     return _basicHTTP!(T)(url, cast(void[]) null, conn);
955 }
956 
957 @system unittest
958 {
959     import std.algorithm.searching : canFind;
960 
961     testServer.handle((s) {
962         auto req = s.recvReq;
963         assert(req.hdrs.canFind("CONNECT /path"));
964         s.send(httpOK("CONNECTRESPONSE"));
965     });
966     auto res = connect(testServer.addr ~ "/path");
967     assert(res == "CONNECTRESPONSE");
968 }
969 
970 
971 /** HTTP patch content.
972  *
973  * Params:
974  * url = resource to patch
975  * patchData = data to send as the body of the request. An array
976  *           of an arbitrary type is accepted and will be cast to ubyte[]
977  *           before sending it.
978  * conn = HTTP connection to use
979  *
980  * The template parameter `T` specifies the type to return. Possible values
981  * are `char` and `ubyte` to return `char[]` or `ubyte[]`.
982  *
983  * Example:
984  * ----
985  * auto http = HTTP();
986  * http.addRequestHeader("Content-Type", "application/json");
987  * auto content = patch("https://httpbin.org/patch", `{"title": "Patched Title"}`, http);
988  * ----
989  *
990  * Returns:
991  * A T[] range containing the content of the resource pointed to by the URL.
992  *
993  * See_Also: $(LREF HTTP.Method)
994  */
995 T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData,
996                                HTTP conn = HTTP())
997 if (is(T == char) || is(T == ubyte))
998 {
999     conn.method = HTTP.Method.patch;
1000     return _basicHTTP!(T)(url, patchData, conn);
1001 }
1002 
1003 @system unittest
1004 {
1005     import std.algorithm.searching : canFind;
1006 
1007     testServer.handle((s) {
1008         auto req = s.recvReq;
1009         assert(req.hdrs.canFind("PATCH /path"));
1010         assert(req.bdy.canFind("PATCHBODY"));
1011         s.send(httpOK("PATCHRESPONSE"));
1012     });
1013     auto res = patch(testServer.addr ~ "/path", "PATCHBODY");
1014     assert(res == "PATCHRESPONSE");
1015 }
1016 
1017 
1018 /*
1019  * Helper function for the high level interface.
1020  *
1021  * It performs an HTTP request using the client which must have
1022  * been setup correctly before calling this function.
1023  */
1024 private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client)
1025 {
1026     import std.algorithm.comparison : min;
1027     import std.format : format;
1028     import std.exception : enforce;
1029     import etc.c.curl : CurlSeek, CurlSeekPos;
1030 
1031     immutable doSend = sendData !is null &&
1032         (client.method == HTTP.Method.post ||
1033          client.method == HTTP.Method.put ||
1034          client.method == HTTP.Method.patch);
1035 
1036     scope (exit)
1037     {
1038         client.onReceiveHeader = null;
1039         client.onReceiveStatusLine = null;
1040         client.onReceive = null;
1041 
1042         if (doSend)
1043         {
1044             client.onSend = null;
1045             client.handle.onSeek = null;
1046             client.contentLength = 0;
1047         }
1048     }
1049     client.url = url;
1050     HTTP.StatusLine statusLine;
1051     import std.array : appender;
1052     auto content = appender!(ubyte[])();
1053     client.onReceive = (ubyte[] data)
1054     {
1055         content ~= data;
1056         return data.length;
1057     };
1058 
1059     if (doSend)
1060     {
1061         client.contentLength = sendData.length;
1062         auto remainingData = sendData;
1063         client.onSend = delegate size_t(void[] buf)
1064         {
1065             size_t minLen = min(buf.length, remainingData.length);
1066             if (minLen == 0) return 0;
1067             buf[0 .. minLen] = remainingData[0 .. minLen];
1068             remainingData = remainingData[minLen..$];
1069             return minLen;
1070         };
1071         client.handle.onSeek = delegate(long offset, CurlSeekPos mode)
1072         {
1073             switch (mode)
1074             {
1075                 case CurlSeekPos.set:
1076                     remainingData = sendData[cast(size_t) offset..$];
1077                     return CurlSeek.ok;
1078                 default:
1079                     // As of curl 7.18.0, libcurl will not pass
1080                     // anything other than CurlSeekPos.set.
1081                     return CurlSeek.cantseek;
1082             }
1083         };
1084     }
1085 
1086     client.onReceiveHeader = (in char[] key,
1087                               in char[] value)
1088     {
1089         if (key == "content-length")
1090         {
1091             import std.conv : to;
1092             content.reserve(value.to!size_t);
1093         }
1094     };
1095     client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
1096     client.perform();
1097     enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code,
1098             format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason)));
1099 
1100     return _decodeContent!T(content.data, client.p.charset);
1101 }
1102 
1103 @system unittest
1104 {
1105     import std.algorithm.searching : canFind;
1106     import std.exception : collectException;
1107 
1108     testServer.handle((s) {
1109         auto req = s.recvReq;
1110         assert(req.hdrs.canFind("GET /path"));
1111         s.send(httpNotFound());
1112     });
1113     auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path"));
1114     assert(e.msg == "HTTP request returned status code 404 (Not Found)");
1115     assert(e.status == 404);
1116 }
1117 
1118 // Content length must be reset after post
1119 // https://issues.dlang.org/show_bug.cgi?id=14760
1120 @system unittest
1121 {
1122     import std.algorithm.searching : canFind;
1123 
1124     testServer.handle((s) {
1125         auto req = s.recvReq;
1126         assert(req.hdrs.canFind("POST /"));
1127         assert(req.bdy.canFind("POSTBODY"));
1128         s.send(httpOK("POSTRESPONSE"));
1129 
1130         req = s.recvReq;
1131         assert(req.hdrs.canFind("TRACE /"));
1132         assert(req.bdy.empty);
1133         s.blocking = false;
1134         ubyte[6] buf = void;
1135         assert(s.receive(buf[]) < 0);
1136         s.send(httpOK("TRACERESPONSE"));
1137     });
1138     auto http = HTTP();
1139     auto res = post(testServer.addr, "POSTBODY", http);
1140     assert(res == "POSTRESPONSE");
1141     res = trace(testServer.addr, http);
1142     assert(res == "TRACERESPONSE");
1143 }
1144 
1145 @system unittest // charset detection and transcoding to T
1146 {
1147     testServer.handle((s) {
1148         s.send("HTTP/1.1 200 OK\r\n"~
1149         "Content-Length: 4\r\n"~
1150         "Content-Type: text/plain; charset=utf-8\r\n" ~
1151         "\r\n" ~
1152         "äbc");
1153     });
1154     auto client = HTTP();
1155     auto result = _basicHTTP!char(testServer.addr, "", client);
1156     assert(result == "äbc");
1157 
1158     testServer.handle((s) {
1159         s.send("HTTP/1.1 200 OK\r\n"~
1160         "Content-Length: 3\r\n"~
1161         "Content-Type: text/plain; charset=iso-8859-1\r\n" ~
1162         "\r\n" ~
1163         0xE4 ~ "bc");
1164     });
1165     client = HTTP();
1166     result = _basicHTTP!char(testServer.addr, "", client);
1167     assert(result == "äbc");
1168 }
1169 
1170 /*
1171  * Helper function for the high level interface.
1172  *
1173  * It performs an FTP request using the client which must have
1174  * been setup correctly before calling this function.
1175  */
1176 private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client)
1177 {
1178     import std.algorithm.comparison : min;
1179 
1180     scope (exit)
1181     {
1182         client.onReceive = null;
1183         if (!sendData.empty)
1184             client.onSend = null;
1185     }
1186 
1187     ubyte[] content;
1188 
1189     if (client.encoding.empty)
1190         client.encoding = "ISO-8859-1";
1191 
1192     client.url = url;
1193     client.onReceive = (ubyte[] data)
1194     {
1195         content ~= data;
1196         return data.length;
1197     };
1198 
1199     if (!sendData.empty)
1200     {
1201         client.handle.set(CurlOption.upload, 1L);
1202         client.onSend = delegate size_t(void[] buf)
1203         {
1204             size_t minLen = min(buf.length, sendData.length);
1205             if (minLen == 0) return 0;
1206             buf[0 .. minLen] = sendData[0 .. minLen];
1207             sendData = sendData[minLen..$];
1208             return minLen;
1209         };
1210     }
1211 
1212     client.perform();
1213 
1214     return _decodeContent!T(content, client.encoding);
1215 }
1216 
1217 /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to
1218  * correct string format
1219  */
1220 private auto _decodeContent(T)(ubyte[] content, string encoding)
1221 {
1222     static if (is(T == ubyte))
1223     {
1224         return content;
1225     }
1226     else
1227     {
1228         import std.exception : enforce;
1229         import std.format : format;
1230 
1231         // Optimally just return the utf8 encoded content
1232         if (encoding == "UTF-8")
1233             return cast(char[])(content);
1234 
1235         // The content has to be re-encoded to utf8
1236         auto scheme = EncodingScheme.create(encoding);
1237         enforce!CurlException(scheme !is null,
1238                                 format("Unknown encoding '%s'", encoding));
1239 
1240         auto strInfo = decodeString(content, scheme);
1241         enforce!CurlException(strInfo[0] != size_t.max,
1242                                 format("Invalid encoding sequence for encoding '%s'",
1243                                        encoding));
1244 
1245         return strInfo[1];
1246     }
1247 }
1248 
1249 alias KeepTerminator = Flag!"keepTerminator";
1250 /+
1251 struct ByLineBuffer(Char)
1252 {
1253     bool linePresent;
1254     bool EOF;
1255     Char[] buffer;
1256     ubyte[] decodeRemainder;
1257 
1258     bool append(const(ubyte)[] data)
1259     {
1260         byLineBuffer ~= data;
1261     }
1262 
1263     @property bool linePresent()
1264     {
1265         return byLinePresent;
1266     }
1267 
1268     Char[] get()
1269     {
1270         if (!linePresent)
1271         {
1272             // Decode ubyte[] into Char[] until a Terminator is found.
1273             // If not Terminator is found and EOF is false then raise an
1274             // exception.
1275         }
1276         return byLineBuffer;
1277     }
1278 
1279 }
1280 ++/
1281 /** HTTP/FTP fetch content as a range of lines.
1282  *
1283  * A range of lines is returned when the request is complete. If the method or
1284  * other request properties is to be customized then set the `conn` parameter
1285  * with a HTTP/FTP instance that has these properties set.
1286  *
1287  * Example:
1288  * ----
1289  * import std.net.curl, std.stdio;
1290  * foreach (line; byLine("dlang.org"))
1291  *     writeln(line);
1292  * ----
1293  *
1294  * Params:
1295  * url = The url to receive content from
1296  * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be
1297  *                  returned as part of the lines in the range.
1298  * terminator = The character that terminates a line
1299  * conn = The connection to use e.g. HTTP or FTP.
1300  *
1301  * Returns:
1302  * A range of Char[] with the content of the resource pointer to by the URL
1303  */
1304 auto byLine(Conn = AutoProtocol, Terminator = char, Char = char)
1305            (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
1306             Terminator terminator = '\n', Conn conn = Conn())
1307 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
1308 {
1309     static struct SyncLineInputRange
1310     {
1311 
1312         private Char[] lines;
1313         private Char[] current;
1314         private bool currentValid;
1315         private bool keepTerminator;
1316         private Terminator terminator;
1317 
1318         this(Char[] lines, bool kt, Terminator terminator)
1319         {
1320             this.lines = lines;
1321             this.keepTerminator = kt;
1322             this.terminator = terminator;
1323             currentValid = true;
1324             popFront();
1325         }
1326 
1327         @property @safe bool empty()
1328         {
1329             return !currentValid;
1330         }
1331 
1332         @property @safe Char[] front()
1333         {
1334             import std.exception : enforce;
1335             enforce!CurlException(currentValid, "Cannot call front() on empty range");
1336             return current;
1337         }
1338 
1339         void popFront()
1340         {
1341             import std.algorithm.searching : findSplitAfter, findSplit;
1342             import std.exception : enforce;
1343 
1344             enforce!CurlException(currentValid, "Cannot call popFront() on empty range");
1345             if (lines.empty)
1346             {
1347                 currentValid = false;
1348                 return;
1349             }
1350 
1351             if (keepTerminator)
1352             {
1353                 auto r = findSplitAfter(lines, [ terminator ]);
1354                 if (r[0].empty)
1355                 {
1356                     current = r[1];
1357                     lines = r[0];
1358                 }
1359                 else
1360                 {
1361                     current = r[0];
1362                     lines = r[1];
1363                 }
1364             }
1365             else
1366             {
1367                 auto r = findSplit(lines, [ terminator ]);
1368                 current = r[0];
1369                 lines = r[2];
1370             }
1371         }
1372     }
1373 
1374     auto result = _getForRange!Char(url, conn);
1375     return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator);
1376 }
1377 
1378 @system unittest
1379 {
1380     import std.algorithm.comparison : equal;
1381 
1382     foreach (host; [testServer.addr, "http://"~testServer.addr])
1383     {
1384         testServer.handle((s) {
1385             auto req = s.recvReq;
1386             s.send(httpOK("Line1\nLine2\nLine3"));
1387         });
1388         assert(byLine(host).equal(["Line1", "Line2", "Line3"]));
1389     }
1390 }
1391 
1392 /** HTTP/FTP fetch content as a range of chunks.
1393  *
1394  * A range of chunks is returned when the request is complete. If the method or
1395  * other request properties is to be customized then set the `conn` parameter
1396  * with a HTTP/FTP instance that has these properties set.
1397  *
1398  * Example:
1399  * ----
1400  * import std.net.curl, std.stdio;
1401  * foreach (chunk; byChunk("dlang.org", 100))
1402  *     writeln(chunk); // chunk is ubyte[100]
1403  * ----
1404  *
1405  * Params:
1406  * url = The url to receive content from
1407  * chunkSize = The size of each chunk
1408  * conn = The connection to use e.g. HTTP or FTP.
1409  *
1410  * Returns:
1411  * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL
1412  */
1413 auto byChunk(Conn = AutoProtocol)
1414             (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn())
1415 if (isCurlConn!(Conn))
1416 {
1417     static struct SyncChunkInputRange
1418     {
1419         private size_t chunkSize;
1420         private ubyte[] _bytes;
1421         private size_t offset;
1422 
1423         this(ubyte[] bytes, size_t chunkSize)
1424         {
1425             this._bytes = bytes;
1426             this.chunkSize = chunkSize;
1427         }
1428 
1429         @property @safe auto empty()
1430         {
1431             return offset == _bytes.length;
1432         }
1433 
1434         @property ubyte[] front()
1435         {
1436             size_t nextOffset = offset + chunkSize;
1437             if (nextOffset > _bytes.length) nextOffset = _bytes.length;
1438             return _bytes[offset .. nextOffset];
1439         }
1440 
1441         @safe void popFront()
1442         {
1443             offset += chunkSize;
1444             if (offset > _bytes.length) offset = _bytes.length;
1445         }
1446     }
1447 
1448     auto result = _getForRange!ubyte(url, conn);
1449     return SyncChunkInputRange(result, chunkSize);
1450 }
1451 
1452 @system unittest
1453 {
1454     import std.algorithm.comparison : equal;
1455 
1456     foreach (host; [testServer.addr, "http://"~testServer.addr])
1457     {
1458         testServer.handle((s) {
1459             auto req = s.recvReq;
1460             s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
1461         });
1462         assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
1463     }
1464 }
1465 
1466 private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn)
1467 {
1468     static if (is(Conn : HTTP))
1469     {
1470         conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method;
1471         return _basicHTTP!(T)(url, null, conn);
1472     }
1473     else static if (is(Conn : FTP))
1474     {
1475         return _basicFTP!(T)(url, null, conn);
1476     }
1477     else
1478     {
1479         if (isFTPUrl(url))
1480             return get!(FTP,T)(url, FTP());
1481         else
1482             return get!(HTTP,T)(url, HTTP());
1483     }
1484 }
1485 
1486 /*
1487   Main thread part of the message passing protocol used for all async
1488   curl protocols.
1489  */
1490 private mixin template WorkerThreadProtocol(Unit, alias units)
1491 {
1492     import core.time : Duration;
1493 
1494     @property bool empty()
1495     {
1496         tryEnsureUnits();
1497         return state == State.done;
1498     }
1499 
1500     @property Unit[] front()
1501     {
1502         import std.format : format;
1503         tryEnsureUnits();
1504         assert(state == State.gotUnits,
1505                format("Expected %s but got $s",
1506                       State.gotUnits, state));
1507         return units;
1508     }
1509 
1510     void popFront()
1511     {
1512         import std.concurrency : send;
1513         import std.format : format;
1514 
1515         tryEnsureUnits();
1516         assert(state == State.gotUnits,
1517                format("Expected %s but got $s",
1518                       State.gotUnits, state));
1519         state = State.needUnits;
1520         // Send to worker thread for buffer reuse
1521         workerTid.send(cast(immutable(Unit)[]) units);
1522         units = null;
1523     }
1524 
1525     /** Wait for duration or until data is available and return true if data is
1526          available
1527     */
1528     bool wait(Duration d)
1529     {
1530         import core.time : dur;
1531         import std.datetime.stopwatch : StopWatch;
1532         import std.concurrency : receiveTimeout;
1533 
1534         if (state == State.gotUnits)
1535             return true;
1536 
1537         enum noDur = dur!"hnsecs"(0);
1538         StopWatch sw;
1539         sw.start();
1540         while (state != State.gotUnits && d > noDur)
1541         {
1542             final switch (state)
1543             {
1544             case State.needUnits:
1545                 receiveTimeout(d,
1546                         (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
1547                         {
1548                             if (origin != workerTid)
1549                                 return false;
1550                             units = cast(Unit[]) _data.data;
1551                             state = State.gotUnits;
1552                             return true;
1553                         },
1554                         (Tid origin, CurlMessage!bool f)
1555                         {
1556                             if (origin != workerTid)
1557                                 return false;
1558                             state = state.done;
1559                             return true;
1560                         }
1561                         );
1562                 break;
1563             case State.gotUnits: return true;
1564             case State.done:
1565                 return false;
1566             }
1567             d -= sw.peek();
1568             sw.reset();
1569         }
1570         return state == State.gotUnits;
1571     }
1572 
1573     enum State
1574     {
1575         needUnits,
1576         gotUnits,
1577         done
1578     }
1579     State state;
1580 
1581     void tryEnsureUnits()
1582     {
1583         import std.concurrency : receive;
1584         while (true)
1585         {
1586             final switch (state)
1587             {
1588             case State.needUnits:
1589                 receive(
1590                         (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
1591                         {
1592                             if (origin != workerTid)
1593                                 return false;
1594                             units = cast(Unit[]) _data.data;
1595                             state = State.gotUnits;
1596                             return true;
1597                         },
1598                         (Tid origin, CurlMessage!bool f)
1599                         {
1600                             if (origin != workerTid)
1601                                 return false;
1602                             state = state.done;
1603                             return true;
1604                         }
1605                         );
1606                 break;
1607             case State.gotUnits: return;
1608             case State.done:
1609                 return;
1610             }
1611         }
1612     }
1613 }
1614 
1615 /** HTTP/FTP fetch content as a range of lines asynchronously.
1616  *
1617  * A range of lines is returned immediately and the request that fetches the
1618  * lines is performed in another thread. If the method or other request
1619  * properties is to be customized then set the `conn` parameter with a
1620  * HTTP/FTP instance that has these properties set.
1621  *
1622  * If `postData` is non-_null the method will be set to `post` for HTTP
1623  * requests.
1624  *
1625  * The background thread will buffer up to transmitBuffers number of lines
1626  * before it stops receiving data from network. When the main thread reads the
1627  * lines from the range it frees up buffers and allows for the background thread
1628  * to receive more data from the network.
1629  *
1630  * If no data is available and the main thread accesses the range it will block
1631  * until data becomes available. An exception to this is the `wait(Duration)` method on
1632  * the $(LREF LineInputRange). This method will wait at maximum for the
1633  * specified duration and return true if data is available.
1634  *
1635  * Example:
1636  * ----
1637  * import std.net.curl, std.stdio;
1638  * // Get some pages in the background
1639  * auto range1 = byLineAsync("www.google.com");
1640  * auto range2 = byLineAsync("www.wikipedia.org");
1641  * foreach (line; byLineAsync("dlang.org"))
1642  *     writeln(line);
1643  *
1644  * // Lines already fetched in the background and ready
1645  * foreach (line; range1) writeln(line);
1646  * foreach (line; range2) writeln(line);
1647  * ----
1648  *
1649  * ----
1650  * import std.net.curl, std.stdio;
1651  * // Get a line in a background thread and wait in
1652  * // main thread for 2 seconds for it to arrive.
1653  * auto range3 = byLineAsync("dlang.com");
1654  * if (range3.wait(dur!"seconds"(2)))
1655  *     writeln(range3.front);
1656  * else
1657  *     writeln("No line received after 2 seconds!");
1658  * ----
1659  *
1660  * Params:
1661  * url = The url to receive content from
1662  * postData = Data to HTTP Post
1663  * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be
1664  *                  returned as part of the lines in the range.
1665  * terminator = The character that terminates a line
1666  * transmitBuffers = The number of lines buffered asynchronously
1667  * conn = The connection to use e.g. HTTP or FTP.
1668  *
1669  * Returns:
1670  * A range of Char[] with the content of the resource pointer to by the
1671  * URL.
1672  */
1673 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit)
1674             (const(char)[] url, const(PostUnit)[] postData,
1675              KeepTerminator keepTerminator = No.keepTerminator,
1676              Terminator terminator = '\n',
1677              size_t transmitBuffers = 10, Conn conn = Conn())
1678 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
1679 {
1680     static if (is(Conn : AutoProtocol))
1681     {
1682         if (isFTPUrl(url))
1683             return byLineAsync(url, postData, keepTerminator,
1684                                terminator, transmitBuffers, FTP());
1685         else
1686             return byLineAsync(url, postData, keepTerminator,
1687                                terminator, transmitBuffers, HTTP());
1688     }
1689     else
1690     {
1691         import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid;
1692         // 50 is just an arbitrary number for now
1693         setMaxMailboxSize(thisTid, 50, OnCrowding.block);
1694         auto tid = spawn(&_async!().spawn!(Conn, Char, Terminator));
1695         tid.send(thisTid);
1696         tid.send(terminator);
1697         tid.send(keepTerminator == Yes.keepTerminator);
1698 
1699         _async!().duplicateConnection(url, conn, postData, tid);
1700 
1701         return _async!().LineInputRange!Char(tid, transmitBuffers,
1702                                              Conn.defaultAsyncStringBufferSize);
1703     }
1704 }
1705 
1706 /// ditto
1707 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char)
1708             (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
1709              Terminator terminator = '\n',
1710              size_t transmitBuffers = 10, Conn conn = Conn())
1711 {
1712     static if (is(Conn : AutoProtocol))
1713     {
1714         if (isFTPUrl(url))
1715             return byLineAsync(url, cast(void[]) null, keepTerminator,
1716                                terminator, transmitBuffers, FTP());
1717         else
1718             return byLineAsync(url, cast(void[]) null, keepTerminator,
1719                                terminator, transmitBuffers, HTTP());
1720     }
1721     else
1722     {
1723         return byLineAsync(url, cast(void[]) null, keepTerminator,
1724                            terminator, transmitBuffers, conn);
1725     }
1726 }
1727 
1728 @system unittest
1729 {
1730     import std.algorithm.comparison : equal;
1731 
1732     foreach (host; [testServer.addr, "http://"~testServer.addr])
1733     {
1734         testServer.handle((s) {
1735             auto req = s.recvReq;
1736             s.send(httpOK("Line1\nLine2\nLine3"));
1737         });
1738         assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"]));
1739     }
1740 }
1741 
1742 /** HTTP/FTP fetch content as a range of chunks asynchronously.
1743  *
1744  * A range of chunks is returned immediately and the request that fetches the
1745  * chunks is performed in another thread. If the method or other request
1746  * properties is to be customized then set the `conn` parameter with a
1747  * HTTP/FTP instance that has these properties set.
1748  *
1749  * If `postData` is non-_null the method will be set to `post` for HTTP
1750  * requests.
1751  *
1752  * The background thread will buffer up to transmitBuffers number of chunks
1753  * before is stops receiving data from network. When the main thread reads the
1754  * chunks from the range it frees up buffers and allows for the background
1755  * thread to receive more data from the network.
1756  *
1757  * If no data is available and the main thread access the range it will block
1758  * until data becomes available. An exception to this is the `wait(Duration)`
1759  * method on the $(LREF ChunkInputRange). This method will wait at maximum for the specified
1760  * duration and return true if data is available.
1761  *
1762  * Example:
1763  * ----
1764  * import std.net.curl, std.stdio;
1765  * // Get some pages in the background
1766  * auto range1 = byChunkAsync("www.google.com", 100);
1767  * auto range2 = byChunkAsync("www.wikipedia.org");
1768  * foreach (chunk; byChunkAsync("dlang.org"))
1769  *     writeln(chunk); // chunk is ubyte[100]
1770  *
1771  * // Chunks already fetched in the background and ready
1772  * foreach (chunk; range1) writeln(chunk);
1773  * foreach (chunk; range2) writeln(chunk);
1774  * ----
1775  *
1776  * ----
1777  * import std.net.curl, std.stdio;
1778  * // Get a line in a background thread and wait in
1779  * // main thread for 2 seconds for it to arrive.
1780  * auto range3 = byChunkAsync("dlang.com", 10);
1781  * if (range3.wait(dur!"seconds"(2)))
1782  *     writeln(range3.front);
1783  * else
1784  *     writeln("No chunk received after 2 seconds!");
1785  * ----
1786  *
1787  * Params:
1788  * url = The url to receive content from
1789  * postData = Data to HTTP Post
1790  * chunkSize = The size of the chunks
1791  * transmitBuffers = The number of chunks buffered asynchronously
1792  * conn = The connection to use e.g. HTTP or FTP.
1793  *
1794  * Returns:
1795  * A range of ubyte[chunkSize] with the content of the resource pointer to by
1796  * the URL.
1797  */
1798 auto byChunkAsync(Conn = AutoProtocol, PostUnit)
1799            (const(char)[] url, const(PostUnit)[] postData,
1800             size_t chunkSize = 1024, size_t transmitBuffers = 10,
1801             Conn conn = Conn())
1802 if (isCurlConn!(Conn))
1803 {
1804     static if (is(Conn : AutoProtocol))
1805     {
1806         if (isFTPUrl(url))
1807             return byChunkAsync(url, postData, chunkSize,
1808                                 transmitBuffers, FTP());
1809         else
1810             return byChunkAsync(url, postData, chunkSize,
1811                                 transmitBuffers, HTTP());
1812     }
1813     else
1814     {
1815         import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid;
1816         // 50 is just an arbitrary number for now
1817         setMaxMailboxSize(thisTid, 50, OnCrowding.block);
1818         auto tid = spawn(&_async!().spawn!(Conn, ubyte));
1819         tid.send(thisTid);
1820 
1821         _async!().duplicateConnection(url, conn, postData, tid);
1822 
1823         return _async!().ChunkInputRange(tid, transmitBuffers, chunkSize);
1824     }
1825 }
1826 
1827 /// ditto
1828 auto byChunkAsync(Conn = AutoProtocol)
1829            (const(char)[] url,
1830             size_t chunkSize = 1024, size_t transmitBuffers = 10,
1831             Conn conn = Conn())
1832 if (isCurlConn!(Conn))
1833 {
1834     static if (is(Conn : AutoProtocol))
1835     {
1836         if (isFTPUrl(url))
1837             return byChunkAsync(url, cast(void[]) null, chunkSize,
1838                                 transmitBuffers, FTP());
1839         else
1840             return byChunkAsync(url, cast(void[]) null, chunkSize,
1841                                 transmitBuffers, HTTP());
1842     }
1843     else
1844     {
1845         return byChunkAsync(url, cast(void[]) null, chunkSize,
1846                             transmitBuffers, conn);
1847     }
1848 }
1849 
1850 @system unittest
1851 {
1852     import std.algorithm.comparison : equal;
1853 
1854     foreach (host; [testServer.addr, "http://"~testServer.addr])
1855     {
1856         testServer.handle((s) {
1857             auto req = s.recvReq;
1858             s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
1859         });
1860         assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
1861     }
1862 }
1863 
1864 
1865 /*
1866   Mixin template for all supported curl protocols. This is the commom
1867   functionallity such as timeouts and network interface settings. This should
1868   really be in the HTTP/FTP/SMTP structs but the documentation tool does not
1869   support a mixin to put its doc strings where a mixin is done. Therefore docs
1870   in this template is copied into each of HTTP/FTP/SMTP below.
1871 */
1872 private mixin template Protocol()
1873 {
1874     import etc.c.curl : CurlReadFunc, RawCurlProxy = CurlProxy;
1875     import core.time : Duration;
1876     import std.socket : InternetAddress;
1877 
1878     /// Value to return from `onSend`/`onReceive` delegates in order to
1879     /// pause a request
1880     alias requestPause = CurlReadFunc.pause;
1881 
1882     /// Value to return from onSend delegate in order to abort a request
1883     alias requestAbort = CurlReadFunc.abort;
1884 
1885     static uint defaultAsyncStringBufferSize = 100;
1886 
1887     /**
1888        The curl handle used by this connection.
1889     */
1890     @property ref Curl handle() return
1891     {
1892         return p.curl;
1893     }
1894 
1895     /**
1896        True if the instance is stopped. A stopped instance is not usable.
1897     */
1898     @property bool isStopped()
1899     {
1900         return p.curl.stopped;
1901     }
1902 
1903     /// Stop and invalidate this instance.
1904     void shutdown()
1905     {
1906         p.curl.shutdown();
1907     }
1908 
1909     /** Set verbose.
1910         This will print request information to stderr.
1911      */
1912     @property void verbose(bool on)
1913     {
1914         p.curl.set(CurlOption.verbose, on ? 1L : 0L);
1915     }
1916 
1917     // Connection settings
1918 
1919     /// Set timeout for activity on connection.
1920     @property void dataTimeout(Duration d)
1921     {
1922         p.curl.set(CurlOption.low_speed_limit, 1);
1923         p.curl.set(CurlOption.low_speed_time, d.total!"seconds");
1924     }
1925 
1926     /** Set maximum time an operation is allowed to take.
1927         This includes dns resolution, connecting, data transfer, etc.
1928      */
1929     @property void operationTimeout(Duration d)
1930     {
1931         p.curl.set(CurlOption.timeout_ms, d.total!"msecs");
1932     }
1933 
1934     /// Set timeout for connecting.
1935     @property void connectTimeout(Duration d)
1936     {
1937         p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs");
1938     }
1939 
1940     // Network settings
1941 
1942     /** Proxy
1943      *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
1944      */
1945     @property void proxy(const(char)[] host)
1946     {
1947         p.curl.set(CurlOption.proxy, host);
1948     }
1949 
1950     /** Proxy port
1951      *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
1952      */
1953     @property void proxyPort(ushort port)
1954     {
1955         p.curl.set(CurlOption.proxyport, cast(long) port);
1956     }
1957 
1958     /// Type of proxy
1959     alias CurlProxy = RawCurlProxy;
1960 
1961     /** Proxy type
1962      *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
1963      */
1964     @property void proxyType(CurlProxy type)
1965     {
1966         p.curl.set(CurlOption.proxytype, cast(long) type);
1967     }
1968 
1969     /// DNS lookup timeout.
1970     @property void dnsTimeout(Duration d)
1971     {
1972         p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs");
1973     }
1974 
1975     /**
1976      * The network interface to use in form of the the IP of the interface.
1977      *
1978      * Example:
1979      * ----
1980      * theprotocol.netInterface = "192.168.1.32";
1981      * theprotocol.netInterface = [ 192, 168, 1, 32 ];
1982      * ----
1983      *
1984      * See: $(REF InternetAddress, std,socket)
1985      */
1986     @property void netInterface(const(char)[] i)
1987     {
1988         p.curl.set(CurlOption.intrface, i);
1989     }
1990 
1991     /// ditto
1992     @property void netInterface(const(ubyte)[4] i)
1993     {
1994         import std.format : format;
1995         const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]);
1996         netInterface = str;
1997     }
1998 
1999     /// ditto
2000     @property void netInterface(InternetAddress i)
2001     {
2002         netInterface = i.toAddrString();
2003     }
2004 
2005     /**
2006        Set the local outgoing port to use.
2007        Params:
2008        port = the first outgoing port number to try and use
2009     */
2010     @property void localPort(ushort port)
2011     {
2012         p.curl.set(CurlOption.localport, cast(long) port);
2013     }
2014 
2015     /**
2016        Set the no proxy flag for the specified host names.
2017        Params:
2018        test = a list of comma host names that do not require
2019               proxy to get reached
2020     */
2021     void setNoProxy(string hosts)
2022     {
2023         p.curl.set(CurlOption.noproxy, hosts);
2024     }
2025 
2026     /**
2027        Set the local outgoing port range to use.
2028        This can be used together with the localPort property.
2029        Params:
2030        range = if the first port is occupied then try this many
2031                port number forwards
2032     */
2033     @property void localPortRange(ushort range)
2034     {
2035         p.curl.set(CurlOption.localportrange, cast(long) range);
2036     }
2037 
2038     /** Set the tcp no-delay socket option on or off.
2039         See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
2040     */
2041     @property void tcpNoDelay(bool on)
2042     {
2043         p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) );
2044     }
2045 
2046     /** Sets whether SSL peer certificates should be verified.
2047         See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer)
2048     */
2049     @property void verifyPeer(bool on)
2050     {
2051       p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0);
2052     }
2053 
2054     /** Sets whether the host within an SSL certificate should be verified.
2055         See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer)
2056     */
2057     @property void verifyHost(bool on)
2058     {
2059       p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0);
2060     }
2061 
2062     // Authentication settings
2063 
2064     /**
2065        Set the user name, password and optionally domain for authentication
2066        purposes.
2067 
2068        Some protocols may need authentication in some cases. Use this
2069        function to provide credentials.
2070 
2071        Params:
2072        username = the username
2073        password = the password
2074        domain = used for NTLM authentication only and is set to the NTLM domain
2075                 name
2076     */
2077     void setAuthentication(const(char)[] username, const(char)[] password,
2078                            const(char)[] domain = "")
2079     {
2080         import std.format : format;
2081         if (!domain.empty)
2082             username = format("%s/%s", domain, username);
2083         p.curl.set(CurlOption.userpwd, format("%s:%s", username, password));
2084     }
2085 
2086     @system unittest
2087     {
2088         import std.algorithm.searching : canFind;
2089 
2090         testServer.handle((s) {
2091             auto req = s.recvReq;
2092             assert(req.hdrs.canFind("GET /"));
2093             assert(req.hdrs.canFind("Basic dXNlcjpwYXNz"));
2094             s.send(httpOK());
2095         });
2096 
2097         auto http = HTTP(testServer.addr);
2098         http.onReceive = (ubyte[] data) { return data.length; };
2099         http.setAuthentication("user", "pass");
2100         http.perform();
2101 
2102         // https://issues.dlang.org/show_bug.cgi?id=17540
2103         http.setNoProxy("www.example.com");
2104     }
2105 
2106     /**
2107        Set the user name and password for proxy authentication.
2108 
2109        Params:
2110        username = the username
2111        password = the password
2112     */
2113     void setProxyAuthentication(const(char)[] username, const(char)[] password)
2114     {
2115         import std.array : replace;
2116         import std.format : format;
2117 
2118         p.curl.set(CurlOption.proxyuserpwd,
2119             format("%s:%s",
2120                 username.replace(":", "%3A"),
2121                 password.replace(":", "%3A"))
2122         );
2123     }
2124 
2125     /**
2126      * The event handler that gets called when data is needed for sending. The
2127      * length of the `void[]` specifies the maximum number of bytes that can
2128      * be sent.
2129      *
2130      * Returns:
2131      * The callback returns the number of elements in the buffer that have been
2132      * filled and are ready to send.
2133      * The special value `.abortRequest` can be returned in order to abort the
2134      * current request.
2135      * The special value `.pauseRequest` can be returned in order to pause the
2136      * current request.
2137      *
2138      * Example:
2139      * ----
2140      * import std.net.curl;
2141      * string msg = "Hello world";
2142      * auto client = HTTP("dlang.org");
2143      * client.onSend = delegate size_t(void[] data)
2144      * {
2145      *     auto m = cast(void[]) msg;
2146      *     size_t length = m.length > data.length ? data.length : m.length;
2147      *     if (length == 0) return 0;
2148      *     data[0 .. length] = m[0 .. length];
2149      *     msg = msg[length..$];
2150      *     return length;
2151      * };
2152      * client.perform();
2153      * ----
2154      */
2155     @property void onSend(size_t delegate(void[]) callback)
2156     {
2157         p.curl.clear(CurlOption.postfields); // cannot specify data when using callback
2158         p.curl.onSend = callback;
2159     }
2160 
2161     /**
2162       * The event handler that receives incoming data. Be sure to copy the
2163       * incoming ubyte[] since it is not guaranteed to be valid after the
2164       * callback returns.
2165       *
2166       * Returns:
2167       * The callback returns the number of incoming bytes read. If the entire array is
2168       * not read the request will abort.
2169       * The special value .pauseRequest can be returned in order to pause the
2170       * current request.
2171       *
2172       * Example:
2173       * ----
2174       * import std.net.curl, std.stdio;
2175       * auto client = HTTP("dlang.org");
2176       * client.onReceive = (ubyte[] data)
2177       * {
2178       *     writeln("Got data", to!(const(char)[])(data));
2179       *     return data.length;
2180       * };
2181       * client.perform();
2182       * ----
2183       */
2184     @property void onReceive(size_t delegate(ubyte[]) callback)
2185     {
2186         p.curl.onReceive = callback;
2187     }
2188 
2189     /**
2190       * The event handler that gets called to inform of upload/download progress.
2191       *
2192       * Params:
2193       * dlTotal = total bytes to download
2194       * dlNow = currently downloaded bytes
2195       * ulTotal = total bytes to upload
2196       * ulNow = currently uploaded bytes
2197       *
2198       * Returns:
2199       * Return 0 from the callback to signal success, return non-zero to abort
2200       *          transfer
2201       *
2202       * Example:
2203       * ----
2204       * import std.net.curl, std.stdio;
2205       * auto client = HTTP("dlang.org");
2206       * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln)
2207       * {
2208       *     writeln("Progress: downloaded ", dln, " of ", dl);
2209       *     writeln("Progress: uploaded ", uln, " of ", ul);
2210       *     return 0;
2211       * };
2212       * client.perform();
2213       * ----
2214       */
2215     @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
2216                                            size_t ulTotal, size_t ulNow) callback)
2217     {
2218         p.curl.onProgress = callback;
2219     }
2220 }
2221 
2222 /*
2223   Decode `ubyte[]` array using the provided EncodingScheme up to maxChars
2224   Returns: Tuple of ubytes read and the `Char[]` characters decoded.
2225            Not all ubytes are guaranteed to be read in case of decoding error.
2226 */
2227 private Tuple!(size_t,Char[])
2228 decodeString(Char = char)(const(ubyte)[] data,
2229                           EncodingScheme scheme,
2230                           size_t maxChars = size_t.max)
2231 {
2232     import std.encoding : INVALID_SEQUENCE;
2233     Char[] res;
2234     immutable startLen = data.length;
2235     size_t charsDecoded = 0;
2236     while (data.length && charsDecoded < maxChars)
2237     {
2238         immutable dchar dc = scheme.safeDecode(data);
2239         if (dc == INVALID_SEQUENCE)
2240         {
2241             return typeof(return)(size_t.max, cast(Char[]) null);
2242         }
2243         charsDecoded++;
2244         res ~= dc;
2245     }
2246     return typeof(return)(startLen-data.length, res);
2247 }
2248 
2249 /*
2250   Decode `ubyte[]` array using the provided `EncodingScheme` until a the
2251   line terminator specified is found. The basesrc parameter is effectively
2252   prepended to src as the first thing.
2253 
2254   This function is used for decoding as much of the src buffer as
2255   possible until either the terminator is found or decoding fails. If
2256   it fails as the last data in the src it may mean that the src buffer
2257   were missing some bytes in order to represent a correct code
2258   point. Upon the next call to this function more bytes have been
2259   received from net and the failing bytes should be given as the
2260   basesrc parameter. It is done this way to minimize data copying.
2261 
2262   Returns: true if a terminator was found
2263            Not all ubytes are guaranteed to be read in case of decoding error.
2264            any decoded chars will be inserted into dst.
2265 */
2266 private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc,
2267                                                      ref const(ubyte)[] src,
2268                                                      ref Char[] dst,
2269                                                      EncodingScheme scheme,
2270                                                      Terminator terminator)
2271 {
2272     import std.algorithm.searching : endsWith;
2273     import std.encoding : INVALID_SEQUENCE;
2274     import std.exception : enforce;
2275 
2276     // if there is anything in the basesrc then try to decode that
2277     // first.
2278     if (basesrc.length != 0)
2279     {
2280         // Try to ensure 4 entries in the basesrc by copying from src.
2281         immutable blen = basesrc.length;
2282         immutable len = (basesrc.length + src.length) >= 4 ?
2283                      4 : basesrc.length + src.length;
2284         basesrc.length = len;
2285 
2286         immutable dchar dc = scheme.safeDecode(basesrc);
2287         if (dc == INVALID_SEQUENCE)
2288         {
2289             enforce!CurlException(len != 4, "Invalid code sequence");
2290             return false;
2291         }
2292         dst ~= dc;
2293         src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src
2294         basesrc.length = 0;
2295     }
2296 
2297     while (src.length)
2298     {
2299         const lsrc = src;
2300         dchar dc = scheme.safeDecode(src);
2301         if (dc == INVALID_SEQUENCE)
2302         {
2303             if (src.empty)
2304             {
2305                 // The invalid sequence was in the end of the src.  Maybe there
2306                 // just need to be more bytes available so these last bytes are
2307                 // put back to src for later use.
2308                 src = lsrc;
2309                 return false;
2310             }
2311             dc = '?';
2312         }
2313         dst ~= dc;
2314 
2315         if (dst.endsWith(terminator))
2316             return true;
2317     }
2318     return false; // no terminator found
2319 }
2320 
2321 /**
2322   * HTTP client functionality.
2323   *
2324   * Example:
2325   *
2326   * Get with custom data receivers:
2327   *
2328   * ---
2329   * import std.net.curl, std.stdio;
2330   *
2331   * auto http = HTTP("https://dlang.org");
2332   * http.onReceiveHeader =
2333   *     (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); };
2334   * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
2335   * http.perform();
2336   * ---
2337   *
2338   */
2339 
2340 /**
2341   * Put with data senders:
2342   *
2343   * ---
2344   * import std.net.curl, std.stdio;
2345   *
2346   * auto http = HTTP("https://dlang.org");
2347   * auto msg = "Hello world";
2348   * http.contentLength = msg.length;
2349   * http.onSend = (void[] data)
2350   * {
2351   *     auto m = cast(void[]) msg;
2352   *     size_t len = m.length > data.length ? data.length : m.length;
2353   *     if (len == 0) return len;
2354   *     data[0 .. len] = m[0 .. len];
2355   *     msg = msg[len..$];
2356   *     return len;
2357   * };
2358   * http.perform();
2359   * ---
2360   *
2361   */
2362 
2363 /**
2364   * Tracking progress:
2365   *
2366   * ---
2367   * import std.net.curl, std.stdio;
2368   *
2369   * auto http = HTTP();
2370   * http.method = HTTP.Method.get;
2371   * http.url = "http://upload.wikimedia.org/wikipedia/commons/" ~
2372   *            "5/53/Wikipedia-logo-en-big.png";
2373   * http.onReceive = (ubyte[] data) { return data.length; };
2374   * http.onProgress = (size_t dltotal, size_t dlnow,
2375   *                    size_t ultotal, size_t ulnow)
2376   * {
2377   *     writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow);
2378   *     return 0;
2379   * };
2380   * http.perform();
2381   * ---
2382   *
2383   * See_Also: $(LINK2 http://www.ietf.org/rfc/rfc2616.txt, RFC2616)
2384   *
2385   */
2386 struct HTTP
2387 {
2388     mixin Protocol;
2389 
2390     import std.datetime.systime : SysTime;
2391     import std.typecons : RefCounted;
2392     import etc.c.curl : CurlAuth, CurlInfo, curl_slist, CURLVERSION_NOW, curl_off_t;
2393 
2394     /// Authentication method equal to $(REF CurlAuth, etc,c,curl)
2395     alias AuthMethod = CurlAuth;
2396 
2397     static private uint defaultMaxRedirects = 10;
2398 
2399     private struct Impl
2400     {
2401         ~this()
2402         {
2403             if (headersOut !is null)
2404                 Curl.curl.slist_free_all(headersOut);
2405             if (curl.handle !is null) // work around RefCounted/emplace bug
2406                 curl.shutdown();
2407         }
2408         Curl curl;
2409         curl_slist* headersOut;
2410         string[string] headersIn;
2411         string charset;
2412 
2413         /// The status line of the final sub-request in a request.
2414         StatusLine status;
2415         private void delegate(StatusLine) onReceiveStatusLine;
2416 
2417         /// The HTTP method to use.
2418         Method method = Method.undefined;
2419 
2420         @system @property void onReceiveHeader(void delegate(in char[] key,
2421                                                      in char[] value) callback)
2422         {
2423             import std.algorithm.searching : startsWith;
2424             import std.regex : regex, match;
2425             import std.uni : toLower;
2426 
2427             // Wrap incoming callback in order to separate http status line from
2428             // http headers.  On redirected requests there may be several such
2429             // status lines. The last one is the one recorded.
2430             auto dg = (in char[] header)
2431             {
2432                 import std.utf : UTFException;
2433                 try
2434                 {
2435                     if (header.empty)
2436                     {
2437                         // header delimiter
2438                         return;
2439                     }
2440                     if (header.startsWith("HTTP/"))
2441                     {
2442                         headersIn.clear();
2443                         if (parseStatusLine(header, status))
2444                         {
2445                             if (onReceiveStatusLine != null)
2446                                 onReceiveStatusLine(status);
2447                         }
2448                         return;
2449                     }
2450 
2451                     // Normal http header
2452                     auto m = match(cast(char[]) header, regex("(.*?): (.*)$"));
2453 
2454                     auto fieldName = m.captures[1].toLower().idup;
2455                     if (fieldName == "content-type")
2456                     {
2457                         auto mct = match(cast(char[]) m.captures[2],
2458                                          regex("charset=([^;]*)", "i"));
2459                         if (!mct.empty && mct.captures.length > 1)
2460                             charset = mct.captures[1].idup;
2461                     }
2462 
2463                     if (!m.empty && callback !is null)
2464                         callback(fieldName, m.captures[2]);
2465                     headersIn[fieldName] = m.captures[2].idup;
2466                 }
2467                 catch (UTFException e)
2468                 {
2469                     //munch it - a header should be all ASCII, any "wrong UTF" is broken header
2470                 }
2471             };
2472 
2473             curl.onReceiveHeader = dg;
2474         }
2475     }
2476 
2477     private RefCounted!Impl p;
2478     import etc.c.curl : CurlTimeCond;
2479 
2480     /// Parse status line, as received from / generated by cURL.
2481     private static bool parseStatusLine(const char[] header, out StatusLine status) @safe
2482     {
2483         import std.conv : to;
2484         import std.regex : regex, match;
2485 
2486         const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$"));
2487         if (m.empty)
2488             return false; // Invalid status line
2489         else
2490         {
2491             status.majorVersion = to!ushort(m.captures[1]);
2492             status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0;
2493             status.code = to!ushort(m.captures[3]);
2494             status.reason = m.captures[4].idup;
2495             return true;
2496         }
2497     }
2498 
2499     @safe unittest
2500     {
2501         StatusLine status;
2502         assert(parseStatusLine("HTTP/1.1 200 OK", status)
2503             && status == StatusLine(1, 1, 200, "OK"));
2504         assert(parseStatusLine("HTTP/1.0 304 Not Modified", status)
2505             && status == StatusLine(1, 0, 304, "Not Modified"));
2506         // The HTTP2 protocol is binary; cURL generates this fake text header.
2507         assert(parseStatusLine("HTTP/2 200", status)
2508             && status == StatusLine(2, 0, 200, null));
2509     }
2510 
2511     /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl)
2512 
2513         $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
2514     */
2515     alias TimeCond = CurlTimeCond;
2516 
2517     /**
2518        Constructor taking the url as parameter.
2519     */
2520     static HTTP opCall(const(char)[] url)
2521     {
2522         HTTP http;
2523         http.initialize();
2524         http.url = url;
2525         return http;
2526     }
2527 
2528     ///
2529     static HTTP opCall()
2530     {
2531         HTTP http;
2532         http.initialize();
2533         return http;
2534     }
2535 
2536     ///
2537     HTTP dup()
2538     {
2539         HTTP copy;
2540         copy.initialize();
2541         copy.p.method = p.method;
2542         curl_slist* cur = p.headersOut;
2543         curl_slist* newlist = null;
2544         while (cur)
2545         {
2546             newlist = Curl.curl.slist_append(newlist, cur.data);
2547             cur = cur.next;
2548         }
2549         copy.p.headersOut = newlist;
2550         copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut);
2551         copy.p.curl = p.curl.dup();
2552         copy.dataTimeout = _defaultDataTimeout;
2553         copy.onReceiveHeader = null;
2554         return copy;
2555     }
2556 
2557     private void initialize()
2558     {
2559         p.curl.initialize();
2560         maxRedirects = HTTP.defaultMaxRedirects;
2561         p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC
2562         p.method = Method.undefined;
2563         setUserAgent(HTTP.defaultUserAgent);
2564         dataTimeout = _defaultDataTimeout;
2565         onReceiveHeader = null;
2566         verifyPeer = true;
2567         verifyHost = true;
2568     }
2569 
2570     /**
2571        Perform a http request.
2572 
2573        After the HTTP client has been setup and possibly assigned callbacks the
2574        `perform()` method will start performing the request towards the
2575        specified server.
2576 
2577        Params:
2578        throwOnError = whether to throw an exception or return a CurlCode on error
2579     */
2580     CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
2581     {
2582         p.status.reset();
2583 
2584         CurlOption opt;
2585         final switch (p.method)
2586         {
2587         case Method.head:
2588             p.curl.set(CurlOption.nobody, 1L);
2589             opt = CurlOption.nobody;
2590             break;
2591         case Method.undefined:
2592         case Method.get:
2593             p.curl.set(CurlOption.httpget, 1L);
2594             opt = CurlOption.httpget;
2595             break;
2596         case Method.post:
2597             p.curl.set(CurlOption.post, 1L);
2598             opt = CurlOption.post;
2599             break;
2600         case Method.put:
2601             p.curl.set(CurlOption.upload, 1L);
2602             opt = CurlOption.upload;
2603             break;
2604         case Method.del:
2605             p.curl.set(CurlOption.customrequest, "DELETE");
2606             opt = CurlOption.customrequest;
2607             break;
2608         case Method.options:
2609             p.curl.set(CurlOption.customrequest, "OPTIONS");
2610             opt = CurlOption.customrequest;
2611             break;
2612         case Method.trace:
2613             p.curl.set(CurlOption.customrequest, "TRACE");
2614             opt = CurlOption.customrequest;
2615             break;
2616         case Method.connect:
2617             p.curl.set(CurlOption.customrequest, "CONNECT");
2618             opt = CurlOption.customrequest;
2619             break;
2620         case Method.patch:
2621             p.curl.set(CurlOption.customrequest, "PATCH");
2622             opt = CurlOption.customrequest;
2623             break;
2624         }
2625 
2626         scope (exit) p.curl.clear(opt);
2627         return p.curl.perform(throwOnError);
2628     }
2629 
2630     /// The URL to specify the location of the resource.
2631     @property void url(const(char)[] url)
2632     {
2633         import std.algorithm.searching : startsWith;
2634         import std.uni : toLower;
2635         if (!startsWith(url.toLower(), "http://", "https://"))
2636             url = "http://" ~ url;
2637         p.curl.set(CurlOption.url, url);
2638     }
2639 
2640     /// Set the CA certificate bundle file to use for SSL peer verification
2641     @property void caInfo(const(char)[] caFile)
2642     {
2643         p.curl.set(CurlOption.cainfo, caFile);
2644     }
2645 
2646     // This is a workaround for mixed in content not having its
2647     // docs mixed in.
2648     version (StdDdoc)
2649     {
2650         static import etc.c.curl;
2651 
2652         /// Value to return from `onSend`/`onReceive` delegates in order to
2653         /// pause a request
2654         alias requestPause = CurlReadFunc.pause;
2655 
2656         /// Value to return from onSend delegate in order to abort a request
2657         alias requestAbort = CurlReadFunc.abort;
2658 
2659         /**
2660            True if the instance is stopped. A stopped instance is not usable.
2661         */
2662         @property bool isStopped();
2663 
2664         /// Stop and invalidate this instance.
2665         void shutdown();
2666 
2667         /** Set verbose.
2668             This will print request information to stderr.
2669         */
2670         @property void verbose(bool on);
2671 
2672         // Connection settings
2673 
2674         /// Set timeout for activity on connection.
2675         @property void dataTimeout(Duration d);
2676 
2677         /** Set maximum time an operation is allowed to take.
2678             This includes dns resolution, connecting, data transfer, etc.
2679           */
2680         @property void operationTimeout(Duration d);
2681 
2682         /// Set timeout for connecting.
2683         @property void connectTimeout(Duration d);
2684 
2685         // Network settings
2686 
2687         /** Proxy
2688          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
2689          */
2690         @property void proxy(const(char)[] host);
2691 
2692         /** Proxy port
2693          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
2694          */
2695         @property void proxyPort(ushort port);
2696 
2697         /// Type of proxy
2698         alias CurlProxy = etc.c.curl.CurlProxy;
2699 
2700         /** Proxy type
2701          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
2702          */
2703         @property void proxyType(CurlProxy type);
2704 
2705         /// DNS lookup timeout.
2706         @property void dnsTimeout(Duration d);
2707 
2708         /**
2709          * The network interface to use in form of the the IP of the interface.
2710          *
2711          * Example:
2712          * ----
2713          * theprotocol.netInterface = "192.168.1.32";
2714          * theprotocol.netInterface = [ 192, 168, 1, 32 ];
2715          * ----
2716          *
2717          * See: $(REF InternetAddress, std,socket)
2718          */
2719         @property void netInterface(const(char)[] i);
2720 
2721         /// ditto
2722         @property void netInterface(const(ubyte)[4] i);
2723 
2724         /// ditto
2725         @property void netInterface(InternetAddress i);
2726 
2727         /**
2728            Set the local outgoing port to use.
2729            Params:
2730            port = the first outgoing port number to try and use
2731         */
2732         @property void localPort(ushort port);
2733 
2734         /**
2735            Set the local outgoing port range to use.
2736            This can be used together with the localPort property.
2737            Params:
2738            range = if the first port is occupied then try this many
2739            port number forwards
2740         */
2741         @property void localPortRange(ushort range);
2742 
2743         /** Set the tcp no-delay socket option on or off.
2744             See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
2745         */
2746         @property void tcpNoDelay(bool on);
2747 
2748         // Authentication settings
2749 
2750         /**
2751            Set the user name, password and optionally domain for authentication
2752            purposes.
2753 
2754            Some protocols may need authentication in some cases. Use this
2755            function to provide credentials.
2756 
2757            Params:
2758            username = the username
2759            password = the password
2760            domain = used for NTLM authentication only and is set to the NTLM domain
2761            name
2762         */
2763         void setAuthentication(const(char)[] username, const(char)[] password,
2764                                const(char)[] domain = "");
2765 
2766         /**
2767            Set the user name and password for proxy authentication.
2768 
2769            Params:
2770            username = the username
2771            password = the password
2772         */
2773         void setProxyAuthentication(const(char)[] username, const(char)[] password);
2774 
2775         /**
2776          * The event handler that gets called when data is needed for sending. The
2777          * length of the `void[]` specifies the maximum number of bytes that can
2778          * be sent.
2779          *
2780          * Returns:
2781          * The callback returns the number of elements in the buffer that have been
2782          * filled and are ready to send.
2783          * The special value `.abortRequest` can be returned in order to abort the
2784          * current request.
2785          * The special value `.pauseRequest` can be returned in order to pause the
2786          * current request.
2787          *
2788          * Example:
2789          * ----
2790          * import std.net.curl;
2791          * string msg = "Hello world";
2792          * auto client = HTTP("dlang.org");
2793          * client.onSend = delegate size_t(void[] data)
2794          * {
2795          *     auto m = cast(void[]) msg;
2796          *     size_t length = m.length > data.length ? data.length : m.length;
2797          *     if (length == 0) return 0;
2798          *     data[0 .. length] = m[0 .. length];
2799          *     msg = msg[length..$];
2800          *     return length;
2801          * };
2802          * client.perform();
2803          * ----
2804          */
2805         @property void onSend(size_t delegate(void[]) callback);
2806 
2807         /**
2808          * The event handler that receives incoming data. Be sure to copy the
2809          * incoming ubyte[] since it is not guaranteed to be valid after the
2810          * callback returns.
2811          *
2812          * Returns:
2813          * The callback returns the incoming bytes read. If not the entire array is
2814          * the request will abort.
2815          * The special value .pauseRequest can be returned in order to pause the
2816          * current request.
2817          *
2818          * Example:
2819          * ----
2820          * import std.net.curl, std.stdio;
2821          * auto client = HTTP("dlang.org");
2822          * client.onReceive = (ubyte[] data)
2823          * {
2824          *     writeln("Got data", to!(const(char)[])(data));
2825          *     return data.length;
2826          * };
2827          * client.perform();
2828          * ----
2829          */
2830         @property void onReceive(size_t delegate(ubyte[]) callback);
2831 
2832         /**
2833          * Register an event handler that gets called to inform of
2834          * upload/download progress.
2835          *
2836          * Callback_parameters:
2837          * $(CALLBACK_PARAMS)
2838          *
2839          * Callback_returns: Return 0 to signal success, return non-zero to
2840          * abort transfer.
2841          *
2842          * Example:
2843          * ----
2844          * import std.net.curl, std.stdio;
2845          * auto client = HTTP("dlang.org");
2846          * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln)
2847          * {
2848          *     writeln("Progress: downloaded ", dln, " of ", dl);
2849          *     writeln("Progress: uploaded ", uln, " of ", ul);
2850          *     return 0;
2851          * };
2852          * client.perform();
2853          * ----
2854          */
2855         @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
2856                                                size_t ulTotal, size_t ulNow) callback);
2857     }
2858 
2859     /** Clear all outgoing headers.
2860     */
2861     void clearRequestHeaders()
2862     {
2863         if (p.headersOut !is null)
2864             Curl.curl.slist_free_all(p.headersOut);
2865         p.headersOut = null;
2866         p.curl.clear(CurlOption.httpheader);
2867     }
2868 
2869     /** Add a header e.g. "X-CustomField: Something is fishy".
2870      *
2871      * There is no remove header functionality. Do a $(LREF clearRequestHeaders)
2872      * and set the needed headers instead.
2873      *
2874      * Example:
2875      * ---
2876      * import std.net.curl;
2877      * auto client = HTTP();
2878      * client.addRequestHeader("X-Custom-ABC", "This is the custom value");
2879      * auto content = get("dlang.org", client);
2880      * ---
2881      */
2882     void addRequestHeader(const(char)[] name, const(char)[] value)
2883     {
2884         import std.format : format;
2885         import std.internal.cstring : tempCString;
2886         import std.uni : icmp;
2887 
2888         if (icmp(name, "User-Agent") == 0)
2889             return setUserAgent(value);
2890         string nv = format("%s: %s", name, value);
2891         p.headersOut = Curl.curl.slist_append(p.headersOut,
2892                                               nv.tempCString().buffPtr);
2893         p.curl.set(CurlOption.httpheader, p.headersOut);
2894     }
2895 
2896     /**
2897      * The default "User-Agent" value send with a request.
2898      * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))"
2899      */
2900     static string defaultUserAgent() @property
2901     {
2902         import std.compiler : version_major, version_minor;
2903         import std.format : format, sformat;
2904 
2905         // http://curl.haxx.se/docs/versions.html
2906         enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)";
2907         enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3;
2908 
2909         static char[maxLen] buf = void;
2910         static string userAgent;
2911 
2912         if (!userAgent.length)
2913         {
2914             auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num;
2915             userAgent = cast(immutable) sformat(
2916                 buf, fmt, version_major, version_minor,
2917                 curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF);
2918         }
2919         return userAgent;
2920     }
2921 
2922     /** Set the value of the user agent request header field.
2923      *
2924      * By default a request has it's "User-Agent" field set to $(LREF
2925      * defaultUserAgent) even if `setUserAgent` was never called.  Pass
2926      * an empty string to suppress the "User-Agent" field altogether.
2927      */
2928     void setUserAgent(const(char)[] userAgent)
2929     {
2930         p.curl.set(CurlOption.useragent, userAgent);
2931     }
2932 
2933     /**
2934      * Get various timings defined in $(REF CurlInfo, etc, c, curl).
2935      * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`.
2936      *
2937      * Params:
2938      *      timing = one of the timings defined in $(REF CurlInfo, etc, c, curl).
2939      *               The values are:
2940      *               `etc.c.curl.CurlInfo.namelookup_time`,
2941      *               `etc.c.curl.CurlInfo.connect_time`,
2942      *               `etc.c.curl.CurlInfo.pretransfer_time`,
2943      *               `etc.c.curl.CurlInfo.starttransfer_time`,
2944      *               `etc.c.curl.CurlInfo.redirect_time`,
2945      *               `etc.c.curl.CurlInfo.appconnect_time`,
2946      *               `etc.c.curl.CurlInfo.total_time`.
2947      *      val    = the actual value of the inquired timing.
2948      *
2949      * Returns:
2950      *      The return code of the operation. The value stored in val
2951      *      should be used only if the return value is `etc.c.curl.CurlInfo.ok`.
2952      *
2953      * Example:
2954      * ---
2955      * import std.net.curl;
2956      * import etc.c.curl : CurlError, CurlInfo;
2957      *
2958      * auto client = HTTP("dlang.org");
2959      * client.perform();
2960      *
2961      * double val;
2962      * CurlCode code;
2963      *
2964      * code = client.getTiming(CurlInfo.namelookup_time, val);
2965      * assert(code == CurlError.ok);
2966      * ---
2967      */
2968     CurlCode getTiming(CurlInfo timing, ref double val)
2969     {
2970         return p.curl.getTiming(timing, val);
2971     }
2972 
2973     /** The headers read from a successful response.
2974      *
2975      */
2976     @property string[string] responseHeaders()
2977     {
2978         return p.headersIn;
2979     }
2980 
2981     /// HTTP method used.
2982     @property void method(Method m)
2983     {
2984         p.method = m;
2985     }
2986 
2987     /// ditto
2988     @property Method method()
2989     {
2990         return p.method;
2991     }
2992 
2993     /**
2994        HTTP status line of last response. One call to perform may
2995        result in several requests because of redirection.
2996     */
2997     @property StatusLine statusLine()
2998     {
2999         return p.status;
3000     }
3001 
3002     /// Set the active cookie string e.g. "name1=value1;name2=value2"
3003     void setCookie(const(char)[] cookie)
3004     {
3005         p.curl.set(CurlOption.cookie, cookie);
3006     }
3007 
3008     /// Set a file path to where a cookie jar should be read/stored.
3009     void setCookieJar(const(char)[] path)
3010     {
3011         p.curl.set(CurlOption.cookiefile, path);
3012         if (path.length)
3013             p.curl.set(CurlOption.cookiejar, path);
3014     }
3015 
3016     /// Flush cookie jar to disk.
3017     void flushCookieJar()
3018     {
3019         p.curl.set(CurlOption.cookielist, "FLUSH");
3020     }
3021 
3022     /// Clear session cookies.
3023     void clearSessionCookies()
3024     {
3025         p.curl.set(CurlOption.cookielist, "SESS");
3026     }
3027 
3028     /// Clear all cookies.
3029     void clearAllCookies()
3030     {
3031         p.curl.set(CurlOption.cookielist, "ALL");
3032     }
3033 
3034     /**
3035        Set time condition on the request.
3036 
3037        Params:
3038        cond =  `CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}`
3039        timestamp = Timestamp for the condition
3040 
3041        $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
3042     */
3043     void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp)
3044     {
3045         p.curl.set(CurlOption.timecondition, cond);
3046         p.curl.set(CurlOption.timevalue, timestamp.toUnixTime());
3047     }
3048 
3049     /** Specifying data to post when not using the onSend callback.
3050       *
3051       * The data is NOT copied by the library.  Content-Type will default to
3052       * application/octet-stream.  Data is not converted or encoded by this
3053       * method.
3054       *
3055       * Example:
3056       * ----
3057       * import std.net.curl, std.stdio;
3058       * auto http = HTTP("http://www.mydomain.com");
3059       * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3060       * http.postData = [1,2,3,4,5];
3061       * http.perform();
3062       * ----
3063       */
3064     @property void postData(const(void)[] data)
3065     {
3066         setPostData(data, "application/octet-stream");
3067     }
3068 
3069     /** Specifying data to post when not using the onSend callback.
3070       *
3071       * The data is NOT copied by the library.  Content-Type will default to
3072       * text/plain.  Data is not converted or encoded by this method.
3073       *
3074       * Example:
3075       * ----
3076       * import std.net.curl, std.stdio;
3077       * auto http = HTTP("http://www.mydomain.com");
3078       * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3079       * http.postData = "The quick....";
3080       * http.perform();
3081       * ----
3082       */
3083     @property void postData(const(char)[] data)
3084     {
3085         setPostData(data, "text/plain");
3086     }
3087 
3088     /**
3089      * Specify data to post when not using the onSend callback, with
3090      * user-specified Content-Type.
3091      * Params:
3092      *  data = Data to post.
3093      *  contentType = MIME type of the data, for example, "text/plain" or
3094      *      "application/octet-stream". See also:
3095      *      $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type,
3096      *      Internet media type) on Wikipedia.
3097      * -----
3098      * import std.net.curl;
3099      * auto http = HTTP("http://onlineform.example.com");
3100      * auto data = "app=login&username=bob&password=s00perS3kret";
3101      * http.setPostData(data, "application/x-www-form-urlencoded");
3102      * http.onReceive = (ubyte[] data) { return data.length; };
3103      * http.perform();
3104      * -----
3105      */
3106     void setPostData(const(void)[] data, string contentType)
3107     {
3108         // cannot use callback when specifying data directly so it is disabled here.
3109         p.curl.clear(CurlOption.readfunction);
3110         addRequestHeader("Content-Type", contentType);
3111         p.curl.set(CurlOption.postfields, cast(void*) data.ptr);
3112         p.curl.set(CurlOption.postfieldsize, data.length);
3113         if (method == Method.undefined)
3114             method = Method.post;
3115     }
3116 
3117     @system unittest
3118     {
3119         import std.algorithm.searching : canFind;
3120 
3121         testServer.handle((s) {
3122             auto req = s.recvReq!ubyte;
3123             assert(req.hdrs.canFind("POST /path"));
3124             assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
3125             assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
3126             s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
3127         });
3128         auto data = new ubyte[](256);
3129         foreach (i, ref ub; data)
3130             ub = cast(ubyte) i;
3131 
3132         auto http = HTTP(testServer.addr~"/path");
3133         http.postData = data;
3134         ubyte[] res;
3135         http.onReceive = (data) { res ~= data; return data.length; };
3136         http.perform();
3137         assert(res == cast(ubyte[])[17, 27, 35, 41]);
3138     }
3139 
3140     /**
3141       * Set the event handler that receives incoming headers.
3142       *
3143       * The callback will receive a header field key, value as parameter. The
3144       * `const(char)[]` arrays are not valid after the delegate has returned.
3145       *
3146       * Example:
3147       * ----
3148       * import std.net.curl, std.stdio;
3149       * auto http = HTTP("dlang.org");
3150       * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
3151       * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); };
3152       * http.perform();
3153       * ----
3154       */
3155     @property void onReceiveHeader(void delegate(in char[] key,
3156                                                  in char[] value) callback)
3157     {
3158         p.onReceiveHeader = callback;
3159     }
3160 
3161     /**
3162        Callback for each received StatusLine.
3163 
3164        Notice that several callbacks can be done for each call to
3165        `perform()` due to redirections.
3166 
3167        See_Also: $(LREF StatusLine)
3168      */
3169     @property void onReceiveStatusLine(void delegate(StatusLine) callback)
3170     {
3171         p.onReceiveStatusLine = callback;
3172     }
3173 
3174     /**
3175        The content length in bytes when using request that has content
3176        e.g. POST/PUT and not using chunked transfer. Is set as the
3177        "Content-Length" header.  Set to ulong.max to reset to chunked transfer.
3178     */
3179     @property void contentLength(ulong len)
3180     {
3181         import std.conv : to;
3182 
3183         CurlOption lenOpt;
3184 
3185         // Force post if necessary
3186         if (p.method != Method.put && p.method != Method.post &&
3187             p.method != Method.patch)
3188             p.method = Method.post;
3189 
3190         if (p.method == Method.post || p.method == Method.patch)
3191             lenOpt = CurlOption.postfieldsize_large;
3192         else
3193             lenOpt = CurlOption.infilesize_large;
3194 
3195         if (size_t.max != ulong.max && len == size_t.max)
3196             len = ulong.max; // check size_t.max for backwards compat, turn into error
3197 
3198         if (len == ulong.max)
3199         {
3200             // HTTP 1.1 supports requests with no length header set.
3201             addRequestHeader("Transfer-Encoding", "chunked");
3202             addRequestHeader("Expect", "100-continue");
3203         }
3204         else
3205         {
3206             p.curl.set(lenOpt, to!curl_off_t(len));
3207         }
3208     }
3209 
3210     /**
3211        Authentication method as specified in $(LREF AuthMethod).
3212     */
3213     @property void authenticationMethod(AuthMethod authMethod)
3214     {
3215         p.curl.set(CurlOption.httpauth, cast(long) authMethod);
3216     }
3217 
3218     /**
3219        Set max allowed redirections using the location header.
3220        uint.max for infinite.
3221     */
3222     @property void maxRedirects(uint maxRedirs)
3223     {
3224         if (maxRedirs == uint.max)
3225         {
3226             // Disable
3227             p.curl.set(CurlOption.followlocation, 0);
3228         }
3229         else
3230         {
3231             p.curl.set(CurlOption.followlocation, 1);
3232             p.curl.set(CurlOption.maxredirs, maxRedirs);
3233         }
3234     }
3235 
3236     /** <a name="HTTP.Method"/>The standard HTTP methods :
3237      *  $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1)
3238      */
3239     enum Method
3240     {
3241         undefined,
3242         head, ///
3243         get,  ///
3244         post, ///
3245         put,  ///
3246         del,  ///
3247         options, ///
3248         trace,   ///
3249         connect,  ///
3250         patch, ///
3251     }
3252 
3253     /**
3254        HTTP status line ie. the first line returned in an HTTP response.
3255 
3256        If authentication or redirections are done then the status will be for
3257        the last response received.
3258     */
3259     struct StatusLine
3260     {
3261         ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0.
3262         ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0.
3263         ushort code;         /// HTTP status line code e.g. 200.
3264         string reason;       /// HTTP status line reason string.
3265 
3266         /// Reset this status line
3267         @safe void reset()
3268         {
3269             majorVersion = 0;
3270             minorVersion = 0;
3271             code = 0;
3272             reason = "";
3273         }
3274 
3275         ///
3276         string toString() const
3277         {
3278             import std.format : format;
3279             return format("%s %s (%s.%s)",
3280                           code, reason, majorVersion, minorVersion);
3281         }
3282     }
3283 
3284 } // HTTP
3285 
3286 @system unittest // charset/Charset/CHARSET/...
3287 {
3288     import etc.c.curl;
3289 
3290     static foreach (c; ["charset", "Charset", "CHARSET", "CharSet", "charSet",
3291         "ChArSeT", "cHaRsEt"])
3292     {{
3293         testServer.handle((s) {
3294             s.send("HTTP/1.1 200 OK\r\n"~
3295                 "Content-Length: 0\r\n"~
3296                 "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~
3297                 "\r\n");
3298         });
3299 
3300         auto http = HTTP(testServer.addr);
3301         http.perform();
3302         assert(http.p.charset == "foo");
3303 
3304         // https://issues.dlang.org/show_bug.cgi?id=16736
3305         double val;
3306         CurlCode code;
3307 
3308         code = http.getTiming(CurlInfo.total_time, val);
3309         assert(code == CurlError.ok);
3310         code = http.getTiming(CurlInfo.namelookup_time, val);
3311         assert(code == CurlError.ok);
3312         code = http.getTiming(CurlInfo.connect_time, val);
3313         assert(code == CurlError.ok);
3314         code = http.getTiming(CurlInfo.pretransfer_time, val);
3315         assert(code == CurlError.ok);
3316         code = http.getTiming(CurlInfo.starttransfer_time, val);
3317         assert(code == CurlError.ok);
3318         code = http.getTiming(CurlInfo.redirect_time, val);
3319         assert(code == CurlError.ok);
3320         code = http.getTiming(CurlInfo.appconnect_time, val);
3321         assert(code == CurlError.ok);
3322     }}
3323 }
3324 
3325 /**
3326    FTP client functionality.
3327 
3328    See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959)
3329 */
3330 struct FTP
3331 {
3332 
3333     mixin Protocol;
3334 
3335     import std.typecons : RefCounted;
3336     import etc.c.curl : CurlError, CurlInfo, curl_off_t, curl_slist;
3337 
3338     private struct Impl
3339     {
3340         ~this()
3341         {
3342             if (commands !is null)
3343                 Curl.curl.slist_free_all(commands);
3344             if (curl.handle !is null) // work around RefCounted/emplace bug
3345                 curl.shutdown();
3346         }
3347         curl_slist* commands;
3348         Curl curl;
3349         string encoding;
3350     }
3351 
3352     private RefCounted!Impl p;
3353 
3354     /**
3355        FTP access to the specified url.
3356     */
3357     static FTP opCall(const(char)[] url)
3358     {
3359         FTP ftp;
3360         ftp.initialize();
3361         ftp.url = url;
3362         return ftp;
3363     }
3364 
3365     ///
3366     static FTP opCall()
3367     {
3368         FTP ftp;
3369         ftp.initialize();
3370         return ftp;
3371     }
3372 
3373     ///
3374     FTP dup()
3375     {
3376         FTP copy = FTP();
3377         copy.initialize();
3378         copy.p.encoding = p.encoding;
3379         copy.p.curl = p.curl.dup();
3380         curl_slist* cur = p.commands;
3381         curl_slist* newlist = null;
3382         while (cur)
3383         {
3384             newlist = Curl.curl.slist_append(newlist, cur.data);
3385             cur = cur.next;
3386         }
3387         copy.p.commands = newlist;
3388         copy.p.curl.set(CurlOption.postquote, copy.p.commands);
3389         copy.dataTimeout = _defaultDataTimeout;
3390         return copy;
3391     }
3392 
3393     private void initialize()
3394     {
3395         p.curl.initialize();
3396         p.encoding = "ISO-8859-1";
3397         dataTimeout = _defaultDataTimeout;
3398     }
3399 
3400     /**
3401        Performs the ftp request as it has been configured.
3402 
3403        After a FTP client has been setup and possibly assigned callbacks the $(D
3404        perform()) method will start performing the actual communication with the
3405        server.
3406 
3407        Params:
3408        throwOnError = whether to throw an exception or return a CurlCode on error
3409     */
3410     CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
3411     {
3412         return p.curl.perform(throwOnError);
3413     }
3414 
3415     /// The URL to specify the location of the resource.
3416     @property void url(const(char)[] url)
3417     {
3418         import std.algorithm.searching : startsWith;
3419         import std.uni : toLower;
3420 
3421         if (!startsWith(url.toLower(), "ftp://", "ftps://"))
3422             url = "ftp://" ~ url;
3423         p.curl.set(CurlOption.url, url);
3424     }
3425 
3426     // This is a workaround for mixed in content not having its
3427     // docs mixed in.
3428     version (StdDdoc)
3429     {
3430         static import etc.c.curl;
3431 
3432         /// Value to return from `onSend`/`onReceive` delegates in order to
3433         /// pause a request
3434         alias requestPause = CurlReadFunc.pause;
3435 
3436         /// Value to return from onSend delegate in order to abort a request
3437         alias requestAbort = CurlReadFunc.abort;
3438 
3439         /**
3440            True if the instance is stopped. A stopped instance is not usable.
3441         */
3442         @property bool isStopped();
3443 
3444         /// Stop and invalidate this instance.
3445         void shutdown();
3446 
3447         /** Set verbose.
3448             This will print request information to stderr.
3449         */
3450         @property void verbose(bool on);
3451 
3452         // Connection settings
3453 
3454         /// Set timeout for activity on connection.
3455         @property void dataTimeout(Duration d);
3456 
3457         /** Set maximum time an operation is allowed to take.
3458             This includes dns resolution, connecting, data transfer, etc.
3459           */
3460         @property void operationTimeout(Duration d);
3461 
3462         /// Set timeout for connecting.
3463         @property void connectTimeout(Duration d);
3464 
3465         // Network settings
3466 
3467         /** Proxy
3468          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
3469          */
3470         @property void proxy(const(char)[] host);
3471 
3472         /** Proxy port
3473          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
3474          */
3475         @property void proxyPort(ushort port);
3476 
3477         /// Type of proxy
3478         alias CurlProxy = etc.c.curl.CurlProxy;
3479 
3480         /** Proxy type
3481          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
3482          */
3483         @property void proxyType(CurlProxy type);
3484 
3485         /// DNS lookup timeout.
3486         @property void dnsTimeout(Duration d);
3487 
3488         /**
3489          * The network interface to use in form of the the IP of the interface.
3490          *
3491          * Example:
3492          * ----
3493          * theprotocol.netInterface = "192.168.1.32";
3494          * theprotocol.netInterface = [ 192, 168, 1, 32 ];
3495          * ----
3496          *
3497          * See: $(REF InternetAddress, std,socket)
3498          */
3499         @property void netInterface(const(char)[] i);
3500 
3501         /// ditto
3502         @property void netInterface(const(ubyte)[4] i);
3503 
3504         /// ditto
3505         @property void netInterface(InternetAddress i);
3506 
3507         /**
3508            Set the local outgoing port to use.
3509            Params:
3510            port = the first outgoing port number to try and use
3511         */
3512         @property void localPort(ushort port);
3513 
3514         /**
3515            Set the local outgoing port range to use.
3516            This can be used together with the localPort property.
3517            Params:
3518            range = if the first port is occupied then try this many
3519            port number forwards
3520         */
3521         @property void localPortRange(ushort range);
3522 
3523         /** Set the tcp no-delay socket option on or off.
3524             See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
3525         */
3526         @property void tcpNoDelay(bool on);
3527 
3528         // Authentication settings
3529 
3530         /**
3531            Set the user name, password and optionally domain for authentication
3532            purposes.
3533 
3534            Some protocols may need authentication in some cases. Use this
3535            function to provide credentials.
3536 
3537            Params:
3538            username = the username
3539            password = the password
3540            domain = used for NTLM authentication only and is set to the NTLM domain
3541            name
3542         */
3543         void setAuthentication(const(char)[] username, const(char)[] password,
3544                                const(char)[] domain = "");
3545 
3546         /**
3547            Set the user name and password for proxy authentication.
3548 
3549            Params:
3550            username = the username
3551            password = the password
3552         */
3553         void setProxyAuthentication(const(char)[] username, const(char)[] password);
3554 
3555         /**
3556          * The event handler that gets called when data is needed for sending. The
3557          * length of the `void[]` specifies the maximum number of bytes that can
3558          * be sent.
3559          *
3560          * Returns:
3561          * The callback returns the number of elements in the buffer that have been
3562          * filled and are ready to send.
3563          * The special value `.abortRequest` can be returned in order to abort the
3564          * current request.
3565          * The special value `.pauseRequest` can be returned in order to pause the
3566          * current request.
3567          *
3568          */
3569         @property void onSend(size_t delegate(void[]) callback);
3570 
3571         /**
3572          * The event handler that receives incoming data. Be sure to copy the
3573          * incoming ubyte[] since it is not guaranteed to be valid after the
3574          * callback returns.
3575          *
3576          * Returns:
3577          * The callback returns the incoming bytes read. If not the entire array is
3578          * the request will abort.
3579          * The special value .pauseRequest can be returned in order to pause the
3580          * current request.
3581          *
3582          */
3583         @property void onReceive(size_t delegate(ubyte[]) callback);
3584 
3585         /**
3586          * The event handler that gets called to inform of upload/download progress.
3587          *
3588          * Callback_parameters:
3589          * $(CALLBACK_PARAMS)
3590          *
3591          * Callback_returns:
3592          * Return 0 from the callback to signal success, return non-zero to
3593          * abort transfer.
3594          */
3595         @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
3596                                                size_t ulTotal, size_t ulNow) callback);
3597     }
3598 
3599     /** Clear all commands send to ftp server.
3600     */
3601     void clearCommands()
3602     {
3603         if (p.commands !is null)
3604             Curl.curl.slist_free_all(p.commands);
3605         p.commands = null;
3606         p.curl.clear(CurlOption.postquote);
3607     }
3608 
3609     /** Add a command to send to ftp server.
3610      *
3611      * There is no remove command functionality. Do a $(LREF clearCommands) and
3612      * set the needed commands instead.
3613      *
3614      * Example:
3615      * ---
3616      * import std.net.curl;
3617      * auto client = FTP();
3618      * client.addCommand("RNFR my_file.txt");
3619      * client.addCommand("RNTO my_renamed_file.txt");
3620      * upload("my_file.txt", "ftp.digitalmars.com", client);
3621      * ---
3622      */
3623     void addCommand(const(char)[] command)
3624     {
3625         import std.internal.cstring : tempCString;
3626         p.commands = Curl.curl.slist_append(p.commands,
3627                                             command.tempCString().buffPtr);
3628         p.curl.set(CurlOption.postquote, p.commands);
3629     }
3630 
3631     /// Connection encoding. Defaults to ISO-8859-1.
3632     @property void encoding(string name)
3633     {
3634         p.encoding = name;
3635     }
3636 
3637     /// ditto
3638     @property string encoding()
3639     {
3640         return p.encoding;
3641     }
3642 
3643     /**
3644        The content length in bytes of the ftp data.
3645     */
3646     @property void contentLength(ulong len)
3647     {
3648         import std.conv : to;
3649         p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len));
3650     }
3651 
3652     /**
3653      * Get various timings defined in $(REF CurlInfo, etc, c, curl).
3654      * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`.
3655      *
3656      * Params:
3657      *      timing = one of the timings defined in $(REF CurlInfo, etc, c, curl).
3658      *               The values are:
3659      *               `etc.c.curl.CurlInfo.namelookup_time`,
3660      *               `etc.c.curl.CurlInfo.connect_time`,
3661      *               `etc.c.curl.CurlInfo.pretransfer_time`,
3662      *               `etc.c.curl.CurlInfo.starttransfer_time`,
3663      *               `etc.c.curl.CurlInfo.redirect_time`,
3664      *               `etc.c.curl.CurlInfo.appconnect_time`,
3665      *               `etc.c.curl.CurlInfo.total_time`.
3666      *      val    = the actual value of the inquired timing.
3667      *
3668      * Returns:
3669      *      The return code of the operation. The value stored in val
3670      *      should be used only if the return value is `etc.c.curl.CurlInfo.ok`.
3671      *
3672      * Example:
3673      * ---
3674      * import std.net.curl;
3675      * import etc.c.curl : CurlError, CurlInfo;
3676      *
3677      * auto client = FTP();
3678      * client.addCommand("RNFR my_file.txt");
3679      * client.addCommand("RNTO my_renamed_file.txt");
3680      * upload("my_file.txt", "ftp.digitalmars.com", client);
3681      *
3682      * double val;
3683      * CurlCode code;
3684      *
3685      * code = client.getTiming(CurlInfo.namelookup_time, val);
3686      * assert(code == CurlError.ok);
3687      * ---
3688      */
3689     CurlCode getTiming(CurlInfo timing, ref double val)
3690     {
3691         return p.curl.getTiming(timing, val);
3692     }
3693 
3694     @system unittest
3695     {
3696         auto client = FTP();
3697 
3698         double val;
3699         CurlCode code;
3700 
3701         code = client.getTiming(CurlInfo.total_time, val);
3702         assert(code == CurlError.ok);
3703         code = client.getTiming(CurlInfo.namelookup_time, val);
3704         assert(code == CurlError.ok);
3705         code = client.getTiming(CurlInfo.connect_time, val);
3706         assert(code == CurlError.ok);
3707         code = client.getTiming(CurlInfo.pretransfer_time, val);
3708         assert(code == CurlError.ok);
3709         code = client.getTiming(CurlInfo.starttransfer_time, val);
3710         assert(code == CurlError.ok);
3711         code = client.getTiming(CurlInfo.redirect_time, val);
3712         assert(code == CurlError.ok);
3713         code = client.getTiming(CurlInfo.appconnect_time, val);
3714         assert(code == CurlError.ok);
3715     }
3716 }
3717 
3718 /**
3719   * Basic SMTP protocol support.
3720   *
3721   * Example:
3722   * ---
3723   * import std.net.curl;
3724   *
3725   * // Send an email with SMTPS
3726   * auto smtp = SMTP("smtps://smtp.gmail.com");
3727   * smtp.setAuthentication("from.addr@gmail.com", "password");
3728   * smtp.mailTo = ["<to.addr@gmail.com>"];
3729   * smtp.mailFrom = "<from.addr@gmail.com>";
3730   * smtp.message = "Example Message";
3731   * smtp.perform();
3732   * ---
3733   *
3734   * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821)
3735   */
3736 struct SMTP
3737 {
3738     mixin Protocol;
3739     import std.typecons : RefCounted;
3740     import etc.c.curl : CurlUseSSL, curl_slist;
3741 
3742     private struct Impl
3743     {
3744         ~this()
3745         {
3746             if (curl.handle !is null) // work around RefCounted/emplace bug
3747                 curl.shutdown();
3748         }
3749         Curl curl;
3750 
3751         @property void message(string msg)
3752         {
3753             import std.algorithm.comparison : min;
3754 
3755             auto _message = msg;
3756             /**
3757                 This delegate reads the message text and copies it.
3758             */
3759             curl.onSend = delegate size_t(void[] data)
3760             {
3761                 if (!msg.length) return 0;
3762                 size_t to_copy = min(data.length, _message.length);
3763                 data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy];
3764                 _message = _message[to_copy..$];
3765                 return to_copy;
3766             };
3767         }
3768     }
3769 
3770     private RefCounted!Impl p;
3771 
3772     /**
3773         Sets to the URL of the SMTP server.
3774     */
3775     static SMTP opCall(const(char)[] url)
3776     {
3777         SMTP smtp;
3778         smtp.initialize();
3779         smtp.url = url;
3780         return smtp;
3781     }
3782 
3783     ///
3784     static SMTP opCall()
3785     {
3786         SMTP smtp;
3787         smtp.initialize();
3788         return smtp;
3789     }
3790 
3791     /+ TODO: The other structs have this function.
3792     SMTP dup()
3793     {
3794         SMTP copy = SMTP();
3795         copy.initialize();
3796         copy.p.encoding = p.encoding;
3797         copy.p.curl = p.curl.dup();
3798         curl_slist* cur = p.commands;
3799         curl_slist* newlist = null;
3800         while (cur)
3801         {
3802             newlist = Curl.curl.slist_append(newlist, cur.data);
3803             cur = cur.next;
3804         }
3805         copy.p.commands = newlist;
3806         copy.p.curl.set(CurlOption.postquote, copy.p.commands);
3807         copy.dataTimeout = _defaultDataTimeout;
3808         return copy;
3809     }
3810     +/
3811 
3812     /**
3813         Performs the request as configured.
3814         Params:
3815         throwOnError = whether to throw an exception or return a CurlCode on error
3816     */
3817     CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
3818     {
3819         return p.curl.perform(throwOnError);
3820     }
3821 
3822     /// The URL to specify the location of the resource.
3823     @property void url(const(char)[] url)
3824     {
3825         import std.algorithm.searching : startsWith;
3826         import std.exception : enforce;
3827         import std.uni : toLower;
3828 
3829         auto lowered = url.toLower();
3830 
3831         if (lowered.startsWith("smtps://"))
3832         {
3833             p.curl.set(CurlOption.use_ssl, CurlUseSSL.all);
3834         }
3835         else
3836         {
3837             enforce!CurlException(lowered.startsWith("smtp://"),
3838                                     "The url must be for the smtp protocol.");
3839         }
3840         p.curl.set(CurlOption.url, url);
3841     }
3842 
3843     private void initialize()
3844     {
3845         p.curl.initialize();
3846         p.curl.set(CurlOption.upload, 1L);
3847         dataTimeout = _defaultDataTimeout;
3848         verifyPeer = true;
3849         verifyHost = true;
3850     }
3851 
3852     // This is a workaround for mixed in content not having its
3853     // docs mixed in.
3854     version (StdDdoc)
3855     {
3856         static import etc.c.curl;
3857 
3858         /// Value to return from `onSend`/`onReceive` delegates in order to
3859         /// pause a request
3860         alias requestPause = CurlReadFunc.pause;
3861 
3862         /// Value to return from onSend delegate in order to abort a request
3863         alias requestAbort = CurlReadFunc.abort;
3864 
3865         /**
3866            True if the instance is stopped. A stopped instance is not usable.
3867         */
3868         @property bool isStopped();
3869 
3870         /// Stop and invalidate this instance.
3871         void shutdown();
3872 
3873         /** Set verbose.
3874             This will print request information to stderr.
3875         */
3876         @property void verbose(bool on);
3877 
3878         // Connection settings
3879 
3880         /// Set timeout for activity on connection.
3881         @property void dataTimeout(Duration d);
3882 
3883         /** Set maximum time an operation is allowed to take.
3884             This includes dns resolution, connecting, data transfer, etc.
3885           */
3886         @property void operationTimeout(Duration d);
3887 
3888         /// Set timeout for connecting.
3889         @property void connectTimeout(Duration d);
3890 
3891         // Network settings
3892 
3893         /** Proxy
3894          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
3895          */
3896         @property void proxy(const(char)[] host);
3897 
3898         /** Proxy port
3899          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
3900          */
3901         @property void proxyPort(ushort port);
3902 
3903         /// Type of proxy
3904         alias CurlProxy = etc.c.curl.CurlProxy;
3905 
3906         /** Proxy type
3907          *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
3908          */
3909         @property void proxyType(CurlProxy type);
3910 
3911         /// DNS lookup timeout.
3912         @property void dnsTimeout(Duration d);
3913 
3914         /**
3915          * The network interface to use in form of the the IP of the interface.
3916          *
3917          * Example:
3918          * ----
3919          * theprotocol.netInterface = "192.168.1.32";
3920          * theprotocol.netInterface = [ 192, 168, 1, 32 ];
3921          * ----
3922          *
3923          * See: $(REF InternetAddress, std,socket)
3924          */
3925         @property void netInterface(const(char)[] i);
3926 
3927         /// ditto
3928         @property void netInterface(const(ubyte)[4] i);
3929 
3930         /// ditto
3931         @property void netInterface(InternetAddress i);
3932 
3933         /**
3934            Set the local outgoing port to use.
3935            Params:
3936            port = the first outgoing port number to try and use
3937         */
3938         @property void localPort(ushort port);
3939 
3940         /**
3941            Set the local outgoing port range to use.
3942            This can be used together with the localPort property.
3943            Params:
3944            range = if the first port is occupied then try this many
3945            port number forwards
3946         */
3947         @property void localPortRange(ushort range);
3948 
3949         /** Set the tcp no-delay socket option on or off.
3950             See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
3951         */
3952         @property void tcpNoDelay(bool on);
3953 
3954         // Authentication settings
3955 
3956         /**
3957            Set the user name, password and optionally domain for authentication
3958            purposes.
3959 
3960            Some protocols may need authentication in some cases. Use this
3961            function to provide credentials.
3962 
3963            Params:
3964            username = the username
3965            password = the password
3966            domain = used for NTLM authentication only and is set to the NTLM domain
3967            name
3968         */
3969         void setAuthentication(const(char)[] username, const(char)[] password,
3970                                const(char)[] domain = "");
3971 
3972         /**
3973            Set the user name and password for proxy authentication.
3974 
3975            Params:
3976            username = the username
3977            password = the password
3978         */
3979         void setProxyAuthentication(const(char)[] username, const(char)[] password);
3980 
3981         /**
3982          * The event handler that gets called when data is needed for sending. The
3983          * length of the `void[]` specifies the maximum number of bytes that can
3984          * be sent.
3985          *
3986          * Returns:
3987          * The callback returns the number of elements in the buffer that have been
3988          * filled and are ready to send.
3989          * The special value `.abortRequest` can be returned in order to abort the
3990          * current request.
3991          * The special value `.pauseRequest` can be returned in order to pause the
3992          * current request.
3993          */
3994         @property void onSend(size_t delegate(void[]) callback);
3995 
3996         /**
3997          * The event handler that receives incoming data. Be sure to copy the
3998          * incoming ubyte[] since it is not guaranteed to be valid after the
3999          * callback returns.
4000          *
4001          * Returns:
4002          * The callback returns the incoming bytes read. If not the entire array is
4003          * the request will abort.
4004          * The special value .pauseRequest can be returned in order to pause the
4005          * current request.
4006          */
4007         @property void onReceive(size_t delegate(ubyte[]) callback);
4008 
4009         /**
4010          * The event handler that gets called to inform of upload/download progress.
4011          *
4012          * Callback_parameters:
4013          * $(CALLBACK_PARAMS)
4014          *
4015          * Callback_returns:
4016          * Return 0 from the callback to signal success, return non-zero to
4017          * abort transfer.
4018          */
4019         @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
4020                                                size_t ulTotal, size_t ulNow) callback);
4021     }
4022 
4023     /**
4024         Setter for the sender's email address.
4025     */
4026     @property void mailFrom()(const(char)[] sender)
4027     {
4028         assert(!sender.empty, "Sender must not be empty");
4029         p.curl.set(CurlOption.mail_from, sender);
4030     }
4031 
4032     /**
4033         Setter for the recipient email addresses.
4034     */
4035     void mailTo()(const(char)[][] recipients...)
4036     {
4037         import std.internal.cstring : tempCString;
4038         assert(!recipients.empty, "Recipient must not be empty");
4039         curl_slist* recipients_list = null;
4040         foreach (recipient; recipients)
4041         {
4042             recipients_list =
4043                 Curl.curl.slist_append(recipients_list,
4044                                   recipient.tempCString().buffPtr);
4045         }
4046         p.curl.set(CurlOption.mail_rcpt, recipients_list);
4047     }
4048 
4049     /**
4050         Sets the message body text.
4051     */
4052 
4053     @property void message(string msg)
4054     {
4055         p.message = msg;
4056     }
4057 }
4058 
4059 @system unittest
4060 {
4061     import std.net.curl;
4062 
4063     // Send an email with SMTPS
4064     auto smtp = SMTP("smtps://smtp.gmail.com");
4065     smtp.setAuthentication("from.addr@gmail.com", "password");
4066     smtp.mailTo = ["<to.addr@gmail.com>"];
4067     smtp.mailFrom = "<from.addr@gmail.com>";
4068     smtp.message = "Example Message";
4069     //smtp.perform();
4070 }
4071 
4072 
4073 /++
4074     Exception thrown on errors in std.net.curl functions.
4075 +/
4076 class CurlException : Exception
4077 {
4078     /++
4079         Params:
4080             msg  = The message for the exception.
4081             file = The file where the exception occurred.
4082             line = The line number where the exception occurred.
4083             next = The previous exception in the chain of exceptions, if any.
4084       +/
4085     @safe pure nothrow
4086     this(string msg,
4087          string file = __FILE__,
4088          size_t line = __LINE__,
4089          Throwable next = null)
4090     {
4091         super(msg, file, line, next);
4092     }
4093 }
4094 
4095 /++
4096     Exception thrown on timeout errors in std.net.curl functions.
4097 +/
4098 class CurlTimeoutException : CurlException
4099 {
4100     /++
4101         Params:
4102             msg  = The message for the exception.
4103             file = The file where the exception occurred.
4104             line = The line number where the exception occurred.
4105             next = The previous exception in the chain of exceptions, if any.
4106       +/
4107     @safe pure nothrow
4108     this(string msg,
4109          string file = __FILE__,
4110          size_t line = __LINE__,
4111          Throwable next = null)
4112     {
4113         super(msg, file, line, next);
4114     }
4115 }
4116 
4117 /++
4118     Exception thrown on HTTP request failures, e.g. 404 Not Found.
4119 +/
4120 class HTTPStatusException : CurlException
4121 {
4122     /++
4123         Params:
4124             status = The HTTP status code.
4125             msg  = The message for the exception.
4126             file = The file where the exception occurred.
4127             line = The line number where the exception occurred.
4128             next = The previous exception in the chain of exceptions, if any.
4129       +/
4130     @safe pure nothrow
4131     this(int status,
4132          string msg,
4133          string file = __FILE__,
4134          size_t line = __LINE__,
4135          Throwable next = null)
4136     {
4137         super(msg, file, line, next);
4138         this.status = status;
4139     }
4140 
4141     immutable int status; /// The HTTP status code
4142 }
4143 
4144 /// Equal to $(REF CURLcode, etc,c,curl)
4145 alias CurlCode = CURLcode;
4146 
4147 /// Flag to specify whether or not an exception is thrown on error.
4148 alias ThrowOnError = Flag!"throwOnError";
4149 
4150 private struct CurlAPI
4151 {
4152     import etc.c.curl : CurlGlobal;
4153     static struct API
4154     {
4155     import etc.c.curl : curl_version_info, curl_version_info_data,
4156                         CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist;
4157     extern(C):
4158         import core.stdc.config : c_long;
4159         CURLcode function(c_long flags) global_init;
4160         void function() global_cleanup;
4161         curl_version_info_data * function(CURLversion) version_info;
4162         CURL* function() easy_init;
4163         CURLcode function(CURL *curl, CURLoption option,...) easy_setopt;
4164         CURLcode function(CURL *curl) easy_perform;
4165         CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo;
4166         CURL* function(CURL *curl) easy_duphandle;
4167         char* function(CURLcode) easy_strerror;
4168         CURLcode function(CURL *handle, int bitmask) easy_pause;
4169         void function(CURL *curl) easy_cleanup;
4170         curl_slist* function(curl_slist *, char *) slist_append;
4171         void function(curl_slist *) slist_free_all;
4172     }
4173     __gshared API _api;
4174     __gshared void* _handle;
4175 
4176     static ref API instance() @property
4177     {
4178         import std.concurrency : initOnce;
4179         initOnce!_handle(loadAPI());
4180         return _api;
4181     }
4182 
4183     static void* loadAPI()
4184     {
4185         import std.exception : enforce;
4186 
4187         version (Posix)
4188         {
4189             import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY;
4190             alias loadSym = dlsym;
4191         }
4192         else version (Windows)
4193         {
4194             import core.sys.windows.winbase : GetProcAddress, GetModuleHandleA,
4195                 LoadLibraryA;
4196             alias loadSym = GetProcAddress;
4197         }
4198         else
4199             static assert(0, "unimplemented");
4200 
4201         void* handle;
4202         version (Posix)
4203             handle = dlopen(null, RTLD_LAZY);
4204         else version (Windows)
4205             handle = GetModuleHandleA(null);
4206         assert(handle !is null);
4207 
4208         // try to load curl from the executable to allow static linking
4209         if (loadSym(handle, "curl_global_init") is null)
4210         {
4211             import std.format : format;
4212             version (Posix)
4213                 dlclose(handle);
4214 
4215             version (LibcurlPath)
4216             {
4217                 import std..string : strip;
4218                 static immutable names = [strip(import("LibcurlPathFile"))];
4219             }
4220             else version (OSX)
4221                 static immutable names = ["libcurl.4.dylib"];
4222             else version (Posix)
4223             {
4224                 static immutable names = ["libcurl.so", "libcurl.so.4",
4225                 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"];
4226             }
4227             else version (Windows)
4228                 static immutable names = ["libcurl.dll", "curl.dll"];
4229 
4230             foreach (name; names)
4231             {
4232                 version (Posix)
4233                     handle = dlopen(name.ptr, RTLD_LAZY);
4234                 else version (Windows)
4235                     handle = LoadLibraryA(name.ptr);
4236                 if (handle !is null) break;
4237             }
4238 
4239             enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names));
4240         }
4241 
4242         foreach (i, FP; typeof(API.tupleof))
4243         {
4244             enum name = __traits(identifier, _api.tupleof[i]);
4245             auto p = enforce!CurlException(loadSym(handle, "curl_"~name),
4246                                            "Couldn't load curl_"~name~" from libcurl.");
4247             _api.tupleof[i] = cast(FP) p;
4248         }
4249 
4250         enforce!CurlException(!_api.global_init(CurlGlobal.all),
4251                               "Failed to initialize libcurl");
4252 
4253         static extern(C) void cleanup()
4254         {
4255             if (_handle is null) return;
4256             _api.global_cleanup();
4257             version (Posix)
4258             {
4259                 import core.sys.posix.dlfcn : dlclose;
4260                 dlclose(_handle);
4261             }
4262             else version (Windows)
4263             {
4264                 import core.sys.windows.winbase : FreeLibrary;
4265                 FreeLibrary(_handle);
4266             }
4267             else
4268                 static assert(0, "unimplemented");
4269             _api = API.init;
4270             _handle = null;
4271         }
4272 
4273         import core.stdc.stdlib : atexit;
4274         atexit(&cleanup);
4275 
4276         return handle;
4277     }
4278 }
4279 
4280 /**
4281   Wrapper to provide a better interface to libcurl than using the plain C API.
4282   It is recommended to use the `HTTP`/`FTP` etc. structs instead unless
4283   raw access to libcurl is needed.
4284 
4285   Warning: This struct uses interior pointers for callbacks. Only allocate it
4286   on the stack if you never move or copy it. This also means passing by reference
4287   when passing Curl to other functions. Otherwise always allocate on
4288   the heap.
4289 */
4290 struct Curl
4291 {
4292     import etc.c.curl : CURL, CurlError, CurlPause, CurlSeek, CurlSeekPos,
4293                         curl_socket_t, CurlSockType,
4294                         CurlReadFunc, CurlInfo, curlsocktype, curl_off_t,
4295                         LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR, LIBCURL_VERSION_PATCH;
4296 
4297     alias OutData = void[];
4298     alias InData = ubyte[];
4299     private bool _stopped;
4300 
4301     private static auto ref curl() @property { return CurlAPI.instance; }
4302 
4303     // A handle should not be used by two threads simultaneously
4304     private CURL* handle;
4305 
4306     // May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE`
4307     private size_t delegate(OutData) _onSend;
4308     private size_t delegate(InData) _onReceive;
4309     private void delegate(in char[]) _onReceiveHeader;
4310     private CurlSeek delegate(long,CurlSeekPos) _onSeek;
4311     private int delegate(curl_socket_t,CurlSockType) _onSocketOption;
4312     private int delegate(size_t dltotal, size_t dlnow,
4313                          size_t ultotal, size_t ulnow) _onProgress;
4314 
4315     alias requestPause = CurlReadFunc.pause;
4316     alias requestAbort = CurlReadFunc.abort;
4317 
4318     /**
4319        Initialize the instance by creating a working curl handle.
4320     */
4321     void initialize()
4322     {
4323         import std.exception : enforce;
4324         enforce!CurlException(!handle, "Curl instance already initialized");
4325         handle = curl.easy_init();
4326         enforce!CurlException(handle, "Curl instance couldn't be initialized");
4327         _stopped = false;
4328         set(CurlOption.nosignal, 1);
4329     }
4330 
4331     ///
4332     @property bool stopped() const
4333     {
4334         return _stopped;
4335     }
4336 
4337     /**
4338        Duplicate this handle.
4339 
4340        The new handle will have all options set as the one it was duplicated
4341        from. An exception to this is that all options that cannot be shared
4342        across threads are reset thereby making it safe to use the duplicate
4343        in a new thread.
4344     */
4345     Curl dup()
4346     {
4347         import std.meta : AliasSeq;
4348         Curl copy;
4349         copy.handle = curl.easy_duphandle(handle);
4350         copy._stopped = false;
4351 
4352         with (CurlOption) {
4353             auto tt = AliasSeq!(file, writefunction, writeheader,
4354                 headerfunction, infile, readfunction, ioctldata, ioctlfunction,
4355                 seekdata, seekfunction, sockoptdata, sockoptfunction,
4356                 opensocketdata, opensocketfunction, progressdata,
4357                 progressfunction, debugdata, debugfunction, interleavedata,
4358                 interleavefunction, chunk_data, chunk_bgn_function,
4359                 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields);
4360 
4361             foreach (option; tt)
4362                 copy.clear(option);
4363         }
4364 
4365         // The options are only supported by libcurl when it has been built
4366         // against certain versions of OpenSSL - if your libcurl uses an old
4367         // OpenSSL, or uses an entirely different SSL engine, attempting to
4368         // clear these normally will raise an exception
4369         copy.clearIfSupported(CurlOption.ssl_ctx_function);
4370         copy.clearIfSupported(CurlOption.ssh_keydata);
4371 
4372         // Enable for curl version > 7.21.7
4373         static if (LIBCURL_VERSION_MAJOR >= 7 &&
4374                    LIBCURL_VERSION_MINOR >= 21 &&
4375                    LIBCURL_VERSION_PATCH >= 7)
4376         {
4377             copy.clear(CurlOption.closesocketdata);
4378             copy.clear(CurlOption.closesocketfunction);
4379         }
4380 
4381         copy.set(CurlOption.nosignal, 1);
4382 
4383         // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared
4384         // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared
4385 
4386         /*
4387           Allow sharing of conv functions
4388           copy.clear(CurlOption.conv_to_network_function);
4389           copy.clear(CurlOption.conv_from_network_function);
4390           copy.clear(CurlOption.conv_from_utf8_function);
4391         */
4392 
4393         return copy;
4394     }
4395 
4396     private void _check(CurlCode code)
4397     {
4398         import std.exception : enforce;
4399         enforce!CurlTimeoutException(code != CurlError.operation_timedout,
4400                                        errorString(code));
4401 
4402         enforce!CurlException(code == CurlError.ok,
4403                                 errorString(code));
4404     }
4405 
4406     private string errorString(CurlCode code)
4407     {
4408         import core.stdc..string : strlen;
4409         import std.format : format;
4410 
4411         auto msgZ = curl.easy_strerror(code);
4412         // doing the following (instead of just using std.conv.to!string) avoids 1 allocation
4413         return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle);
4414     }
4415 
4416     private void throwOnStopped(string message = null)
4417     {
4418         import std.exception : enforce;
4419         auto def = "Curl instance called after being cleaned up";
4420         enforce!CurlException(!stopped,
4421                                 message == null ? def : message);
4422     }
4423 
4424     /**
4425         Stop and invalidate this curl instance.
4426         Warning: Do not call this from inside a callback handler e.g. `onReceive`.
4427     */
4428     void shutdown()
4429     {
4430         throwOnStopped();
4431         _stopped = true;
4432         curl.easy_cleanup(this.handle);
4433         this.handle = null;
4434     }
4435 
4436     /**
4437        Pausing and continuing transfers.
4438     */
4439     void pause(bool sendingPaused, bool receivingPaused)
4440     {
4441         throwOnStopped();
4442         _check(curl.easy_pause(this.handle,
4443                                (sendingPaused ? CurlPause.send_cont : CurlPause.send) |
4444                                (receivingPaused ? CurlPause.recv_cont : CurlPause.recv)));
4445     }
4446 
4447     /**
4448        Set a string curl option.
4449        Params:
4450        option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4451        value = The string
4452     */
4453     void set(CurlOption option, const(char)[] value)
4454     {
4455         import std.internal.cstring : tempCString;
4456         throwOnStopped();
4457         _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr));
4458     }
4459 
4460     /**
4461        Set a long curl option.
4462        Params:
4463        option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4464        value = The long
4465     */
4466     void set(CurlOption option, long value)
4467     {
4468         throwOnStopped();
4469         _check(curl.easy_setopt(this.handle, option, value));
4470     }
4471 
4472     /**
4473        Set a void* curl option.
4474        Params:
4475        option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4476        value = The pointer
4477     */
4478     void set(CurlOption option, void* value)
4479     {
4480         throwOnStopped();
4481         _check(curl.easy_setopt(this.handle, option, value));
4482     }
4483 
4484     /**
4485        Clear a pointer option.
4486        Params:
4487        option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4488     */
4489     void clear(CurlOption option)
4490     {
4491         throwOnStopped();
4492         _check(curl.easy_setopt(this.handle, option, null));
4493     }
4494 
4495     /**
4496        Clear a pointer option. Does not raise an exception if the underlying
4497        libcurl does not support the option. Use sparingly.
4498        Params:
4499        option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
4500     */
4501     void clearIfSupported(CurlOption option)
4502     {
4503         throwOnStopped();
4504         auto rval = curl.easy_setopt(this.handle, option, null);
4505         if (rval != CurlError.unknown_option && rval != CurlError.not_built_in)
4506             _check(rval);
4507     }
4508 
4509     /**
4510        perform the curl request by doing the HTTP,FTP etc. as it has
4511        been setup beforehand.
4512 
4513        Params:
4514        throwOnError = whether to throw an exception or return a CurlCode on error
4515     */
4516     CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
4517     {
4518         throwOnStopped();
4519         CurlCode code = curl.easy_perform(this.handle);
4520         if (throwOnError)
4521             _check(code);
4522         return code;
4523     }
4524 
4525     /**
4526        Get the various timings like name lookup time, total time, connect time etc.
4527        The timed category is passed through the timing parameter while the timing
4528        value is stored at val. The value is usable only if res is equal to
4529        `etc.c.curl.CurlError.ok`.
4530     */
4531     CurlCode getTiming(CurlInfo timing, ref double val)
4532     {
4533         CurlCode code;
4534         code = curl.easy_getinfo(handle, timing, &val);
4535         return code;
4536     }
4537 
4538     /**
4539       * The event handler that receives incoming data.
4540       *
4541       * Params:
4542       * callback = the callback that receives the `ubyte[]` data.
4543       * Be sure to copy the incoming data and not store
4544       * a slice.
4545       *
4546       * Returns:
4547       * The callback returns the incoming bytes read. If not the entire array is
4548       * the request will abort.
4549       * The special value HTTP.pauseRequest can be returned in order to pause the
4550       * current request.
4551       *
4552       * Example:
4553       * ----
4554       * import std.net.curl, std.stdio;
4555       * Curl curl;
4556       * curl.initialize();
4557       * curl.set(CurlOption.url, "http://dlang.org");
4558       * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;};
4559       * curl.perform();
4560       * ----
4561       */
4562     @property void onReceive(size_t delegate(InData) callback)
4563     {
4564         _onReceive = (InData id)
4565         {
4566             throwOnStopped("Receive callback called on cleaned up Curl instance");
4567             return callback(id);
4568         };
4569         set(CurlOption.file, cast(void*) &this);
4570         set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback);
4571     }
4572 
4573     /**
4574       * The event handler that receives incoming headers for protocols
4575       * that uses headers.
4576       *
4577       * Params:
4578       * callback = the callback that receives the header string.
4579       * Make sure the callback copies the incoming params if
4580       * it needs to store it because they are references into
4581       * the backend and may very likely change.
4582       *
4583       * Example:
4584       * ----
4585       * import std.net.curl, std.stdio;
4586       * Curl curl;
4587       * curl.initialize();
4588       * curl.set(CurlOption.url, "http://dlang.org");
4589       * curl.onReceiveHeader = (in char[] header) { writeln(header); };
4590       * curl.perform();
4591       * ----
4592       */
4593     @property void onReceiveHeader(void delegate(in char[]) callback)
4594     {
4595         _onReceiveHeader = (in char[] od)
4596         {
4597             throwOnStopped("Receive header callback called on "~
4598                            "cleaned up Curl instance");
4599             callback(od);
4600         };
4601         set(CurlOption.writeheader, cast(void*) &this);
4602         set(CurlOption.headerfunction,
4603             cast(void*) &Curl._receiveHeaderCallback);
4604     }
4605 
4606     /**
4607       * The event handler that gets called when data is needed for sending.
4608       *
4609       * Params:
4610       * callback = the callback that has a `void[]` buffer to be filled
4611       *
4612       * Returns:
4613       * The callback returns the number of elements in the buffer that have been
4614       * filled and are ready to send.
4615       * The special value `Curl.abortRequest` can be returned in
4616       * order to abort the current request.
4617       * The special value `Curl.pauseRequest` can be returned in order to
4618       * pause the current request.
4619       *
4620       * Example:
4621       * ----
4622       * import std.net.curl;
4623       * Curl curl;
4624       * curl.initialize();
4625       * curl.set(CurlOption.url, "http://dlang.org");
4626       *
4627       * string msg = "Hello world";
4628       * curl.onSend = (void[] data)
4629       * {
4630       *     auto m = cast(void[]) msg;
4631       *     size_t length = m.length > data.length ? data.length : m.length;
4632       *     if (length == 0) return 0;
4633       *     data[0 .. length] = m[0 .. length];
4634       *     msg = msg[length..$];
4635       *     return length;
4636       * };
4637       * curl.perform();
4638       * ----
4639       */
4640     @property void onSend(size_t delegate(OutData) callback)
4641     {
4642         _onSend = (OutData od)
4643         {
4644             throwOnStopped("Send callback called on cleaned up Curl instance");
4645             return callback(od);
4646         };
4647         set(CurlOption.infile, cast(void*) &this);
4648         set(CurlOption.readfunction, cast(void*) &Curl._sendCallback);
4649     }
4650 
4651     /**
4652       * The event handler that gets called when the curl backend needs to seek
4653       * the data to be sent.
4654       *
4655       * Params:
4656       * callback = the callback that receives a seek offset and a seek position
4657       *            $(REF CurlSeekPos, etc,c,curl)
4658       *
4659       * Returns:
4660       * The callback returns the success state of the seeking
4661       * $(REF CurlSeek, etc,c,curl)
4662       *
4663       * Example:
4664       * ----
4665       * import std.net.curl;
4666       * Curl curl;
4667       * curl.initialize();
4668       * curl.set(CurlOption.url, "http://dlang.org");
4669       * curl.onSeek = (long p, CurlSeekPos sp)
4670       * {
4671       *     return CurlSeek.cantseek;
4672       * };
4673       * curl.perform();
4674       * ----
4675       */
4676     @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback)
4677     {
4678         _onSeek = (long ofs, CurlSeekPos sp)
4679         {
4680             throwOnStopped("Seek callback called on cleaned up Curl instance");
4681             return callback(ofs, sp);
4682         };
4683         set(CurlOption.seekdata, cast(void*) &this);
4684         set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback);
4685     }
4686 
4687     /**
4688       * The event handler that gets called when the net socket has been created
4689       * but a `connect()` call has not yet been done. This makes it possible to set
4690       * misc. socket options.
4691       *
4692       * Params:
4693       * callback = the callback that receives the socket and socket type
4694       * $(REF CurlSockType, etc,c,curl)
4695       *
4696       * Returns:
4697       * Return 0 from the callback to signal success, return 1 to signal error
4698       * and make curl close the socket
4699       *
4700       * Example:
4701       * ----
4702       * import std.net.curl;
4703       * Curl curl;
4704       * curl.initialize();
4705       * curl.set(CurlOption.url, "http://dlang.org");
4706       * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ };
4707       * curl.perform();
4708       * ----
4709       */
4710     @property void onSocketOption(int delegate(curl_socket_t,
4711                                                CurlSockType) callback)
4712     {
4713         _onSocketOption = (curl_socket_t sock, CurlSockType st)
4714         {
4715             throwOnStopped("Socket option callback called on "~
4716                            "cleaned up Curl instance");
4717             return callback(sock, st);
4718         };
4719         set(CurlOption.sockoptdata, cast(void*) &this);
4720         set(CurlOption.sockoptfunction,
4721             cast(void*) &Curl._socketOptionCallback);
4722     }
4723 
4724     /**
4725       * The event handler that gets called to inform of upload/download progress.
4726       *
4727       * Params:
4728       * callback = the callback that receives the (total bytes to download,
4729       * currently downloaded bytes, total bytes to upload, currently uploaded
4730       * bytes).
4731       *
4732       * Returns:
4733       * Return 0 from the callback to signal success, return non-zero to abort
4734       * transfer
4735       *
4736       * Example:
4737       * ----
4738       * import std.net.curl, std.stdio;
4739       * Curl curl;
4740       * curl.initialize();
4741       * curl.set(CurlOption.url, "http://dlang.org");
4742       * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
4743       * {
4744       *     writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal);
4745       *     writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal);
4746       *     return 0;
4747       * };
4748       * curl.perform();
4749       * ----
4750       */
4751     @property void onProgress(int delegate(size_t dlTotal,
4752                                            size_t dlNow,
4753                                            size_t ulTotal,
4754                                            size_t ulNow) callback)
4755     {
4756         _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln)
4757         {
4758             throwOnStopped("Progress callback called on cleaned "~
4759                            "up Curl instance");
4760             return callback(dlt, dln, ult, uln);
4761         };
4762         set(CurlOption.noprogress, 0);
4763         set(CurlOption.progressdata, cast(void*) &this);
4764         set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback);
4765     }
4766 
4767     // Internal C callbacks to register with libcurl
4768     extern (C) private static
4769     size_t _receiveCallback(const char* str,
4770                             size_t size, size_t nmemb, void* ptr)
4771     {
4772         auto b = cast(Curl*) ptr;
4773         if (b._onReceive != null)
4774             return b._onReceive(cast(InData)(str[0 .. size*nmemb]));
4775         return size*nmemb;
4776     }
4777 
4778     extern (C) private static
4779     size_t _receiveHeaderCallback(const char* str,
4780                                   size_t size, size_t nmemb, void* ptr)
4781     {
4782         import std..string : chomp;
4783 
4784         auto b = cast(Curl*) ptr;
4785         auto s = str[0 .. size*nmemb].chomp();
4786         if (b._onReceiveHeader != null)
4787             b._onReceiveHeader(s);
4788 
4789         return size*nmemb;
4790     }
4791 
4792     extern (C) private static
4793     size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr)
4794     {
4795         Curl* b = cast(Curl*) ptr;
4796         auto a = cast(void[]) str[0 .. size*nmemb];
4797         if (b._onSend == null)
4798             return 0;
4799         return b._onSend(a);
4800     }
4801 
4802     extern (C) private static
4803     int _seekCallback(void *ptr, curl_off_t offset, int origin)
4804     {
4805         auto b = cast(Curl*) ptr;
4806         if (b._onSeek == null)
4807             return CurlSeek.cantseek;
4808 
4809         // origin: CurlSeekPos.set/current/end
4810         // return: CurlSeek.ok/fail/cantseek
4811         return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin);
4812     }
4813 
4814     extern (C) private static
4815     int _socketOptionCallback(void *ptr,
4816                               curl_socket_t curlfd, curlsocktype purpose)
4817     {
4818         auto b = cast(Curl*) ptr;
4819         if (b._onSocketOption == null)
4820             return 0;
4821 
4822         // return: 0 ok, 1 fail
4823         return b._onSocketOption(curlfd, cast(CurlSockType) purpose);
4824     }
4825 
4826     extern (C) private static
4827     int _progressCallback(void *ptr,
4828                           double dltotal, double dlnow,
4829                           double ultotal, double ulnow)
4830     {
4831         auto b = cast(Curl*) ptr;
4832         if (b._onProgress == null)
4833             return 0;
4834 
4835         // return: 0 ok, 1 fail
4836         return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow,
4837                              cast(size_t) ultotal, cast(size_t) ulnow);
4838     }
4839 
4840 }
4841 
4842 // Internal messages send between threads.
4843 // The data is wrapped in this struct in order to ensure that
4844 // other std.concurrency.receive calls does not pick up our messages
4845 // by accident.
4846 private struct CurlMessage(T)
4847 {
4848     public T data;
4849 }
4850 
4851 private static CurlMessage!T curlMessage(T)(T data)
4852 {
4853     return CurlMessage!T(data);
4854 }
4855 
4856 // Pool of to be used for reusing buffers
4857 private struct Pool(Data)
4858 {
4859     private struct Entry
4860     {
4861         Data data;
4862         Entry* next;
4863     }
4864     private Entry*  root;
4865     private Entry* freeList;
4866 
4867     @safe @property bool empty()
4868     {
4869         return root == null;
4870     }
4871 
4872     @safe nothrow void push(Data d)
4873     {
4874         if (freeList == null)
4875         {
4876             // Allocate new Entry since there is no one
4877             // available in the freeList
4878             freeList = new Entry;
4879         }
4880         freeList.data = d;
4881         Entry* oldroot = root;
4882         root = freeList;
4883         freeList = freeList.next;
4884         root.next = oldroot;
4885     }
4886 
4887     @safe Data pop()
4888     {
4889         import std.exception : enforce;
4890         enforce!Exception(root != null, "pop() called on empty pool");
4891         auto d = root.data;
4892         auto n = root.next;
4893         root.next = freeList;
4894         freeList = root;
4895         root = n;
4896         return d;
4897     }
4898 }
4899 
4900 // Lazily-instantiated namespace to avoid importing std.concurrency until needed.
4901 private struct _async()
4902 {
4903 static:
4904     // https://issues.dlang.org/show_bug.cgi?id=15831
4905     // this should be inside byLineAsync
4906     // Range that reads one chunk at a time asynchronously.
4907     private struct ChunkInputRange
4908     {
4909         import std.concurrency : Tid, send;
4910 
4911         private ubyte[] chunk;
4912         mixin WorkerThreadProtocol!(ubyte, chunk);
4913 
4914         private Tid workerTid;
4915         private State running;
4916 
4917         private this(Tid tid, size_t transmitBuffers, size_t chunkSize)
4918         {
4919             workerTid = tid;
4920             state = State.needUnits;
4921 
4922             // Send buffers to other thread for it to use.  Since no mechanism is in
4923             // place for moving ownership a cast to shared is done here and a cast
4924             // back to non-shared in the receiving end.
4925             foreach (i ; 0 .. transmitBuffers)
4926             {
4927                 ubyte[] arr = new ubyte[](chunkSize);
4928                 workerTid.send(cast(immutable(ubyte[]))arr);
4929             }
4930         }
4931     }
4932 
4933     // https://issues.dlang.org/show_bug.cgi?id=15831
4934     // this should be inside byLineAsync
4935     // Range that reads one line at a time asynchronously.
4936     private static struct LineInputRange(Char)
4937     {
4938         private Char[] line;
4939         mixin WorkerThreadProtocol!(Char, line);
4940 
4941         private Tid workerTid;
4942         private State running;
4943 
4944         private this(Tid tid, size_t transmitBuffers, size_t bufferSize)
4945         {
4946             import std.concurrency : send;
4947 
4948             workerTid = tid;
4949             state = State.needUnits;
4950 
4951             // Send buffers to other thread for it to use.  Since no mechanism is in
4952             // place for moving ownership a cast to shared is done here and casted
4953             // back to non-shared in the receiving end.
4954             foreach (i ; 0 .. transmitBuffers)
4955             {
4956                 auto arr = new Char[](bufferSize);
4957                 workerTid.send(cast(immutable(Char[]))arr);
4958             }
4959         }
4960     }
4961 
4962     import std.concurrency : Tid;
4963 
4964     // Shared function for reading incoming chunks of data and
4965     // sending the to a parent thread
4966     private size_t receiveChunks(ubyte[] data, ref ubyte[] outdata,
4967                                  Pool!(ubyte[]) freeBuffers,
4968                                  ref ubyte[] buffer, Tid fromTid,
4969                                  ref bool aborted)
4970     {
4971         import std.concurrency : receive, send, thisTid;
4972 
4973         immutable datalen = data.length;
4974 
4975         // Copy data to fill active buffer
4976         while (!data.empty)
4977         {
4978 
4979             // Make sure a buffer is present
4980             while ( outdata.empty && freeBuffers.empty)
4981             {
4982                 // Active buffer is invalid and there are no
4983                 // available buffers in the pool. Wait for buffers
4984                 // to return from main thread in order to reuse
4985                 // them.
4986                 receive((immutable(ubyte)[] buf)
4987                         {
4988                             buffer = cast(ubyte[]) buf;
4989                             outdata = buffer[];
4990                         },
4991                         (bool flag) { aborted = true; }
4992                         );
4993                 if (aborted) return cast(size_t) 0;
4994             }
4995             if (outdata.empty)
4996             {
4997                 buffer = freeBuffers.pop();
4998                 outdata = buffer[];
4999             }
5000 
5001             // Copy data
5002             auto copyBytes = outdata.length < data.length ?
5003                 outdata.length : data.length;
5004 
5005             outdata[0 .. copyBytes] = data[0 .. copyBytes];
5006             outdata = outdata[copyBytes..$];
5007             data = data[copyBytes..$];
5008 
5009             if (outdata.empty)
5010                 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer));
5011         }
5012 
5013         return datalen;
5014     }
5015 
5016     // ditto
5017     private void finalizeChunks(ubyte[] outdata, ref ubyte[] buffer,
5018                                 Tid fromTid)
5019     {
5020         import std.concurrency : send, thisTid;
5021         if (!outdata.empty)
5022         {
5023             // Resize the last buffer
5024             buffer.length = buffer.length - outdata.length;
5025             fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer));
5026         }
5027     }
5028 
5029 
5030     // Shared function for reading incoming lines of data and sending the to a
5031     // parent thread
5032     private static size_t receiveLines(Terminator, Unit)
5033         (const(ubyte)[] data, ref EncodingScheme encodingScheme,
5034          bool keepTerminator, Terminator terminator,
5035          ref const(ubyte)[] leftOverBytes, ref bool bufferValid,
5036          ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer,
5037          Tid fromTid, ref bool aborted)
5038     {
5039         import std.concurrency : prioritySend, receive, send, thisTid;
5040         import std.exception : enforce;
5041         import std.format : format;
5042         import std.traits : isArray;
5043 
5044         immutable datalen = data.length;
5045 
5046         // Terminator is specified and buffers should be resized as determined by
5047         // the terminator
5048 
5049         // Copy data to active buffer until terminator is found.
5050 
5051         // Decode as many lines as possible
5052         while (true)
5053         {
5054 
5055             // Make sure a buffer is present
5056             while (!bufferValid && freeBuffers.empty)
5057             {
5058                 // Active buffer is invalid and there are no available buffers in
5059                 // the pool. Wait for buffers to return from main thread in order to
5060                 // reuse them.
5061                 receive((immutable(Unit)[] buf)
5062                         {
5063                             buffer = cast(Unit[]) buf;
5064                             buffer.length = 0;
5065                             buffer.assumeSafeAppend();
5066                             bufferValid = true;
5067                         },
5068                         (bool flag) { aborted = true; }
5069                         );
5070                 if (aborted) return cast(size_t) 0;
5071             }
5072             if (!bufferValid)
5073             {
5074                 buffer = freeBuffers.pop();
5075                 bufferValid = true;
5076             }
5077 
5078             // Try to read a line from left over bytes from last onReceive plus the
5079             // newly received bytes.
5080             try
5081             {
5082                 if (decodeLineInto(leftOverBytes, data, buffer,
5083                                    encodingScheme, terminator))
5084                 {
5085                     if (keepTerminator)
5086                     {
5087                         fromTid.send(thisTid,
5088                                      curlMessage(cast(immutable(Unit)[])buffer));
5089                     }
5090                     else
5091                     {
5092                         static if (isArray!Terminator)
5093                             fromTid.send(thisTid,
5094                                          curlMessage(cast(immutable(Unit)[])
5095                                                  buffer[0..$-terminator.length]));
5096                         else
5097                             fromTid.send(thisTid,
5098                                          curlMessage(cast(immutable(Unit)[])
5099                                                  buffer[0..$-1]));
5100                     }
5101                     bufferValid = false;
5102                 }
5103                 else
5104                 {
5105                     // Could not decode an entire line. Save
5106                     // bytes left in data for next call to
5107                     // onReceive. Can be up to a max of 4 bytes.
5108                     enforce!CurlException(data.length <= 4,
5109                                             format(
5110                                             "Too many bytes left not decoded %s"~
5111                                             " > 4. Maybe the charset specified in"~
5112                                             " headers does not match "~
5113                                             "the actual content downloaded?",
5114                                             data.length));
5115                     leftOverBytes ~= data;
5116                     break;
5117                 }
5118             }
5119             catch (CurlException ex)
5120             {
5121                 prioritySend(fromTid, cast(immutable(CurlException))ex);
5122                 return cast(size_t) 0;
5123             }
5124         }
5125         return datalen;
5126     }
5127 
5128     // ditto
5129     private static
5130     void finalizeLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid)
5131     {
5132         import std.concurrency : send, thisTid;
5133         if (bufferValid && buffer.length != 0)
5134             fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$]));
5135     }
5136 
5137     /* Used by byLineAsync/byChunkAsync to duplicate an existing connection
5138      * that can be used exclusively in a spawned thread.
5139      */
5140     private void duplicateConnection(Conn, PostData)
5141         (const(char)[] url, Conn conn, PostData postData, Tid tid)
5142     {
5143         import std.concurrency : send;
5144         import std.exception : enforce;
5145 
5146         // no move semantic available in std.concurrency ie. must use casting.
5147         auto connDup = conn.dup();
5148         connDup.url = url;
5149 
5150         static if ( is(Conn : HTTP) )
5151         {
5152             connDup.p.headersOut = null;
5153             connDup.method = conn.method == HTTP.Method.undefined ?
5154                 HTTP.Method.get : conn.method;
5155             if (postData !is null)
5156             {
5157                 if (connDup.method == HTTP.Method.put)
5158                 {
5159                     connDup.handle.set(CurlOption.infilesize_large,
5160                                        postData.length);
5161                 }
5162                 else
5163                 {
5164                     // post
5165                     connDup.method = HTTP.Method.post;
5166                     connDup.handle.set(CurlOption.postfieldsize_large,
5167                                        postData.length);
5168                 }
5169                 connDup.handle.set(CurlOption.copypostfields,
5170                                    cast(void*) postData.ptr);
5171             }
5172             tid.send(cast(ulong) connDup.handle.handle);
5173             tid.send(connDup.method);
5174         }
5175         else
5176         {
5177             enforce!CurlException(postData is null,
5178                                     "Cannot put ftp data using byLineAsync()");
5179             tid.send(cast(ulong) connDup.handle.handle);
5180             tid.send(HTTP.Method.undefined);
5181         }
5182         connDup.p.curl.handle = null; // make sure handle is not freed
5183     }
5184 
5185     // Spawn a thread for handling the reading of incoming data in the
5186     // background while the delegate is executing.  This will optimize
5187     // throughput by allowing simultaneous input (this struct) and
5188     // output (e.g. AsyncHTTPLineOutputRange).
5189     private static void spawn(Conn, Unit, Terminator = void)()
5190     {
5191         import std.concurrency : Tid, prioritySend, receiveOnly, send, thisTid;
5192         import etc.c.curl : CURL, CurlError;
5193         Tid fromTid = receiveOnly!Tid();
5194 
5195         // Get buffer to read into
5196         Pool!(Unit[]) freeBuffers;  // Free list of buffer objects
5197 
5198         // Number of bytes filled into active buffer
5199         Unit[] buffer;
5200         bool aborted = false;
5201 
5202         EncodingScheme encodingScheme;
5203         static if ( !is(Terminator == void))
5204         {
5205             // Only lines reading will receive a terminator
5206             const terminator = receiveOnly!Terminator();
5207             const keepTerminator = receiveOnly!bool();
5208 
5209             // max number of bytes to carry over from an onReceive
5210             // callback. This is 4 because it is the max code units to
5211             // decode a code point in the supported encodings.
5212             auto leftOverBytes =  new const(ubyte)[4];
5213             leftOverBytes.length = 0;
5214             auto bufferValid = false;
5215         }
5216         else
5217         {
5218             Unit[] outdata;
5219         }
5220 
5221         // no move semantic available in std.concurrency ie. must use casting.
5222         auto connDup = cast(CURL*) receiveOnly!ulong();
5223         auto client = Conn();
5224         client.p.curl.handle = connDup;
5225 
5226         // receive a method for both ftp and http but just use it for http
5227         auto method = receiveOnly!(HTTP.Method)();
5228 
5229         client.onReceive = (ubyte[] data)
5230         {
5231             // If no terminator is specified the chunk size is fixed.
5232             static if ( is(Terminator == void) )
5233                 return receiveChunks(data, outdata, freeBuffers, buffer,
5234                                      fromTid, aborted);
5235             else
5236                 return receiveLines(data, encodingScheme,
5237                                     keepTerminator, terminator, leftOverBytes,
5238                                     bufferValid, freeBuffers, buffer,
5239                                     fromTid, aborted);
5240         };
5241 
5242         static if ( is(Conn == HTTP) )
5243         {
5244             client.method = method;
5245             // register dummy header handler
5246             client.onReceiveHeader = (in char[] key, in char[] value)
5247             {
5248                 if (key == "content-type")
5249                     encodingScheme = EncodingScheme.create(client.p.charset);
5250             };
5251         }
5252         else
5253         {
5254             encodingScheme = EncodingScheme.create(client.encoding);
5255         }
5256 
5257         // Start the request
5258         CurlCode code;
5259         try
5260         {
5261             code = client.perform(No.throwOnError);
5262         }
5263         catch (Exception ex)
5264         {
5265             prioritySend(fromTid, cast(immutable(Exception)) ex);
5266             fromTid.send(thisTid, curlMessage(true)); // signal done
5267             return;
5268         }
5269 
5270         if (code != CurlError.ok)
5271         {
5272             if (aborted && (code == CurlError.aborted_by_callback ||
5273                             code == CurlError.write_error))
5274             {
5275                 fromTid.send(thisTid, curlMessage(true)); // signal done
5276                 return;
5277             }
5278             prioritySend(fromTid, cast(immutable(CurlException))
5279                          new CurlException(client.p.curl.errorString(code)));
5280 
5281             fromTid.send(thisTid, curlMessage(true)); // signal done
5282             return;
5283         }
5284 
5285         // Send remaining data that is not a full chunk size
5286         static if ( is(Terminator == void) )
5287             finalizeChunks(outdata, buffer, fromTid);
5288         else
5289             finalizeLines(bufferValid, buffer, fromTid);
5290 
5291         fromTid.send(thisTid, curlMessage(true)); // signal done
5292     }
5293 }
Suggestion Box / Bug Report