1 //Written in the D programming language 2 3 /** 4 * Implements functionality to read Comma Separated Values and its variants 5 * from an $(REF_ALTTEXT input range, isInputRange, std,range,primitives) of `dchar`. 6 * 7 * Comma Separated Values provide a simple means to transfer and store 8 * tabular data. It has been common for programs to use their own 9 * variant of the CSV format. This parser will loosely follow the 10 * $(HTTP tools.ietf.org/html/rfc4180, RFC-4180). CSV input should adhere 11 * to the following criteria (differences from RFC-4180 in parentheses): 12 * 13 * $(UL 14 * $(LI A record is separated by a new line (CRLF,LF,CR)) 15 * $(LI A final record may end with a new line) 16 * $(LI A header may be provided as the first record in input) 17 * $(LI A record has fields separated by a comma (customizable)) 18 * $(LI A field containing new lines, commas, or double quotes 19 * should be enclosed in double quotes (customizable)) 20 * $(LI Double quotes in a field are escaped with a double quote) 21 * $(LI Each record should contain the same number of fields) 22 * ) 23 * 24 * Example: 25 * 26 * ------- 27 * import std.algorithm; 28 * import std.array; 29 * import std.csv; 30 * import std.stdio; 31 * import std.typecons; 32 * 33 * void main() 34 * { 35 * auto text = "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n"; 36 * 37 * foreach (record; csvReader!(Tuple!(string, string, int))(text)) 38 * { 39 * writefln("%s works as a %s and earns $%d per year", 40 * record[0], record[1], record[2]); 41 * } 42 * 43 * // To read the same string from the file "filename.csv": 44 * 45 * auto file = File("filename.csv", "r"); 46 * foreach (record; 47 * file.byLine.joiner("\n").csvReader!(Tuple!(string, string, int))) 48 * { 49 * writefln("%s works as a %s and earns $%d per year", 50 * record[0], record[1], record[2]); 51 * } 52 } 53 * } 54 * ------- 55 * 56 * When an input contains a header the `Contents` can be specified as an 57 * associative array. Passing null to signify that a header is present. 58 * 59 * ------- 60 * auto text = "Name,Occupation,Salary\r" 61 * "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n"; 62 * 63 * foreach (record; csvReader!(string[string]) 64 * (text, null)) 65 * { 66 * writefln("%s works as a %s and earns $%s per year.", 67 * record["Name"], record["Occupation"], 68 * record["Salary"]); 69 * } 70 * 71 * // To read the same string from the file "filename.csv": 72 * 73 * auto file = File("filename.csv", "r"); 74 * 75 * foreach (record; csvReader!(string[string]) 76 * (file.byLine.joiner("\n"), null)) 77 * { 78 * writefln("%s works as a %s and earns $%s per year.", 79 * record["Name"], record["Occupation"], 80 * record["Salary"]); 81 * } 82 * ------- 83 * 84 * This module allows content to be iterated by record stored in a struct, 85 * class, associative array, or as a range of fields. Upon detection of an 86 * error an CSVException is thrown (can be disabled). csvNextToken has been 87 * made public to allow for attempted recovery. 88 * 89 * Disabling exceptions will lift many restrictions specified above. A quote 90 * can appear in a field if the field was not quoted. If in a quoted field any 91 * quote by itself, not at the end of a field, will end processing for that 92 * field. The field is ended when there is no input, even if the quote was not 93 * closed. 94 * 95 * See_Also: 96 * $(HTTP en.wikipedia.org/wiki/Comma-separated_values, Wikipedia 97 * Comma-separated values) 98 * 99 * Copyright: Copyright 2011 100 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 101 * Authors: Jesse Phillips 102 * Source: $(PHOBOSSRC std/csv.d) 103 */ 104 module std.csv; 105 106 import std.conv; 107 import std.exception : basicExceptionCtors; 108 import std.range.primitives; 109 import std.traits; 110 111 /** 112 * Exception containing the row and column for when an exception was thrown. 113 * 114 * Numbering of both row and col start at one and corresponds to the location 115 * in the file rather than any specified header. Special consideration should 116 * be made when there is failure to match the header see $(LREF 117 * HeaderMismatchException) for details. 118 * 119 * When performing type conversions, $(REF ConvException, std,conv) is stored in 120 * the `next` field. 121 */ 122 class CSVException : Exception 123 { 124 /// 125 size_t row, col; 126 127 // FIXME: Use std.exception.basicExceptionCtors here once 128 // https://issues.dlang.org/show_bug.cgi?id=11500 is fixed 129 130 this(string msg, string file = __FILE__, size_t line = __LINE__, 131 Throwable next = null) @nogc @safe pure nothrow 132 { 133 super(msg, file, line, next); 134 } 135 136 this(string msg, Throwable next, string file = __FILE__, 137 size_t line = __LINE__) @nogc @safe pure nothrow 138 { 139 super(msg, file, line, next); 140 } 141 142 this(string msg, size_t row, size_t col, Throwable next = null, 143 string file = __FILE__, size_t line = __LINE__) @nogc @safe pure nothrow 144 { 145 super(msg, next, file, line); 146 this.row = row; 147 this.col = col; 148 } 149 150 override string toString() @safe pure const 151 { 152 return "(Row: " ~ to!string(row) ~ 153 ", Col: " ~ to!string(col) ~ ") " ~ msg; 154 } 155 } 156 157 /// 158 @safe unittest 159 { 160 import std.exception : collectException; 161 import std.algorithm.searching : count; 162 string text = "a,b,c\nHello,65"; 163 auto ex = collectException!CSVException(csvReader(text).count); 164 assert(ex.toString == "(Row: 0, Col: 0) Row 2's length 2 does not match previous length of 3."); 165 } 166 167 /// 168 @safe unittest 169 { 170 import std.exception : collectException; 171 import std.algorithm.searching : count; 172 import std.typecons : Tuple; 173 string text = "a,b\nHello,65"; 174 auto ex = collectException!CSVException(csvReader!(Tuple!(string,int))(text).count); 175 assert(ex.toString == "(Row: 1, Col: 2) Unexpected 'b' when converting from type string to type int"); 176 } 177 178 @safe pure unittest 179 { 180 import std..string; 181 auto e1 = new Exception("Foobar"); 182 auto e2 = new CSVException("args", e1); 183 assert(e2.next is e1); 184 185 size_t r = 13; 186 size_t c = 37; 187 188 auto e3 = new CSVException("argv", r, c); 189 assert(e3.row == r); 190 assert(e3.col == c); 191 192 auto em = e3.toString(); 193 assert(em.indexOf("13") != -1); 194 assert(em.indexOf("37") != -1); 195 } 196 197 /** 198 * Exception thrown when a Token is identified to not be completed: a quote is 199 * found in an unquoted field, data continues after a closing quote, or the 200 * quoted field was not closed before data was empty. 201 */ 202 class IncompleteCellException : CSVException 203 { 204 /** 205 * Data pulled from input before finding a problem 206 * 207 * This field is populated when using $(LREF csvReader) 208 * but not by $(LREF csvNextToken) as this data will have 209 * already been fed to the output range. 210 */ 211 dstring partialData; 212 213 mixin basicExceptionCtors; 214 } 215 216 /// 217 @safe unittest 218 { 219 import std.exception : assertThrown; 220 string text = "a,\"b,c\nHello,65,2.5"; 221 assertThrown!IncompleteCellException(text.csvReader(["a","b","c"])); 222 } 223 224 @safe pure unittest 225 { 226 auto e1 = new Exception("Foobar"); 227 auto e2 = new IncompleteCellException("args", e1); 228 assert(e2.next is e1); 229 } 230 231 /** 232 * Exception thrown under different conditions based on the type of $(D 233 * Contents). 234 * 235 * Structure, Class, and Associative Array 236 * $(UL 237 * $(LI When a header is provided but a matching column is not found) 238 * ) 239 * 240 * Other 241 * $(UL 242 * $(LI When a header is provided but a matching column is not found) 243 * $(LI Order did not match that found in the input) 244 * ) 245 * 246 * Since a row and column is not meaningful when a column specified by the 247 * header is not found in the data, both row and col will be zero. Otherwise 248 * row is always one and col is the first instance found in header that 249 * occurred before the previous starting at one. 250 */ 251 class HeaderMismatchException : CSVException 252 { 253 mixin basicExceptionCtors; 254 } 255 256 /// 257 @safe unittest 258 { 259 import std.exception : assertThrown; 260 string text = "a,b,c\nHello,65,2.5"; 261 assertThrown!HeaderMismatchException(text.csvReader(["b","c","invalid"])); 262 } 263 264 @safe pure unittest 265 { 266 auto e1 = new Exception("Foobar"); 267 auto e2 = new HeaderMismatchException("args", e1); 268 assert(e2.next is e1); 269 } 270 271 /** 272 * Determines the behavior for when an error is detected. 273 * 274 * Disabling exception will follow these rules: 275 * $(UL 276 * $(LI A quote can appear in a field if the field was not quoted.) 277 * $(LI If in a quoted field any quote by itself, not at the end of a 278 * field, will end processing for that field.) 279 * $(LI The field is ended when there is no input, even if the quote was 280 * not closed.) 281 * $(LI If the given header does not match the order in the input, the 282 * content will return as it is found in the input.) 283 * $(LI If the given header contains columns not found in the input they 284 * will be ignored.) 285 * ) 286 */ 287 enum Malformed 288 { 289 ignore, /// No exceptions are thrown due to incorrect CSV. 290 throwException /// Use exceptions when input has incorrect CSV. 291 } 292 293 /// 294 @safe unittest 295 { 296 import std.algorithm.comparison : equal; 297 import std.algorithm.searching : count; 298 import std.exception : assertThrown; 299 300 string text = "a,b,c\nHello,65,\"2.5"; 301 assertThrown!IncompleteCellException(text.csvReader.count); 302 303 // ignore the exceptions and try to handle invalid CSV 304 auto firstLine = text.csvReader!(string, Malformed.ignore)(null).front; 305 assert(firstLine.equal(["Hello", "65", "2.5"])); 306 } 307 308 /** 309 Returns an $(REF_ALTTEXT input range, isInputRange, std,range,primitives) 310 for iterating over records found in `input`. 311 312 An optional `header` can be provided. The first record will be read in 313 as the header. If `Contents` is a struct then the header provided is 314 expected to correspond to the fields in the struct. When `Contents` is 315 not a type which can contain the entire record, the `header` must be 316 provided in the same order as the input or an exception is thrown. 317 318 Returns: 319 An input range R as defined by 320 $(REF isInputRange, std,range,primitives). When `Contents` is a 321 struct, class, or an associative array, the element type of R is 322 `Contents`, otherwise the element type of R is itself a range with 323 element type `Contents`. 324 325 If a `header` argument is provided, 326 the returned range provides a `header` field for accessing the header 327 from the input in array form. 328 329 Throws: 330 $(LREF CSVException) When a quote is found in an unquoted field, 331 data continues after a closing quote, the quoted field was not 332 closed before data was empty, a conversion failed, or when the row's 333 length does not match the previous length. 334 335 $(LREF HeaderMismatchException) when a header is provided but a 336 matching column is not found or the order did not match that found in 337 the input. Read the exception documentation for specific details of 338 when the exception is thrown for different types of `Contents`. 339 */ 340 auto csvReader(Contents = string,Malformed ErrorLevel = Malformed.throwException, Range, Separator = char)(Range input, 341 Separator delimiter = ',', Separator quote = '"') 342 if (isInputRange!Range && is(immutable ElementType!Range == immutable dchar) 343 && isSomeChar!(Separator) 344 && !is(Contents T : T[U], U : string)) 345 { 346 return CsvReader!(Contents,ErrorLevel,Range, 347 Unqual!(ElementType!Range),string[]) 348 (input, delimiter, quote); 349 } 350 351 /// ditto 352 auto csvReader(Contents = string, 353 Malformed ErrorLevel = Malformed.throwException, 354 Range, Header, Separator = char) 355 (Range input, Header header, 356 Separator delimiter = ',', Separator quote = '"') 357 if (isInputRange!Range && is(immutable ElementType!Range == immutable dchar) 358 && isSomeChar!(Separator) 359 && isForwardRange!Header 360 && isSomeString!(ElementType!Header)) 361 { 362 return CsvReader!(Contents,ErrorLevel,Range, 363 Unqual!(ElementType!Range),Header) 364 (input, header, delimiter, quote); 365 } 366 367 /// ditto 368 auto csvReader(Contents = string, 369 Malformed ErrorLevel = Malformed.throwException, 370 Range, Header, Separator = char) 371 (Range input, Header header, 372 Separator delimiter = ',', Separator quote = '"') 373 if (isInputRange!Range && is(immutable ElementType!Range == immutable dchar) 374 && isSomeChar!(Separator) 375 && is(Header : typeof(null))) 376 { 377 return CsvReader!(Contents,ErrorLevel,Range, 378 Unqual!(ElementType!Range),string[]) 379 (input, cast(string[]) null, delimiter, quote); 380 } 381 382 383 /** 384 The `Contents` of the input can be provided if all the records are the 385 same type such as all integer data: 386 */ 387 @safe unittest 388 { 389 import std.algorithm.comparison : equal; 390 string text = "76,26,22"; 391 auto records = text.csvReader!int; 392 assert(records.equal!equal([ 393 [76, 26, 22], 394 ])); 395 } 396 397 /** 398 Using a struct with modified delimiter: 399 */ 400 @safe unittest 401 { 402 import std.algorithm.comparison : equal; 403 string text = "Hello;65;2.5\nWorld;123;7.5"; 404 struct Layout 405 { 406 string name; 407 int value; 408 double other; 409 } 410 411 auto records = text.csvReader!Layout(';'); 412 assert(records.equal([ 413 Layout("Hello", 65, 2.5), 414 Layout("World", 123, 7.5), 415 ])); 416 } 417 418 /** 419 Specifying `ErrorLevel` as $(LREF Malformed.ignore) will lift restrictions 420 on the format. This example shows that an exception is not thrown when 421 finding a quote in a field not quoted. 422 */ 423 @safe unittest 424 { 425 string text = "A \" is now part of the data"; 426 auto records = text.csvReader!(string, Malformed.ignore); 427 auto record = records.front; 428 429 assert(record.front == text); 430 } 431 432 /// Read only column "b" 433 @safe unittest 434 { 435 import std.algorithm.comparison : equal; 436 string text = "a,b,c\nHello,65,63.63\nWorld,123,3673.562"; 437 auto records = text.csvReader!int(["b"]); 438 439 assert(records.equal!equal([ 440 [65], 441 [123], 442 ])); 443 } 444 445 /// Read while rearranging the columns by specifying a header with a different order" 446 @safe unittest 447 { 448 import std.algorithm.comparison : equal; 449 string text = "a,b,c\nHello,65,2.5\nWorld,123,7.5"; 450 struct Layout 451 { 452 int value; 453 double other; 454 string name; 455 } 456 457 auto records = text.csvReader!Layout(["b","c","a"]); 458 assert(records.equal([ 459 Layout(65, 2.5, "Hello"), 460 Layout(123, 7.5, "World") 461 ])); 462 } 463 464 /** 465 The header can also be left empty if the input contains a header row 466 and all columns should be iterated. 467 The header from the input can always be accessed from the `header` field. 468 */ 469 @safe unittest 470 { 471 string text = "a,b,c\nHello,65,63.63"; 472 auto records = text.csvReader(null); 473 474 assert(records.header == ["a","b","c"]); 475 } 476 477 // Test standard iteration over input. 478 @safe pure unittest 479 { 480 string str = `one,"two ""quoted"""` ~ "\n\"three\nnew line\",\nfive,six"; 481 auto records = csvReader(str); 482 483 int count; 484 foreach (record; records) 485 { 486 foreach (cell; record) 487 { 488 count++; 489 } 490 } 491 assert(count == 6); 492 } 493 494 // Test newline on last record 495 @safe pure unittest 496 { 497 string str = "one,two\nthree,four\n"; 498 auto records = csvReader(str); 499 records.popFront(); 500 records.popFront(); 501 assert(records.empty); 502 } 503 504 // Test shorter row length 505 @safe pure unittest 506 { 507 wstring str = "one,1\ntwo\nthree"w; 508 struct Layout 509 { 510 string name; 511 int value; 512 } 513 514 Layout[3] ans; 515 ans[0].name = "one"; 516 ans[0].value = 1; 517 ans[1].name = "two"; 518 ans[1].value = 0; 519 ans[2].name = "three"; 520 ans[2].value = 0; 521 522 auto records = csvReader!(Layout,Malformed.ignore)(str); 523 524 int count; 525 foreach (record; records) 526 { 527 assert(ans[count].name == record.name); 528 assert(ans[count].value == record.value); 529 count++; 530 } 531 } 532 533 // Test shorter row length exception 534 @safe pure unittest 535 { 536 import std.exception; 537 538 struct A 539 { 540 string a,b,c; 541 } 542 543 auto strs = ["one,1\ntwo", 544 "one\ntwo,2,二\nthree,3,三", 545 "one\ntwo,2\nthree,3", 546 "one,1\ntwo\nthree,3"]; 547 548 foreach (str; strs) 549 { 550 auto records = csvReader!A(str); 551 assertThrown!CSVException((){foreach (record; records) { }}()); 552 } 553 } 554 555 556 // Test structure conversion interface with unicode. 557 @safe pure unittest 558 { 559 import std.math : abs; 560 561 wstring str = "\U00010143Hello,65,63.63\nWorld,123,3673.562"w; 562 struct Layout 563 { 564 string name; 565 int value; 566 double other; 567 } 568 569 Layout[2] ans; 570 ans[0].name = "\U00010143Hello"; 571 ans[0].value = 65; 572 ans[0].other = 63.63; 573 ans[1].name = "World"; 574 ans[1].value = 123; 575 ans[1].other = 3673.562; 576 577 auto records = csvReader!Layout(str); 578 579 int count; 580 foreach (record; records) 581 { 582 assert(ans[count].name == record.name); 583 assert(ans[count].value == record.value); 584 assert(abs(ans[count].other - record.other) < 0.00001); 585 count++; 586 } 587 assert(count == ans.length); 588 } 589 590 // Test input conversion interface 591 @safe pure unittest 592 { 593 import std.algorithm; 594 string str = `76,26,22`; 595 int[] ans = [76,26,22]; 596 auto records = csvReader!int(str); 597 598 foreach (record; records) 599 { 600 assert(equal(record, ans)); 601 } 602 } 603 604 // Test struct & header interface and same unicode 605 @safe unittest 606 { 607 import std.math : abs; 608 609 string str = "a,b,c\nHello,65,63.63\n➊➋➂❹,123,3673.562"; 610 struct Layout 611 { 612 int value; 613 double other; 614 string name; 615 } 616 617 auto records = csvReader!Layout(str, ["b","c","a"]); 618 619 Layout[2] ans; 620 ans[0].name = "Hello"; 621 ans[0].value = 65; 622 ans[0].other = 63.63; 623 ans[1].name = "➊➋➂❹"; 624 ans[1].value = 123; 625 ans[1].other = 3673.562; 626 627 int count; 628 foreach (record; records) 629 { 630 assert(ans[count].name == record.name); 631 assert(ans[count].value == record.value); 632 assert(abs(ans[count].other - record.other) < 0.00001); 633 count++; 634 } 635 assert(count == ans.length); 636 637 } 638 639 // Test header interface 640 @safe unittest 641 { 642 import std.algorithm; 643 644 string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562"; 645 auto records = csvReader!int(str, ["b"]); 646 647 auto ans = [[65],[123]]; 648 foreach (record; records) 649 { 650 assert(equal(record, ans.front)); 651 ans.popFront(); 652 } 653 654 try 655 { 656 csvReader(str, ["c","b"]); 657 assert(0); 658 } 659 catch (HeaderMismatchException e) 660 { 661 assert(e.col == 2); 662 } 663 auto records2 = csvReader!(string,Malformed.ignore) 664 (str, ["b","a"], ',', '"'); 665 666 auto ans2 = [["Hello","65"],["World","123"]]; 667 foreach (record; records2) 668 { 669 assert(equal(record, ans2.front)); 670 ans2.popFront(); 671 } 672 673 str = "a,c,e\nJoe,Carpenter,300000\nFred,Fly,4"; 674 records2 = csvReader!(string,Malformed.ignore) 675 (str, ["a","b","c","d"], ',', '"'); 676 677 ans2 = [["Joe","Carpenter"],["Fred","Fly"]]; 678 foreach (record; records2) 679 { 680 assert(equal(record, ans2.front)); 681 ans2.popFront(); 682 } 683 } 684 685 // Test null header interface 686 @safe unittest 687 { 688 string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562"; 689 auto records = csvReader(str, ["a"]); 690 691 assert(records.header == ["a","b","c"]); 692 } 693 694 // Test unchecked read 695 @safe pure unittest 696 { 697 string str = "one \"quoted\""; 698 foreach (record; csvReader!(string,Malformed.ignore)(str)) 699 { 700 foreach (cell; record) 701 { 702 assert(cell == "one \"quoted\""); 703 } 704 } 705 706 str = "one \"quoted\",two \"quoted\" end"; 707 struct Ans 708 { 709 string a,b; 710 } 711 foreach (record; csvReader!(Ans,Malformed.ignore)(str)) 712 { 713 assert(record.a == "one \"quoted\""); 714 assert(record.b == "two \"quoted\" end"); 715 } 716 } 717 718 // Test partial data returned 719 @safe pure unittest 720 { 721 string str = "\"one\nnew line"; 722 723 try 724 { 725 foreach (record; csvReader(str)) 726 {} 727 assert(0); 728 } 729 catch (IncompleteCellException ice) 730 { 731 assert(ice.partialData == "one\nnew line"); 732 } 733 } 734 735 // Test Windows line break 736 @safe pure unittest 737 { 738 string str = "one,two\r\nthree"; 739 740 auto records = csvReader(str); 741 auto record = records.front; 742 assert(record.front == "one"); 743 record.popFront(); 744 assert(record.front == "two"); 745 records.popFront(); 746 record = records.front; 747 assert(record.front == "three"); 748 } 749 750 751 // Test associative array support with unicode separator 752 @safe unittest 753 { 754 string str = "1❁2❁3\n34❁65❁63\n34❁65❁63"; 755 756 auto records = csvReader!(string[string])(str,["3","1"],'❁'); 757 int count; 758 foreach (record; records) 759 { 760 count++; 761 assert(record["1"] == "34"); 762 assert(record["3"] == "63"); 763 } 764 assert(count == 2); 765 } 766 767 // Test restricted range 768 @safe unittest 769 { 770 import std.typecons; 771 struct InputRange 772 { 773 dstring text; 774 775 this(dstring txt) 776 { 777 text = txt; 778 } 779 780 @property auto empty() 781 { 782 return text.empty; 783 } 784 785 void popFront() 786 { 787 text.popFront(); 788 } 789 790 @property dchar front() 791 { 792 return text[0]; 793 } 794 } 795 auto ir = InputRange("Name,Occupation,Salary\r"d~ 796 "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n"d); 797 798 foreach (record; csvReader(ir, cast(string[]) null)) 799 foreach (cell; record) {} 800 foreach (record; csvReader!(Tuple!(string, string, int)) 801 (ir,cast(string[]) null)) {} 802 foreach (record; csvReader!(string[string]) 803 (ir,cast(string[]) null)) {} 804 } 805 806 @safe unittest // const/immutable dchars 807 { 808 import std.algorithm.iteration : map; 809 import std.array : array; 810 const(dchar)[] c = "foo,bar\n"; 811 assert(csvReader(c).map!array.array == [["foo", "bar"]]); 812 immutable(dchar)[] i = "foo,bar\n"; 813 assert(csvReader(i).map!array.array == [["foo", "bar"]]); 814 } 815 816 /* 817 * This struct is stored on the heap for when the structures 818 * are passed around. 819 */ 820 private pure struct Input(Range, Malformed ErrorLevel) 821 { 822 Range range; 823 size_t row, col; 824 static if (ErrorLevel == Malformed.throwException) 825 size_t rowLength; 826 } 827 828 /* 829 * Range for iterating CSV records. 830 * 831 * This range is returned by the $(LREF csvReader) functions. It can be 832 * created in a similar manner to allow `ErrorLevel` be set to $(LREF 833 * Malformed).ignore if best guess processing should take place. 834 */ 835 private struct CsvReader(Contents, Malformed ErrorLevel, Range, Separator, Header) 836 if (isSomeChar!Separator && isInputRange!Range 837 && is(immutable ElementType!Range == immutable dchar) 838 && isForwardRange!Header && isSomeString!(ElementType!Header)) 839 { 840 private: 841 Input!(Range, ErrorLevel)* _input; 842 Separator _separator; 843 Separator _quote; 844 size_t[] indices; 845 bool _empty; 846 static if (is(Contents == struct) || is(Contents == class)) 847 { 848 Contents recordContent; 849 CsvRecord!(string, ErrorLevel, Range, Separator) recordRange; 850 } 851 else static if (is(Contents T : T[U], U : string)) 852 { 853 Contents recordContent; 854 CsvRecord!(T, ErrorLevel, Range, Separator) recordRange; 855 } 856 else 857 CsvRecord!(Contents, ErrorLevel, Range, Separator) recordRange; 858 public: 859 /** 860 * Header from the input in array form. 861 * 862 * ------- 863 * string str = "a,b,c\nHello,65,63.63"; 864 * auto records = csvReader(str, ["a"]); 865 * 866 * assert(records.header == ["a","b","c"]); 867 * ------- 868 */ 869 string[] header; 870 871 /** 872 * Constructor to initialize the input, delimiter and quote for input 873 * without a header. 874 * 875 * ------- 876 * string str = `76;^26^;22`; 877 * int[] ans = [76,26,22]; 878 * auto records = CsvReader!(int,Malformed.ignore,string,char,string[]) 879 * (str, ';', '^'); 880 * 881 * foreach (record; records) 882 * { 883 * assert(equal(record, ans)); 884 * } 885 * ------- 886 */ 887 this(Range input, Separator delimiter, Separator quote) 888 { 889 _input = new Input!(Range, ErrorLevel)(input); 890 _separator = delimiter; 891 _quote = quote; 892 893 prime(); 894 } 895 896 /** 897 * Constructor to initialize the input, delimiter and quote for input 898 * with a header. 899 * 900 * ------- 901 * string str = `high;mean;low\n76;^26^;22`; 902 * auto records = CsvReader!(int,Malformed.ignore,string,char,string[]) 903 * (str, ["high","low"], ';', '^'); 904 * 905 * int[] ans = [76,22]; 906 * foreach (record; records) 907 * { 908 * assert(equal(record, ans)); 909 * } 910 * ------- 911 * 912 * Throws: 913 * $(LREF HeaderMismatchException) when a header is provided but a 914 * matching column is not found or the order did not match that found 915 * in the input (non-struct). 916 */ 917 this(Range input, Header colHeaders, Separator delimiter, Separator quote) 918 { 919 _input = new Input!(Range, ErrorLevel)(input); 920 _separator = delimiter; 921 _quote = quote; 922 923 size_t[string] colToIndex; 924 foreach (h; colHeaders) 925 { 926 colToIndex[h] = size_t.max; 927 } 928 929 auto r = CsvRecord!(string, ErrorLevel, Range, Separator) 930 (_input, _separator, _quote, indices); 931 932 size_t colIndex; 933 foreach (col; r) 934 { 935 header ~= col; 936 auto ptr = col in colToIndex; 937 if (ptr) 938 *ptr = colIndex; 939 colIndex++; 940 } 941 // The above loop empties the header row. 942 recordRange._empty = true; 943 944 indices.length = colToIndex.length; 945 int i; 946 foreach (h; colHeaders) 947 { 948 immutable index = colToIndex[h]; 949 static if (ErrorLevel != Malformed.ignore) 950 if (index == size_t.max) 951 throw new HeaderMismatchException 952 ("Header not found: " ~ to!string(h)); 953 indices[i++] = index; 954 } 955 956 static if (!is(Contents == struct) && !is(Contents == class)) 957 { 958 static if (is(Contents T : T[U], U : string)) 959 { 960 import std.algorithm.sorting : sort; 961 sort(indices); 962 } 963 else static if (ErrorLevel == Malformed.ignore) 964 { 965 import std.algorithm.sorting : sort; 966 sort(indices); 967 } 968 else 969 { 970 import std.algorithm.searching : findAdjacent; 971 import std.algorithm.sorting : isSorted; 972 if (!isSorted(indices)) 973 { 974 auto ex = new HeaderMismatchException 975 ("Header in input does not match specified header."); 976 findAdjacent!"a > b"(indices); 977 ex.row = 1; 978 ex.col = indices.front; 979 980 throw ex; 981 } 982 } 983 } 984 985 popFront(); 986 } 987 988 /** 989 * Part of an input range as defined by 990 * $(REF isInputRange, std,range,primitives). 991 * 992 * Returns: 993 * If `Contents` is a struct, will be filled with record data. 994 * 995 * If `Contents` is a class, will be filled with record data. 996 * 997 * If `Contents` is a associative array, will be filled 998 * with record data. 999 * 1000 * If `Contents` is non-struct, a $(LREF CsvRecord) will be 1001 * returned. 1002 */ 1003 @property auto front() 1004 { 1005 assert(!empty, "Attempting to fetch the front of an empty CsvReader"); 1006 static if (is(Contents == struct) || is(Contents == class)) 1007 { 1008 return recordContent; 1009 } 1010 else static if (is(Contents T : T[U], U : string)) 1011 { 1012 return recordContent; 1013 } 1014 else 1015 { 1016 return recordRange; 1017 } 1018 } 1019 1020 /** 1021 * Part of an input range as defined by 1022 * $(REF isInputRange, std,range,primitives). 1023 */ 1024 @property bool empty() @safe @nogc pure nothrow const 1025 { 1026 return _empty; 1027 } 1028 1029 /** 1030 * Part of an input range as defined by 1031 * $(REF isInputRange, std,range,primitives). 1032 * 1033 * Throws: 1034 * $(LREF CSVException) When a quote is found in an unquoted field, 1035 * data continues after a closing quote, the quoted field was not 1036 * closed before data was empty, a conversion failed, or when the 1037 * row's length does not match the previous length. 1038 */ 1039 void popFront() 1040 { 1041 while (!recordRange.empty) 1042 { 1043 recordRange.popFront(); 1044 } 1045 1046 static if (ErrorLevel == Malformed.throwException) 1047 if (_input.rowLength == 0) 1048 _input.rowLength = _input.col; 1049 1050 _input.col = 0; 1051 1052 if (!_input.range.empty) 1053 { 1054 if (_input.range.front == '\r') 1055 { 1056 _input.range.popFront(); 1057 if (!_input.range.empty && _input.range.front == '\n') 1058 _input.range.popFront(); 1059 } 1060 else if (_input.range.front == '\n') 1061 _input.range.popFront(); 1062 } 1063 1064 if (_input.range.empty) 1065 { 1066 _empty = true; 1067 return; 1068 } 1069 1070 prime(); 1071 } 1072 1073 private void prime() 1074 { 1075 if (_empty) 1076 return; 1077 _input.row++; 1078 static if (is(Contents == struct) || is(Contents == class)) 1079 { 1080 recordRange = typeof(recordRange) 1081 (_input, _separator, _quote, null); 1082 } 1083 else 1084 { 1085 recordRange = typeof(recordRange) 1086 (_input, _separator, _quote, indices); 1087 } 1088 1089 static if (is(Contents T : T[U], U : string)) 1090 { 1091 T[U] aa; 1092 try 1093 { 1094 for (; !recordRange.empty; recordRange.popFront()) 1095 { 1096 aa[header[_input.col-1]] = recordRange.front; 1097 } 1098 } 1099 catch (ConvException e) 1100 { 1101 throw new CSVException(e.msg, _input.row, _input.col, e); 1102 } 1103 1104 recordContent = aa; 1105 } 1106 else static if (is(Contents == struct) || is(Contents == class)) 1107 { 1108 static if (is(Contents == class)) 1109 recordContent = new typeof(recordContent)(); 1110 else 1111 recordContent = typeof(recordContent).init; 1112 size_t colIndex; 1113 try 1114 { 1115 for (; !recordRange.empty;) 1116 { 1117 auto colData = recordRange.front; 1118 scope(exit) colIndex++; 1119 if (indices.length > 0) 1120 { 1121 foreach (ti, ToType; Fields!(Contents)) 1122 { 1123 if (indices[ti] == colIndex) 1124 { 1125 static if (!isSomeString!ToType) skipWS(colData); 1126 recordContent.tupleof[ti] = to!ToType(colData); 1127 } 1128 } 1129 } 1130 else 1131 { 1132 foreach (ti, ToType; Fields!(Contents)) 1133 { 1134 if (ti == colIndex) 1135 { 1136 static if (!isSomeString!ToType) skipWS(colData); 1137 recordContent.tupleof[ti] = to!ToType(colData); 1138 } 1139 } 1140 } 1141 recordRange.popFront(); 1142 } 1143 } 1144 catch (ConvException e) 1145 { 1146 throw new CSVException(e.msg, _input.row, colIndex, e); 1147 } 1148 } 1149 } 1150 } 1151 1152 @safe pure unittest 1153 { 1154 import std.algorithm.comparison : equal; 1155 1156 string str = `76;^26^;22`; 1157 int[] ans = [76,26,22]; 1158 auto records = CsvReader!(int,Malformed.ignore,string,char,string[]) 1159 (str, ';', '^'); 1160 1161 foreach (record; records) 1162 { 1163 assert(equal(record, ans)); 1164 } 1165 } 1166 1167 // https://issues.dlang.org/show_bug.cgi?id=15545 1168 // @system due to the catch for Throwable 1169 @system pure unittest 1170 { 1171 import std.exception : assertNotThrown; 1172 enum failData = 1173 "name, surname, age 1174 Joe, Joker, 99\r"; 1175 auto r = csvReader(failData); 1176 assertNotThrown((){foreach (entry; r){}}()); 1177 } 1178 1179 /* 1180 * This input range is accessible through $(LREF CsvReader) when the 1181 * requested `Contents` type is neither a structure or an associative array. 1182 */ 1183 private struct CsvRecord(Contents, Malformed ErrorLevel, Range, Separator) 1184 if (!is(Contents == class) && !is(Contents == struct)) 1185 { 1186 import std.array : appender; 1187 private: 1188 Input!(Range, ErrorLevel)* _input; 1189 Separator _separator; 1190 Separator _quote; 1191 Contents curContentsoken; 1192 typeof(appender!(dchar[])()) _front; 1193 bool _empty; 1194 size_t[] _popCount; 1195 public: 1196 /* 1197 * Params: 1198 * input = Pointer to a character $(REF_ALTTEXT input range, isInputRange, std,range,primitives) 1199 * delimiter = Separator for each column 1200 * quote = Character used for quotation 1201 * indices = An array containing which columns will be returned. 1202 * If empty, all columns are returned. List must be in order. 1203 */ 1204 this(Input!(Range, ErrorLevel)* input, Separator delimiter, 1205 Separator quote, size_t[] indices) 1206 { 1207 _input = input; 1208 _separator = delimiter; 1209 _quote = quote; 1210 _front = appender!(dchar[])(); 1211 _popCount = indices.dup; 1212 1213 // If a header was given, each call to popFront will need 1214 // to eliminate so many tokens. This calculates 1215 // how many will be skipped to get to the next header column 1216 size_t normalizer; 1217 foreach (ref c; _popCount) 1218 { 1219 static if (ErrorLevel == Malformed.ignore) 1220 { 1221 // If we are not throwing exceptions 1222 // a header may not exist, indices are sorted 1223 // and will be size_t.max if not found. 1224 if (c == size_t.max) 1225 break; 1226 } 1227 c -= normalizer; 1228 normalizer += c + 1; 1229 } 1230 1231 prime(); 1232 } 1233 1234 /** 1235 * Part of an input range as defined by 1236 * $(REF isInputRange, std,range,primitives). 1237 */ 1238 @property Contents front() @safe pure 1239 { 1240 assert(!empty, "Attempting to fetch the front of an empty CsvRecord"); 1241 return curContentsoken; 1242 } 1243 1244 /** 1245 * Part of an input range as defined by 1246 * $(REF isInputRange, std,range,primitives). 1247 */ 1248 @property bool empty() @safe pure nothrow @nogc const 1249 { 1250 return _empty; 1251 } 1252 1253 /* 1254 * CsvRecord is complete when input 1255 * is empty or starts with record break 1256 */ 1257 private bool recordEnd() 1258 { 1259 if (_input.range.empty 1260 || _input.range.front == '\n' 1261 || _input.range.front == '\r') 1262 { 1263 return true; 1264 } 1265 return false; 1266 } 1267 1268 1269 /** 1270 * Part of an input range as defined by 1271 * $(REF isInputRange, std,range,primitives). 1272 * 1273 * Throws: 1274 * $(LREF CSVException) When a quote is found in an unquoted field, 1275 * data continues after a closing quote, the quoted field was not 1276 * closed before data was empty, a conversion failed, or when the 1277 * row's length does not match the previous length. 1278 */ 1279 void popFront() 1280 { 1281 static if (ErrorLevel == Malformed.throwException) 1282 import std.format : format; 1283 // Skip last of record when header is depleted. 1284 if (_popCount.ptr && _popCount.empty) 1285 while (!recordEnd()) 1286 { 1287 prime(1); 1288 } 1289 1290 if (recordEnd()) 1291 { 1292 _empty = true; 1293 static if (ErrorLevel == Malformed.throwException) 1294 if (_input.rowLength != 0) 1295 if (_input.col != _input.rowLength) 1296 throw new CSVException( 1297 format("Row %s's length %s does not match "~ 1298 "previous length of %s.", _input.row, 1299 _input.col, _input.rowLength)); 1300 return; 1301 } 1302 else 1303 { 1304 static if (ErrorLevel == Malformed.throwException) 1305 if (_input.rowLength != 0) 1306 if (_input.col > _input.rowLength) 1307 throw new CSVException( 1308 format("Row %s's length %s does not match "~ 1309 "previous length of %s.", _input.row, 1310 _input.col, _input.rowLength)); 1311 } 1312 1313 // Separator is left on the end of input from the last call. 1314 // This cannot be moved to after the call to csvNextToken as 1315 // there may be an empty record after it. 1316 if (_input.range.front == _separator) 1317 _input.range.popFront(); 1318 1319 _front.shrinkTo(0); 1320 1321 prime(); 1322 } 1323 1324 /* 1325 * Handles moving to the next skipNum token. 1326 */ 1327 private void prime(size_t skipNum) 1328 { 1329 foreach (i; 0 .. skipNum) 1330 { 1331 _input.col++; 1332 _front.shrinkTo(0); 1333 if (_input.range.front == _separator) 1334 _input.range.popFront(); 1335 1336 try 1337 csvNextToken!(Range, ErrorLevel, Separator) 1338 (_input.range, _front, _separator, _quote,false); 1339 catch (IncompleteCellException ice) 1340 { 1341 ice.row = _input.row; 1342 ice.col = _input.col; 1343 ice.partialData = _front.data.idup; 1344 throw ice; 1345 } 1346 catch (ConvException e) 1347 { 1348 throw new CSVException(e.msg, _input.row, _input.col, e); 1349 } 1350 } 1351 } 1352 1353 private void prime() 1354 { 1355 try 1356 { 1357 _input.col++; 1358 csvNextToken!(Range, ErrorLevel, Separator) 1359 (_input.range, _front, _separator, _quote,false); 1360 } 1361 catch (IncompleteCellException ice) 1362 { 1363 ice.row = _input.row; 1364 ice.col = _input.col; 1365 ice.partialData = _front.data.idup; 1366 throw ice; 1367 } 1368 1369 auto skipNum = _popCount.empty ? 0 : _popCount.front; 1370 if (!_popCount.empty) 1371 _popCount.popFront(); 1372 1373 if (skipNum == size_t.max) 1374 { 1375 while (!recordEnd()) 1376 prime(1); 1377 _empty = true; 1378 return; 1379 } 1380 1381 if (skipNum) 1382 prime(skipNum); 1383 1384 auto data = _front.data; 1385 static if (!isSomeString!Contents) skipWS(data); 1386 try curContentsoken = to!Contents(data); 1387 catch (ConvException e) 1388 { 1389 throw new CSVException(e.msg, _input.row, _input.col, e); 1390 } 1391 } 1392 } 1393 1394 /** 1395 * Lower level control over parsing CSV 1396 * 1397 * This function consumes the input. After each call the input will 1398 * start with either a delimiter or record break (\n, \r\n, \r) which 1399 * must be removed for subsequent calls. 1400 * 1401 * Params: 1402 * input = Any CSV input 1403 * ans = The first field in the input 1404 * sep = The character to represent a comma in the specification 1405 * quote = The character to represent a quote in the specification 1406 * startQuoted = Whether the input should be considered to already be in 1407 * quotes 1408 * 1409 * Throws: 1410 * $(LREF IncompleteCellException) When a quote is found in an unquoted 1411 * field, data continues after a closing quote, or the quoted field was 1412 * not closed before data was empty. 1413 */ 1414 void csvNextToken(Range, Malformed ErrorLevel = Malformed.throwException, 1415 Separator, Output) 1416 (ref Range input, ref Output ans, 1417 Separator sep, Separator quote, 1418 bool startQuoted = false) 1419 if (isSomeChar!Separator && isInputRange!Range 1420 && is(immutable ElementType!Range == immutable dchar) 1421 && isOutputRange!(Output, dchar)) 1422 { 1423 bool quoted = startQuoted; 1424 bool escQuote; 1425 if (input.empty) 1426 return; 1427 1428 if (input.front == '\n') 1429 return; 1430 if (input.front == '\r') 1431 return; 1432 1433 if (input.front == quote) 1434 { 1435 quoted = true; 1436 input.popFront(); 1437 } 1438 1439 while (!input.empty) 1440 { 1441 assert(!(quoted && escQuote), 1442 "Invalid quotation state in csvNextToken"); 1443 if (!quoted) 1444 { 1445 // When not quoted the token ends at sep 1446 if (input.front == sep) 1447 break; 1448 if (input.front == '\r') 1449 break; 1450 if (input.front == '\n') 1451 break; 1452 } 1453 if (!quoted && !escQuote) 1454 { 1455 if (input.front == quote) 1456 { 1457 // Not quoted, but quote found 1458 static if (ErrorLevel == Malformed.throwException) 1459 throw new IncompleteCellException( 1460 "Quote located in unquoted token"); 1461 else static if (ErrorLevel == Malformed.ignore) 1462 ans.put(quote); 1463 } 1464 else 1465 { 1466 // Not quoted, non-quote character 1467 ans.put(input.front); 1468 } 1469 } 1470 else 1471 { 1472 if (input.front == quote) 1473 { 1474 // Quoted, quote found 1475 // By turning off quoted and turning on escQuote 1476 // I can tell when to add a quote to the string 1477 // escQuote is turned to false when it escapes a 1478 // quote or is followed by a non-quote (see outside else). 1479 // They are mutually exclusive, but provide different 1480 // information. 1481 if (escQuote) 1482 { 1483 escQuote = false; 1484 quoted = true; 1485 ans.put(quote); 1486 } else 1487 { 1488 escQuote = true; 1489 quoted = false; 1490 } 1491 } 1492 else 1493 { 1494 // Quoted, non-quote character 1495 if (escQuote) 1496 { 1497 static if (ErrorLevel == Malformed.throwException) 1498 throw new IncompleteCellException( 1499 "Content continues after end quote, " ~ 1500 "or needs to be escaped."); 1501 else static if (ErrorLevel == Malformed.ignore) 1502 break; 1503 } 1504 ans.put(input.front); 1505 } 1506 } 1507 input.popFront(); 1508 } 1509 1510 static if (ErrorLevel == Malformed.throwException) 1511 if (quoted && (input.empty || input.front == '\n' || input.front == '\r')) 1512 throw new IncompleteCellException( 1513 "Data continues on future lines or trailing quote"); 1514 1515 } 1516 1517 /// 1518 @safe unittest 1519 { 1520 import std.array : appender; 1521 import std.range.primitives : popFront; 1522 1523 string str = "65,63\n123,3673"; 1524 1525 auto a = appender!(char[])(); 1526 1527 csvNextToken(str,a,',','"'); 1528 assert(a.data == "65"); 1529 assert(str == ",63\n123,3673"); 1530 1531 str.popFront(); 1532 a.shrinkTo(0); 1533 csvNextToken(str,a,',','"'); 1534 assert(a.data == "63"); 1535 assert(str == "\n123,3673"); 1536 1537 str.popFront(); 1538 a.shrinkTo(0); 1539 csvNextToken(str,a,',','"'); 1540 assert(a.data == "123"); 1541 assert(str == ",3673"); 1542 } 1543 1544 // Test csvNextToken on simplest form and correct format. 1545 @safe pure unittest 1546 { 1547 import std.array; 1548 1549 string str = "\U00010143Hello,65,63.63\nWorld,123,3673.562"; 1550 1551 auto a = appender!(dchar[])(); 1552 csvNextToken!string(str,a,',','"'); 1553 assert(a.data == "\U00010143Hello"); 1554 assert(str == ",65,63.63\nWorld,123,3673.562"); 1555 1556 str.popFront(); 1557 a.shrinkTo(0); 1558 csvNextToken(str,a,',','"'); 1559 assert(a.data == "65"); 1560 assert(str == ",63.63\nWorld,123,3673.562"); 1561 1562 str.popFront(); 1563 a.shrinkTo(0); 1564 csvNextToken(str,a,',','"'); 1565 assert(a.data == "63.63"); 1566 assert(str == "\nWorld,123,3673.562"); 1567 1568 str.popFront(); 1569 a.shrinkTo(0); 1570 csvNextToken(str,a,',','"'); 1571 assert(a.data == "World"); 1572 assert(str == ",123,3673.562"); 1573 1574 str.popFront(); 1575 a.shrinkTo(0); 1576 csvNextToken(str,a,',','"'); 1577 assert(a.data == "123"); 1578 assert(str == ",3673.562"); 1579 1580 str.popFront(); 1581 a.shrinkTo(0); 1582 csvNextToken(str,a,',','"'); 1583 assert(a.data == "3673.562"); 1584 assert(str == ""); 1585 } 1586 1587 // Test quoted tokens 1588 @safe pure unittest 1589 { 1590 import std.array; 1591 1592 string str = `one,two,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix"; 1593 1594 auto a = appender!(dchar[])(); 1595 csvNextToken!string(str,a,',','"'); 1596 assert(a.data == "one"); 1597 assert(str == `,two,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix"); 1598 1599 str.popFront(); 1600 a.shrinkTo(0); 1601 csvNextToken(str,a,',','"'); 1602 assert(a.data == "two"); 1603 assert(str == `,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix"); 1604 1605 str.popFront(); 1606 a.shrinkTo(0); 1607 csvNextToken(str,a,',','"'); 1608 assert(a.data == "three \"quoted\""); 1609 assert(str == `,"",` ~ "\"five\nnew line\"\nsix"); 1610 1611 str.popFront(); 1612 a.shrinkTo(0); 1613 csvNextToken(str,a,',','"'); 1614 assert(a.data == ""); 1615 assert(str == ",\"five\nnew line\"\nsix"); 1616 1617 str.popFront(); 1618 a.shrinkTo(0); 1619 csvNextToken(str,a,',','"'); 1620 assert(a.data == "five\nnew line"); 1621 assert(str == "\nsix"); 1622 1623 str.popFront(); 1624 a.shrinkTo(0); 1625 csvNextToken(str,a,',','"'); 1626 assert(a.data == "six"); 1627 assert(str == ""); 1628 } 1629 1630 // Test empty data is pulled at end of record. 1631 @safe pure unittest 1632 { 1633 import std.array; 1634 1635 string str = "one,"; 1636 auto a = appender!(dchar[])(); 1637 csvNextToken(str,a,',','"'); 1638 assert(a.data == "one"); 1639 assert(str == ","); 1640 1641 a.shrinkTo(0); 1642 csvNextToken(str,a,',','"'); 1643 assert(a.data == ""); 1644 } 1645 1646 // Test exceptions 1647 @safe pure unittest 1648 { 1649 import std.array; 1650 1651 string str = "\"one\nnew line"; 1652 1653 typeof(appender!(dchar[])()) a; 1654 try 1655 { 1656 a = appender!(dchar[])(); 1657 csvNextToken(str,a,',','"'); 1658 assert(0); 1659 } 1660 catch (IncompleteCellException ice) 1661 { 1662 assert(a.data == "one\nnew line"); 1663 assert(str == ""); 1664 } 1665 1666 str = "Hello world\""; 1667 1668 try 1669 { 1670 a = appender!(dchar[])(); 1671 csvNextToken(str,a,',','"'); 1672 assert(0); 1673 } 1674 catch (IncompleteCellException ice) 1675 { 1676 assert(a.data == "Hello world"); 1677 assert(str == "\""); 1678 } 1679 1680 str = "one, two \"quoted\" end"; 1681 1682 a = appender!(dchar[])(); 1683 csvNextToken!(string,Malformed.ignore)(str,a,',','"'); 1684 assert(a.data == "one"); 1685 str.popFront(); 1686 a.shrinkTo(0); 1687 csvNextToken!(string,Malformed.ignore)(str,a,',','"'); 1688 assert(a.data == " two \"quoted\" end"); 1689 } 1690 1691 // Test modifying token delimiter 1692 @safe pure unittest 1693 { 1694 import std.array; 1695 1696 string str = `one|two|/three "quoted"/|//`; 1697 1698 auto a = appender!(dchar[])(); 1699 csvNextToken(str,a, '|','/'); 1700 assert(a.data == "one"d); 1701 assert(str == `|two|/three "quoted"/|//`); 1702 1703 str.popFront(); 1704 a.shrinkTo(0); 1705 csvNextToken(str,a, '|','/'); 1706 assert(a.data == "two"d); 1707 assert(str == `|/three "quoted"/|//`); 1708 1709 str.popFront(); 1710 a.shrinkTo(0); 1711 csvNextToken(str,a, '|','/'); 1712 assert(a.data == `three "quoted"`); 1713 assert(str == `|//`); 1714 1715 str.popFront(); 1716 a.shrinkTo(0); 1717 csvNextToken(str,a, '|','/'); 1718 assert(a.data == ""d); 1719 } 1720 1721 // https://issues.dlang.org/show_bug.cgi?id=8908 1722 @safe pure unittest 1723 { 1724 string csv = ` 1.0, 2.0, 3.0 1725 4.0, 5.0, 6.0`; 1726 1727 static struct Data { real a, b, c; } 1728 size_t i = 0; 1729 foreach (data; csvReader!Data(csv)) with (data) 1730 { 1731 int[] row = [cast(int) a, cast(int) b, cast(int) c]; 1732 if (i == 0) 1733 assert(row == [1, 2, 3]); 1734 else 1735 assert(row == [4, 5, 6]); 1736 ++i; 1737 } 1738 1739 i = 0; 1740 foreach (data; csvReader!real(csv)) 1741 { 1742 auto a = data.front; data.popFront(); 1743 auto b = data.front; data.popFront(); 1744 auto c = data.front; 1745 int[] row = [cast(int) a, cast(int) b, cast(int) c]; 1746 if (i == 0) 1747 assert(row == [1, 2, 3]); 1748 else 1749 assert(row == [4, 5, 6]); 1750 ++i; 1751 } 1752 }