1 // Written in the D programming language. 2 3 /** 4 * Compress/decompress data using the $(LINK2 http://www._zlib.net, zlib library). 5 * 6 * References: 7 * $(LINK2 http://en.wikipedia.org/wiki/Zlib, Wikipedia) 8 * 9 * Macros: 10 * WIKI = Phobos/StdZlib 11 * 12 * Copyright: Copyright Digital Mars 2000 - 2011. 13 * License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>. 14 * Authors: $(WEB digitalmars.com, Walter Bright) 15 * Source: $(PHOBOSSRC std/_zlib.d) 16 */ 17 /* Copyright Digital Mars 2000 - 2011. 18 * Distributed under the Boost Software License, Version 1.0. 19 * (See accompanying file LICENSE_1_0.txt or copy at 20 * http://www.boost.org/LICENSE_1_0.txt) 21 */ 22 module arsd.zlib; 23 24 //debug=zlib; // uncomment to turn on debugging printf's 25 26 private import etc.c.zlib, std.conv; 27 28 // Values for 'mode' 29 30 enum 31 { 32 Z_NO_FLUSH = 0, 33 Z_SYNC_FLUSH = 2, 34 Z_FULL_FLUSH = 3, 35 Z_FINISH = 4, 36 } 37 38 /************************************* 39 * Errors throw a ZlibException. 40 */ 41 42 class ZlibException : Exception 43 { 44 this(int errnum) 45 { string msg; 46 47 switch (errnum) 48 { 49 case Z_STREAM_END: msg = "stream end"; break; 50 case Z_NEED_DICT: msg = "need dict"; break; 51 case Z_ERRNO: msg = "errno"; break; 52 case Z_STREAM_ERROR: msg = "stream error"; break; 53 case Z_DATA_ERROR: msg = "data error"; break; 54 case Z_MEM_ERROR: msg = "mem error"; break; 55 case Z_BUF_ERROR: msg = "buf error"; break; 56 case Z_VERSION_ERROR: msg = "version error"; break; 57 default: msg = "unknown error"; break; 58 } 59 super(msg); 60 } 61 } 62 63 /************************************************** 64 * Compute the Adler32 checksum of the data in buf[]. adler is the starting 65 * value when computing a cumulative checksum. 66 */ 67 68 uint adler32(uint adler, const(void)[] buf) 69 { 70 return etc.c.zlib.adler32(adler, cast(ubyte *)buf.ptr, 71 to!uint(buf.length)); 72 } 73 74 unittest 75 { 76 static ubyte[] data = [1,2,3,4,5,6,7,8,9,10]; 77 78 uint adler; 79 80 debug(zlib) printf("D.zlib.adler32.unittest\n"); 81 adler = adler32(0u, cast(void[])data); 82 debug(zlib) printf("adler = %x\n", adler); 83 assert(adler == 0xdc0037); 84 } 85 86 /********************************* 87 * Compute the CRC32 checksum of the data in buf[]. crc is the starting value 88 * when computing a cumulative checksum. 89 */ 90 91 uint crc32(uint crc, const(void)[] buf) 92 { 93 return etc.c.zlib.crc32(crc, cast(ubyte *)buf.ptr, to!uint(buf.length)); 94 } 95 96 unittest 97 { 98 static ubyte[] data = [1,2,3,4,5,6,7,8,9,10]; 99 100 uint crc; 101 102 debug(zlib) printf("D.zlib.crc32.unittest\n"); 103 crc = crc32(0u, cast(void[])data); 104 debug(zlib) printf("crc = %x\n", crc); 105 assert(crc == 0x2520577b); 106 } 107 108 /********************************************* 109 * Compresses the data in srcbuf[] using compression _level level. 110 * The default value 111 * for level is 6, legal values are 1..9, with 1 being the least compression 112 * and 9 being the most. 113 * Returns the compressed data. 114 */ 115 116 const(void)[] compress(const(void)[] srcbuf, int level) 117 in 118 { 119 assert(-1 <= level && level <= 9); 120 } 121 body 122 { 123 auto destlen = srcbuf.length + ((srcbuf.length + 1023) / 1024) + 12; 124 auto destbuf = new ubyte[destlen]; 125 auto err = etc.c.zlib.compress2(destbuf.ptr, &destlen, cast(ubyte *)srcbuf.ptr, srcbuf.length, level); 126 if (err) 127 { delete destbuf; 128 throw new ZlibException(err); 129 } 130 131 destbuf.length = destlen; 132 return destbuf; 133 } 134 135 /********************************************* 136 * ditto 137 */ 138 139 const(void)[] compress(const(void)[] buf) 140 { 141 return compress(buf, Z_DEFAULT_COMPRESSION); 142 } 143 144 /********************************************* 145 * Decompresses the data in srcbuf[]. 146 * Params: destlen = size of the uncompressed data. 147 * It need not be accurate, but the decompression will be faster if the exact 148 * size is supplied. 149 * Returns: the decompressed data. 150 */ 151 152 void[] uncompress(void[] srcbuf, size_t destlen = 0u, int winbits = 15) 153 { 154 int err; 155 ubyte[] destbuf; 156 157 if (!destlen) 158 destlen = srcbuf.length * 2 + 1; 159 160 while (1) 161 { 162 etc.c.zlib.z_stream zs; 163 164 destbuf = new ubyte[destlen]; 165 166 zs.next_in = cast(ubyte*) srcbuf; 167 zs.avail_in = to!uint(srcbuf.length); 168 169 zs.next_out = destbuf.ptr; 170 zs.avail_out = cast(typeof(zs.avail_out))destlen; 171 172 err = etc.c.zlib.inflateInit2(&zs, winbits); 173 if (err) 174 { delete destbuf; 175 throw new ZlibException(err); 176 } 177 err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH); 178 switch (err) 179 { 180 case Z_OK: 181 etc.c.zlib.inflateEnd(&zs); 182 destlen = destbuf.length * 2; 183 continue; 184 185 case Z_STREAM_END: 186 destbuf.length = zs.total_out; 187 err = etc.c.zlib.inflateEnd(&zs); 188 if (err != Z_OK) 189 goto Lerr; 190 return destbuf; 191 192 default: 193 etc.c.zlib.inflateEnd(&zs); 194 Lerr: 195 delete destbuf; 196 throw new ZlibException(err); 197 } 198 } 199 assert(0); 200 } 201 202 unittest 203 { 204 ubyte[] src = cast(ubyte[]) 205 "the quick brown fox jumps over the lazy dog\r 206 the quick brown fox jumps over the lazy dog\r 207 "; 208 ubyte[] dst; 209 ubyte[] result; 210 211 //arrayPrint(src); 212 dst = cast(ubyte[])compress(cast(void[])src); 213 //arrayPrint(dst); 214 result = cast(ubyte[])uncompress(cast(void[])dst); 215 //arrayPrint(result); 216 assert(result == src); 217 } 218 219 /+ 220 void arrayPrint(ubyte[] array) 221 { 222 //printf("array %p,%d\n", cast(void*)array, array.length); 223 for (size_t i = 0; i < array.length; i++) 224 { 225 printf("%02x ", array[i]); 226 if (((i + 1) & 15) == 0) 227 printf("\n"); 228 } 229 printf("\n\n"); 230 } 231 +/ 232 233 /********************************************* 234 * Used when the data to be compressed is not all in one buffer. 235 */ 236 237 class Compress 238 { 239 private: 240 z_stream zs; 241 int level = Z_DEFAULT_COMPRESSION; 242 int inited; 243 244 void error(int err) 245 { 246 if (inited) 247 { deflateEnd(&zs); 248 inited = 0; 249 } 250 throw new ZlibException(err); 251 } 252 253 public: 254 255 /** 256 * Construct. level is the same as for D.zlib.compress(). 257 */ 258 this(int level, bool wantGzip = false) 259 in 260 { 261 assert(1 <= level && level <= 9); 262 } 263 body 264 { 265 this.level = level; 266 this.gzip = wantGzip; 267 } 268 269 /// ditto 270 this(bool wantGzip = false) 271 { 272 gzip = wantGzip; 273 } 274 275 ~this() 276 { int err; 277 278 if (inited) 279 { 280 inited = 0; 281 err = deflateEnd(&zs); 282 if (err) 283 error(err); 284 } 285 } 286 287 immutable bool gzip; 288 289 /** 290 * Compress the data in buf and return the compressed data. 291 * The buffers 292 * returned from successive calls to this should be concatenated together. 293 */ 294 const(void)[] compress(const(void)[] buf) 295 { int err; 296 ubyte[] destbuf; 297 298 if (buf.length == 0) 299 return null; 300 301 if (!inited) 302 { 303 err = deflateInit2(&zs, level, Z_DEFLATED, 15 + (gzip ? 16 : 0), 8, Z_DEFAULT_STRATEGY); 304 if (err) 305 error(err); 306 inited = 1; 307 } 308 309 destbuf = new ubyte[zs.avail_in + buf.length]; 310 zs.next_out = destbuf.ptr; 311 zs.avail_out = to!uint(destbuf.length); 312 313 if (zs.avail_in) 314 buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf; 315 316 zs.next_in = cast(ubyte*) buf.ptr; 317 zs.avail_in = to!uint(buf.length); 318 319 err = deflate(&zs, Z_NO_FLUSH); 320 if (err != Z_STREAM_END && err != Z_OK) 321 { delete destbuf; 322 error(err); 323 } 324 destbuf.length = destbuf.length - zs.avail_out; 325 return destbuf; 326 } 327 328 /*** 329 * Compress and return any remaining data. 330 * The returned data should be appended to that returned by compress(). 331 * Params: 332 * mode = one of the following: 333 * $(DL 334 $(DT Z_SYNC_FLUSH ) 335 $(DD Syncs up flushing to the next byte boundary. 336 Used when more data is to be compressed later on.) 337 $(DT Z_FULL_FLUSH ) 338 $(DD Syncs up flushing to the next byte boundary. 339 Used when more data is to be compressed later on, 340 and the decompressor needs to be restartable at this 341 point.) 342 $(DT Z_FINISH) 343 $(DD (default) Used when finished compressing the data. ) 344 ) 345 */ 346 void[] flush(int mode = Z_FINISH) 347 in 348 { 349 assert(mode == Z_FINISH || mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH); 350 } 351 body 352 { 353 ubyte[] destbuf; 354 ubyte[512] tmpbuf = void; 355 int err; 356 357 if (!inited) 358 return null; 359 360 /* may be zs.avail_out+<some constant> 361 * zs.avail_out is set nonzero by deflate in previous compress() 362 */ 363 //tmpbuf = new void[zs.avail_out]; 364 zs.next_out = tmpbuf.ptr; 365 zs.avail_out = tmpbuf.length; 366 367 while( (err = deflate(&zs, mode)) != Z_STREAM_END) 368 { 369 if (err == Z_OK) 370 { 371 if (zs.avail_out != 0 && mode != Z_FINISH) 372 break; 373 else if(zs.avail_out == 0) 374 { 375 destbuf ~= tmpbuf; 376 zs.next_out = tmpbuf.ptr; 377 zs.avail_out = tmpbuf.length; 378 continue; 379 } 380 err = Z_BUF_ERROR; 381 } 382 delete destbuf; 383 error(err); 384 } 385 destbuf ~= tmpbuf[0 .. (tmpbuf.length - zs.avail_out)]; 386 387 if (mode == Z_FINISH) 388 { 389 err = deflateEnd(&zs); 390 inited = 0; 391 if (err) 392 error(err); 393 } 394 return destbuf; 395 } 396 } 397 398 /****** 399 * Used when the data to be decompressed is not all in one buffer. 400 */ 401 402 class UnCompress 403 { 404 private: 405 z_stream zs; 406 int inited; 407 int done; 408 size_t destbufsize; 409 410 void error(int err) 411 { 412 if (inited) 413 { inflateEnd(&zs); 414 inited = 0; 415 } 416 throw new ZlibException(err); 417 } 418 419 public: 420 421 /** 422 * Construct. destbufsize is the same as for D.zlib.uncompress(). 423 */ 424 this(uint destbufsize) 425 { 426 this.destbufsize = destbufsize; 427 } 428 429 /** ditto */ 430 this() 431 { 432 } 433 434 ~this() 435 { int err; 436 437 if (inited) 438 { 439 inited = 0; 440 err = inflateEnd(&zs); 441 if (err) 442 error(err); 443 } 444 done = 1; 445 } 446 447 /** 448 * Decompress the data in buf and return the decompressed data. 449 * The buffers returned from successive calls to this should be concatenated 450 * together. 451 */ 452 const(void)[] uncompress(const(void)[] buf) 453 in 454 { 455 assert(!done); 456 } 457 body 458 { int err; 459 ubyte[] destbuf; 460 461 if (buf.length == 0) 462 return null; 463 464 if (!inited) 465 { 466 err = inflateInit(&zs); 467 if (err) 468 error(err); 469 inited = 1; 470 } 471 472 if (!destbufsize) 473 destbufsize = to!uint(buf.length) * 2; 474 destbuf = new ubyte[zs.avail_in * 2 + destbufsize]; 475 zs.next_out = destbuf.ptr; 476 zs.avail_out = to!uint(destbuf.length); 477 478 if (zs.avail_in) 479 buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf; 480 481 zs.next_in = cast(ubyte*) buf; 482 zs.avail_in = to!uint(buf.length); 483 484 err = inflate(&zs, Z_NO_FLUSH); 485 if (err != Z_STREAM_END && err != Z_OK) 486 { delete destbuf; 487 error(err); 488 } 489 destbuf.length = destbuf.length - zs.avail_out; 490 return destbuf; 491 } 492 493 /** 494 * Decompress and return any remaining data. 495 * The returned data should be appended to that returned by uncompress(). 496 * The UnCompress object cannot be used further. 497 */ 498 void[] flush() 499 in 500 { 501 assert(!done); 502 } 503 out 504 { 505 assert(done); 506 } 507 body 508 { 509 ubyte[] extra; 510 ubyte[] destbuf; 511 int err; 512 513 done = 1; 514 if (!inited) 515 return null; 516 517 L1: 518 destbuf = new ubyte[zs.avail_in * 2 + 100]; 519 zs.next_out = destbuf.ptr; 520 zs.avail_out = to!uint(destbuf.length); 521 522 err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH); 523 if (err == Z_OK && zs.avail_out == 0) 524 { 525 extra ~= destbuf; 526 goto L1; 527 } 528 if (err != Z_STREAM_END) 529 { 530 delete destbuf; 531 if (err == Z_OK) 532 err = Z_BUF_ERROR; 533 error(err); 534 } 535 destbuf = destbuf.ptr[0 .. zs.next_out - destbuf.ptr]; 536 err = etc.c.zlib.inflateEnd(&zs); 537 inited = 0; 538 if (err) 539 error(err); 540 if (extra.length) 541 destbuf = extra ~ destbuf; 542 return destbuf; 543 } 544 } 545 546 /* ========================== unittest ========================= */ 547 548 private import std.stdio; 549 private import std.random; 550 551 unittest // by Dave 552 { 553 debug(zlib) printf("std.zlib.unittest\n"); 554 555 bool CompressThenUncompress (ubyte[] src) 556 { 557 try { 558 ubyte[] dst = cast(ubyte[])std.zlib.compress(cast(void[])src); 559 double ratio = (dst.length / cast(double)src.length); 560 debug(zlib) writef("src.length: ", src.length, ", dst: ", dst.length, ", Ratio = ", ratio); 561 ubyte[] uncompressedBuf; 562 uncompressedBuf = cast(ubyte[])std.zlib.uncompress(cast(void[])dst); 563 assert(src.length == uncompressedBuf.length); 564 assert(src == uncompressedBuf); 565 } 566 catch { 567 debug(zlib) writefln(" ... Exception thrown when src.length = ", src.length, "."); 568 return false; 569 } 570 return true; 571 } 572 573 574 // smallish buffers 575 for(int idx = 0; idx < 25; idx++) { 576 char[] buf = new char[uniform(0, 100)]; 577 578 // Alternate between more & less compressible 579 foreach(ref char c; buf) 580 c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 2))); 581 582 if(CompressThenUncompress(cast(ubyte[])buf)) { 583 debug(zlib) printf("; Success.\n"); 584 } else { 585 return; 586 } 587 } 588 589 // larger buffers 590 for(int idx = 0; idx < 25; idx++) { 591 char[] buf = new char[uniform(0, 1000/*0000*/)]; 592 593 // Alternate between more & less compressible 594 foreach(ref char c; buf) 595 c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 10))); 596 597 if(CompressThenUncompress(cast(ubyte[])buf)) { 598 debug(zlib) printf("; Success.\n"); 599 } else { 600 return; 601 } 602 } 603 604 debug(zlib) printf("PASSED std.zlib.unittest\n"); 605 } 606 607 608 unittest // by Artem Rebrov 609 { 610 Compress cmp = new Compress; 611 UnCompress decmp = new UnCompress; 612 613 const(void)[] input; 614 input = "tesatdffadf"; 615 616 const(void)[] buf = cmp.compress(input); 617 buf ~= cmp.flush(); 618 const(void)[] output = decmp.uncompress(buf); 619 620 //writefln("input = '%s'", cast(char[])input); 621 //writefln("output = '%s'", cast(char[])output); 622 assert( output[] == input[] ); 623 } 624