1 // Written in the D programming language.
2 
3 /**
4  * Compress/decompress data using the $(LINK2 http://www._zlib.net, zlib library).
5  *
6  * References:
7  *  $(LINK2 http://en.wikipedia.org/wiki/Zlib, Wikipedia)
8  *
9  * Macros:
10  *  WIKI = Phobos/StdZlib
11  *
12  * Copyright: Copyright Digital Mars 2000 - 2011.
13  * License:   <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
14  * Authors:   $(WEB digitalmars.com, Walter Bright)
15  * Source:    $(PHOBOSSRC std/_zlib.d)
16  */
17 /*          Copyright Digital Mars 2000 - 2011.
18  * Distributed under the Boost Software License, Version 1.0.
19  *    (See accompanying file LICENSE_1_0.txt or copy at
20  *          http://www.boost.org/LICENSE_1_0.txt)
21  */
22 module arsd.zlib;
23 
24 //debug=zlib;       // uncomment to turn on debugging printf's
25 
26 private import etc.c.zlib, std.conv;
27 
28 // Values for 'mode'
29 
30 enum
31 {
32     Z_NO_FLUSH      = 0,
33     Z_SYNC_FLUSH    = 2,
34     Z_FULL_FLUSH    = 3,
35     Z_FINISH        = 4,
36 }
37 
38 /*************************************
39  * Errors throw a ZlibException.
40  */
41 
42 class ZlibException : Exception
43 {
44     this(int errnum)
45     {   string msg;
46 
47         switch (errnum)
48         {
49             case Z_STREAM_END:      msg = "stream end"; break;
50             case Z_NEED_DICT:       msg = "need dict"; break;
51             case Z_ERRNO:           msg = "errno"; break;
52             case Z_STREAM_ERROR:    msg = "stream error"; break;
53             case Z_DATA_ERROR:      msg = "data error"; break;
54             case Z_MEM_ERROR:       msg = "mem error"; break;
55             case Z_BUF_ERROR:       msg = "buf error"; break;
56             case Z_VERSION_ERROR:   msg = "version error"; break;
57             default:                msg = "unknown error";  break;
58         }
59         super(msg);
60     }
61 }
62 
63 /**************************************************
64  * Compute the Adler32 checksum of the data in buf[]. adler is the starting
65  * value when computing a cumulative checksum.
66  */
67 
68 uint adler32(uint adler, const(void)[] buf)
69 {
70     return etc.c.zlib.adler32(adler, cast(ubyte *)buf.ptr,
71             to!uint(buf.length));
72 }
73 
74 unittest
75 {
76     static ubyte[] data = [1,2,3,4,5,6,7,8,9,10];
77 
78     uint adler;
79 
80     debug(zlib) printf("D.zlib.adler32.unittest\n");
81     adler = adler32(0u, cast(void[])data);
82     debug(zlib) printf("adler = %x\n", adler);
83     assert(adler == 0xdc0037);
84 }
85 
86 /*********************************
87  * Compute the CRC32 checksum of the data in buf[]. crc is the starting value
88  * when computing a cumulative checksum.
89  */
90 
91 uint crc32(uint crc, const(void)[] buf)
92 {
93     return etc.c.zlib.crc32(crc, cast(ubyte *)buf.ptr, to!uint(buf.length));
94 }
95 
96 unittest
97 {
98     static ubyte[] data = [1,2,3,4,5,6,7,8,9,10];
99 
100     uint crc;
101 
102     debug(zlib) printf("D.zlib.crc32.unittest\n");
103     crc = crc32(0u, cast(void[])data);
104     debug(zlib) printf("crc = %x\n", crc);
105     assert(crc == 0x2520577b);
106 }
107 
108 /*********************************************
109  * Compresses the data in srcbuf[] using compression _level level.
110  * The default value
111  * for level is 6, legal values are 1..9, with 1 being the least compression
112  * and 9 being the most.
113  * Returns the compressed data.
114  */
115 
116 const(void)[] compress(const(void)[] srcbuf, int level)
117 in
118 {
119     assert(-1 <= level && level <= 9);
120 }
121 body
122 {
123     auto destlen = srcbuf.length + ((srcbuf.length + 1023) / 1024) + 12;
124     auto destbuf = new ubyte[destlen];
125     auto err = etc.c.zlib.compress2(destbuf.ptr, &destlen, cast(ubyte *)srcbuf.ptr, srcbuf.length, level);
126     if (err)
127     {   delete destbuf;
128         throw new ZlibException(err);
129     }
130 
131     destbuf.length = destlen;
132     return destbuf;
133 }
134 
135 /*********************************************
136  * ditto
137  */
138 
139 const(void)[] compress(const(void)[] buf)
140 {
141     return compress(buf, Z_DEFAULT_COMPRESSION);
142 }
143 
144 /*********************************************
145  * Decompresses the data in srcbuf[].
146  * Params: destlen = size of the uncompressed data.
147  * It need not be accurate, but the decompression will be faster if the exact
148  * size is supplied.
149  * Returns: the decompressed data.
150  */
151 
152 void[] uncompress(void[] srcbuf, size_t destlen = 0u, int winbits = 15)
153 {
154     int err;
155     ubyte[] destbuf;
156 
157     if (!destlen)
158         destlen = srcbuf.length * 2 + 1;
159 
160     while (1)
161     {
162         etc.c.zlib.z_stream zs;
163 
164         destbuf = new ubyte[destlen];
165 
166         zs.next_in = cast(ubyte*) srcbuf;
167         zs.avail_in = to!uint(srcbuf.length);
168 
169         zs.next_out = destbuf.ptr;
170         zs.avail_out = cast(typeof(zs.avail_out))destlen;
171 
172         err = etc.c.zlib.inflateInit2(&zs, winbits);
173         if (err)
174         {   delete destbuf;
175             throw new ZlibException(err);
176         }
177         err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH);
178         switch (err)
179         {
180             case Z_OK:
181                 etc.c.zlib.inflateEnd(&zs);
182                 destlen = destbuf.length * 2;
183                 continue;
184 
185             case Z_STREAM_END:
186                 destbuf.length = zs.total_out;
187                 err = etc.c.zlib.inflateEnd(&zs);
188                 if (err != Z_OK)
189                     goto Lerr;
190                 return destbuf;
191 
192             default:
193                 etc.c.zlib.inflateEnd(&zs);
194             Lerr:
195                 delete destbuf;
196                 throw new ZlibException(err);
197         }
198     }
199     assert(0);
200 }
201 
202 unittest
203 {
204     ubyte[] src = cast(ubyte[])
205 "the quick brown fox jumps over the lazy dog\r
206 the quick brown fox jumps over the lazy dog\r
207 ";
208     ubyte[] dst;
209     ubyte[] result;
210 
211     //arrayPrint(src);
212     dst = cast(ubyte[])compress(cast(void[])src);
213     //arrayPrint(dst);
214     result = cast(ubyte[])uncompress(cast(void[])dst);
215     //arrayPrint(result);
216     assert(result == src);
217 }
218 
219 /+
220 void arrayPrint(ubyte[] array)
221 {
222     //printf("array %p,%d\n", cast(void*)array, array.length);
223     for (size_t i = 0; i < array.length; i++)
224     {
225         printf("%02x ", array[i]);
226         if (((i + 1) & 15) == 0)
227             printf("\n");
228     }
229     printf("\n\n");
230 }
231 +/
232 
233 /*********************************************
234  * Used when the data to be compressed is not all in one buffer.
235  */
236 
237 class Compress
238 {
239   private:
240     z_stream zs;
241     int level = Z_DEFAULT_COMPRESSION;
242     int inited;
243 
244     void error(int err)
245     {
246         if (inited)
247         {   deflateEnd(&zs);
248             inited = 0;
249         }
250         throw new ZlibException(err);
251     }
252 
253   public:
254 
255     /**
256      * Construct. level is the same as for D.zlib.compress().
257      */
258     this(int level, bool wantGzip = false)
259     in
260     {
261         assert(1 <= level && level <= 9);
262     }
263     body
264     {
265         this.level = level;
266 	this.gzip = wantGzip;
267     }
268 
269     /// ditto
270     this(bool wantGzip = false)
271     {
272         gzip = wantGzip;
273     }
274 
275     ~this()
276     {   int err;
277 
278         if (inited)
279         {
280             inited = 0;
281             err = deflateEnd(&zs);
282             if (err)
283                 error(err);
284         }
285     }
286 
287     immutable bool gzip;
288 
289     /**
290      * Compress the data in buf and return the compressed data.
291      * The buffers
292      * returned from successive calls to this should be concatenated together.
293      */
294     const(void)[] compress(const(void)[] buf)
295     {   int err;
296         ubyte[] destbuf;
297 
298         if (buf.length == 0)
299             return null;
300 
301         if (!inited)
302         {
303             err = deflateInit2(&zs, level, Z_DEFLATED, 15 + (gzip ? 16 : 0), 8, Z_DEFAULT_STRATEGY);
304             if (err)
305                 error(err);
306             inited = 1;
307         }
308 
309         destbuf = new ubyte[zs.avail_in + buf.length];
310         zs.next_out = destbuf.ptr;
311         zs.avail_out = to!uint(destbuf.length);
312 
313         if (zs.avail_in)
314             buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf;
315 
316         zs.next_in = cast(ubyte*) buf.ptr;
317         zs.avail_in = to!uint(buf.length);
318 
319         err = deflate(&zs, Z_NO_FLUSH);
320         if (err != Z_STREAM_END && err != Z_OK)
321         {   delete destbuf;
322             error(err);
323         }
324         destbuf.length = destbuf.length - zs.avail_out;
325         return destbuf;
326     }
327 
328     /***
329      * Compress and return any remaining data.
330      * The returned data should be appended to that returned by compress().
331      * Params:
332      *  mode = one of the following:
333      *          $(DL
334                     $(DT Z_SYNC_FLUSH )
335                     $(DD Syncs up flushing to the next byte boundary.
336                         Used when more data is to be compressed later on.)
337                     $(DT Z_FULL_FLUSH )
338                     $(DD Syncs up flushing to the next byte boundary.
339                         Used when more data is to be compressed later on,
340                         and the decompressor needs to be restartable at this
341                         point.)
342                     $(DT Z_FINISH)
343                     $(DD (default) Used when finished compressing the data. )
344                 )
345      */
346     void[] flush(int mode = Z_FINISH)
347     in
348     {
349         assert(mode == Z_FINISH || mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH);
350     }
351     body
352     {
353         ubyte[] destbuf;
354         ubyte[512] tmpbuf = void;
355         int err;
356 
357         if (!inited)
358             return null;
359 
360         /* may be  zs.avail_out+<some constant>
361          * zs.avail_out is set nonzero by deflate in previous compress()
362          */
363         //tmpbuf = new void[zs.avail_out];
364         zs.next_out = tmpbuf.ptr;
365         zs.avail_out = tmpbuf.length;
366 
367         while( (err = deflate(&zs, mode)) != Z_STREAM_END)
368         {
369             if (err == Z_OK)
370             {
371                 if (zs.avail_out != 0 && mode != Z_FINISH)
372                     break;
373                 else if(zs.avail_out == 0)
374                 {
375                     destbuf ~= tmpbuf;
376                     zs.next_out = tmpbuf.ptr;
377                     zs.avail_out = tmpbuf.length;
378                     continue;
379                 }
380                 err = Z_BUF_ERROR;
381             }
382             delete destbuf;
383             error(err);
384         }
385         destbuf ~= tmpbuf[0 .. (tmpbuf.length - zs.avail_out)];
386 
387         if (mode == Z_FINISH)
388         {
389             err = deflateEnd(&zs);
390             inited = 0;
391             if (err)
392                 error(err);
393         }
394         return destbuf;
395     }
396 }
397 
398 /******
399  * Used when the data to be decompressed is not all in one buffer.
400  */
401 
402 class UnCompress
403 {
404   private:
405     z_stream zs;
406     int inited;
407     int done;
408     size_t destbufsize;
409 
410     void error(int err)
411     {
412         if (inited)
413         {   inflateEnd(&zs);
414             inited = 0;
415         }
416         throw new ZlibException(err);
417     }
418 
419   public:
420 
421     /**
422      * Construct. destbufsize is the same as for D.zlib.uncompress().
423      */
424     this(uint destbufsize)
425     {
426         this.destbufsize = destbufsize;
427     }
428 
429     /** ditto */
430     this()
431     {
432     }
433 
434     ~this()
435     {   int err;
436 
437         if (inited)
438         {
439             inited = 0;
440             err = inflateEnd(&zs);
441             if (err)
442                 error(err);
443         }
444         done = 1;
445     }
446 
447     /**
448      * Decompress the data in buf and return the decompressed data.
449      * The buffers returned from successive calls to this should be concatenated
450      * together.
451      */
452     const(void)[] uncompress(const(void)[] buf)
453     in
454     {
455         assert(!done);
456     }
457     body
458     {   int err;
459         ubyte[] destbuf;
460 
461         if (buf.length == 0)
462             return null;
463 
464         if (!inited)
465         {
466             err = inflateInit(&zs);
467             if (err)
468                 error(err);
469             inited = 1;
470         }
471 
472         if (!destbufsize)
473             destbufsize = to!uint(buf.length) * 2;
474         destbuf = new ubyte[zs.avail_in * 2 + destbufsize];
475         zs.next_out = destbuf.ptr;
476         zs.avail_out = to!uint(destbuf.length);
477 
478         if (zs.avail_in)
479             buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf;
480 
481         zs.next_in = cast(ubyte*) buf;
482         zs.avail_in = to!uint(buf.length);
483 
484         err = inflate(&zs, Z_NO_FLUSH);
485         if (err != Z_STREAM_END && err != Z_OK)
486         {   delete destbuf;
487             error(err);
488         }
489         destbuf.length = destbuf.length - zs.avail_out;
490         return destbuf;
491     }
492 
493     /**
494      * Decompress and return any remaining data.
495      * The returned data should be appended to that returned by uncompress().
496      * The UnCompress object cannot be used further.
497      */
498     void[] flush()
499     in
500     {
501         assert(!done);
502     }
503     out
504     {
505         assert(done);
506     }
507     body
508     {
509         ubyte[] extra;
510         ubyte[] destbuf;
511         int err;
512 
513         done = 1;
514         if (!inited)
515             return null;
516 
517       L1:
518         destbuf = new ubyte[zs.avail_in * 2 + 100];
519         zs.next_out = destbuf.ptr;
520         zs.avail_out = to!uint(destbuf.length);
521 
522         err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH);
523         if (err == Z_OK && zs.avail_out == 0)
524         {
525             extra ~= destbuf;
526             goto L1;
527         }
528         if (err != Z_STREAM_END)
529         {
530             delete destbuf;
531             if (err == Z_OK)
532                 err = Z_BUF_ERROR;
533             error(err);
534         }
535         destbuf = destbuf.ptr[0 .. zs.next_out - destbuf.ptr];
536         err = etc.c.zlib.inflateEnd(&zs);
537         inited = 0;
538         if (err)
539             error(err);
540         if (extra.length)
541             destbuf = extra ~ destbuf;
542         return destbuf;
543     }
544 }
545 
546 /* ========================== unittest ========================= */
547 
548 private import std.stdio;
549 private import std.random;
550 
551 unittest // by Dave
552 {
553     debug(zlib) printf("std.zlib.unittest\n");
554 
555     bool CompressThenUncompress (ubyte[] src)
556     {
557       try {
558         ubyte[] dst = cast(ubyte[])std.zlib.compress(cast(void[])src);
559         double ratio = (dst.length / cast(double)src.length);
560         debug(zlib) writef("src.length:  ", src.length, ", dst: ", dst.length, ", Ratio = ", ratio);
561         ubyte[] uncompressedBuf;
562         uncompressedBuf = cast(ubyte[])std.zlib.uncompress(cast(void[])dst);
563         assert(src.length == uncompressedBuf.length);
564         assert(src == uncompressedBuf);
565       }
566       catch {
567         debug(zlib) writefln(" ... Exception thrown when src.length = ", src.length, ".");
568         return false;
569       }
570       return true;
571     }
572 
573 
574     // smallish buffers
575     for(int idx = 0; idx < 25; idx++) {
576         char[] buf = new char[uniform(0, 100)];
577 
578         // Alternate between more & less compressible
579         foreach(ref char c; buf)
580             c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 2)));
581 
582         if(CompressThenUncompress(cast(ubyte[])buf)) {
583             debug(zlib) printf("; Success.\n");
584         } else {
585             return;
586         }
587     }
588 
589     // larger buffers
590     for(int idx = 0; idx < 25; idx++) {
591         char[] buf = new char[uniform(0, 1000/*0000*/)];
592 
593         // Alternate between more & less compressible
594         foreach(ref char c; buf)
595             c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 10)));
596 
597         if(CompressThenUncompress(cast(ubyte[])buf)) {
598             debug(zlib) printf("; Success.\n");
599         } else {
600             return;
601         }
602     }
603 
604     debug(zlib) printf("PASSED std.zlib.unittest\n");
605 }
606 
607 
608 unittest // by Artem Rebrov
609 {
610     Compress cmp = new Compress;
611     UnCompress decmp = new UnCompress;
612 
613     const(void)[] input;
614     input = "tesatdffadf";
615 
616     const(void)[] buf = cmp.compress(input);
617     buf ~= cmp.flush();
618     const(void)[] output = decmp.uncompress(buf);
619 
620     //writefln("input = '%s'", cast(char[])input);
621     //writefln("output = '%s'", cast(char[])output);
622     assert( output[] == input[] );
623 }
624 
Suggestion Box / Bug Report