1 // Written in the D programming language.
2
3 /**
4 * Read and write memory mapped files.
5 * Copyright: Copyright The D Language Foundation 2004 - 2009.
6 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
7 * Authors: $(HTTP digitalmars.com, Walter Bright),
8 * Matthew Wilson
9 * Source: $(PHOBOSSRC std/mmfile.d)
10 *
11 * $(SCRIPT inhibitQuickIndex = 1;)
12 */
13 /* Copyright The D Language Foundation 2004 - 2009.
14 * Distributed under the Boost Software License, Version 1.0.
15 * (See accompanying file LICENSE_1_0.txt or copy at
16 * http://www.boost.org/LICENSE_1_0.txt)
17 */
18 module std.mmfile;
19
20 import core.stdc.errno;
21 import core.stdc.stdio;
22 import core.stdc.stdlib;
23 import std.conv, std.exception, std.stdio;
24 import std.file;
25 import std.path;
26 import std..string;
27
28 import std.internal.cstring;
29
30 //debug = MMFILE;
31
32 version (Windows)
33 {
34 import core.sys.windows.winbase;
35 import core.sys.windows.winnt;
36 import std.utf;
37 import std.windows.syserror;
38 }
39 else version (Posix)
40 {
41 import core.sys.posix.fcntl;
42 import core.sys.posix.sys.mman;
43 import core.sys.posix.sys.stat;
44 import core.sys.posix.unistd;
45 }
46 else
47 {
48 static assert(0);
49 }
50
51 /**
52 * MmFile objects control the memory mapped file resource.
53 */
54 class MmFile
55 {
56 /**
57 * The mode the memory mapped file is opened with.
58 */
59 enum Mode
60 {
61 read, /// Read existing file
62 readWriteNew, /// Delete existing file, write new file
63 readWrite, /// Read/Write existing file, create if not existing
64 readCopyOnWrite, /// Read/Write existing file, copy on write
65 }
66
67 /**
68 * Open memory mapped file filename for reading.
69 * File is closed when the object instance is deleted.
70 * Throws:
71 * std.file.FileException
72 */
73 this(string filename)
74 {
75 this(filename, Mode.read, 0, null);
76 }
77
78 version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
79 void* address = null, size_t window = 0)
80 {
81 // Save a copy of the File to make sure the fd stays open.
82 this.file = file;
83 this(file.fileno, mode, size, address, window);
84 }
85
86 version (linux) private this(int fildes, Mode mode, ulong size,
87 void* address, size_t window)
88 {
89 int oflag;
90 int fmode;
91
92 final switch (mode)
93 {
94 case Mode.read:
95 flags = MAP_SHARED;
96 prot = PROT_READ;
97 oflag = O_RDONLY;
98 fmode = 0;
99 break;
100
101 case Mode.readWriteNew:
102 assert(size != 0);
103 flags = MAP_SHARED;
104 prot = PROT_READ | PROT_WRITE;
105 oflag = O_CREAT | O_RDWR | O_TRUNC;
106 fmode = octal!660;
107 break;
108
109 case Mode.readWrite:
110 flags = MAP_SHARED;
111 prot = PROT_READ | PROT_WRITE;
112 oflag = O_CREAT | O_RDWR;
113 fmode = octal!660;
114 break;
115
116 case Mode.readCopyOnWrite:
117 flags = MAP_PRIVATE;
118 prot = PROT_READ | PROT_WRITE;
119 oflag = O_RDWR;
120 fmode = 0;
121 break;
122 }
123
124 fd = fildes;
125
126 // Adjust size
127 stat_t statbuf = void;
128 errnoEnforce(fstat(fd, &statbuf) == 0);
129 if (prot & PROT_WRITE && size > statbuf.st_size)
130 {
131 // Need to make the file size bytes big
132 lseek(fd, cast(off_t)(size - 1), SEEK_SET);
133 char c = 0;
134 core.sys.posix.unistd.write(fd, &c, 1);
135 }
136 else if (prot & PROT_READ && size == 0)
137 size = statbuf.st_size;
138 this.size = size;
139
140 // Map the file into memory!
141 size_t initial_map = (window && 2*window<size)
142 ? 2*window : cast(size_t) size;
143 auto p = mmap(address, initial_map, prot, flags, fd, 0);
144 if (p == MAP_FAILED)
145 {
146 errnoEnforce(false, "Could not map file into memory");
147 }
148 data = p[0 .. initial_map];
149 }
150
151 /**
152 * Open memory mapped file filename in mode.
153 * File is closed when the object instance is deleted.
154 * Params:
155 * filename = name of the file.
156 * If null, an anonymous file mapping is created.
157 * mode = access mode defined above.
158 * size = the size of the file. If 0, it is taken to be the
159 * size of the existing file.
160 * address = the preferred address to map the file to,
161 * although the system is not required to honor it.
162 * If null, the system selects the most convenient address.
163 * window = preferred block size of the amount of data to map at one time
164 * with 0 meaning map the entire file. The window size must be a
165 * multiple of the memory allocation page size.
166 * Throws:
167 * std.file.FileException
168 */
169 this(string filename, Mode mode, ulong size, void* address,
170 size_t window = 0)
171 {
172 this.filename = filename;
173 this.mMode = mode;
174 this.window = window;
175 this.address = address;
176
177 version (Windows)
178 {
179 void* p;
180 uint dwDesiredAccess2;
181 uint dwShareMode;
182 uint dwCreationDisposition;
183 uint flProtect;
184
185 final switch (mode)
186 {
187 case Mode.read:
188 dwDesiredAccess2 = GENERIC_READ;
189 dwShareMode = FILE_SHARE_READ;
190 dwCreationDisposition = OPEN_EXISTING;
191 flProtect = PAGE_READONLY;
192 dwDesiredAccess = FILE_MAP_READ;
193 break;
194
195 case Mode.readWriteNew:
196 assert(size != 0);
197 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
198 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
199 dwCreationDisposition = CREATE_ALWAYS;
200 flProtect = PAGE_READWRITE;
201 dwDesiredAccess = FILE_MAP_WRITE;
202 break;
203
204 case Mode.readWrite:
205 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
206 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
207 dwCreationDisposition = OPEN_ALWAYS;
208 flProtect = PAGE_READWRITE;
209 dwDesiredAccess = FILE_MAP_WRITE;
210 break;
211
212 case Mode.readCopyOnWrite:
213 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
214 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
215 dwCreationDisposition = OPEN_EXISTING;
216 flProtect = PAGE_WRITECOPY;
217 dwDesiredAccess = FILE_MAP_COPY;
218 break;
219 }
220
221 if (filename != null)
222 {
223 hFile = CreateFileW(filename.tempCStringW(),
224 dwDesiredAccess2,
225 dwShareMode,
226 null,
227 dwCreationDisposition,
228 FILE_ATTRIBUTE_NORMAL,
229 cast(HANDLE) null);
230 wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
231 }
232 else
233 hFile = INVALID_HANDLE_VALUE;
234
235 scope(failure)
236 {
237 if (hFile != INVALID_HANDLE_VALUE)
238 {
239 CloseHandle(hFile);
240 hFile = INVALID_HANDLE_VALUE;
241 }
242 }
243
244 int hi = cast(int)(size >> 32);
245 hFileMap = CreateFileMappingW(hFile, null, flProtect,
246 hi, cast(uint) size, null);
247 wenforce(hFileMap, "CreateFileMapping");
248 scope(failure)
249 {
250 CloseHandle(hFileMap);
251 hFileMap = null;
252 }
253
254 if (size == 0 && filename != null)
255 {
256 uint sizehi;
257 uint sizelow = GetFileSize(hFile, &sizehi);
258 wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
259 "GetFileSize");
260 size = (cast(ulong) sizehi << 32) + sizelow;
261 }
262 this.size = size;
263
264 size_t initial_map = (window && 2*window<size)
265 ? 2*window : cast(size_t) size;
266 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
267 initial_map, address);
268 wenforce(p, "MapViewOfFileEx");
269 data = p[0 .. initial_map];
270
271 debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
272 }
273 else version (Posix)
274 {
275 void* p;
276 int oflag;
277 int fmode;
278
279 final switch (mode)
280 {
281 case Mode.read:
282 flags = MAP_SHARED;
283 prot = PROT_READ;
284 oflag = O_RDONLY;
285 fmode = 0;
286 break;
287
288 case Mode.readWriteNew:
289 assert(size != 0);
290 flags = MAP_SHARED;
291 prot = PROT_READ | PROT_WRITE;
292 oflag = O_CREAT | O_RDWR | O_TRUNC;
293 fmode = octal!660;
294 break;
295
296 case Mode.readWrite:
297 flags = MAP_SHARED;
298 prot = PROT_READ | PROT_WRITE;
299 oflag = O_CREAT | O_RDWR;
300 fmode = octal!660;
301 break;
302
303 case Mode.readCopyOnWrite:
304 flags = MAP_PRIVATE;
305 prot = PROT_READ | PROT_WRITE;
306 oflag = O_RDWR;
307 fmode = 0;
308 break;
309 }
310
311 if (filename.length)
312 {
313 fd = .open(filename.tempCString(), oflag, fmode);
314 errnoEnforce(fd != -1, "Could not open file "~filename);
315
316 stat_t statbuf;
317 if (fstat(fd, &statbuf))
318 {
319 //printf("\tfstat error, errno = %d\n", errno);
320 .close(fd);
321 fd = -1;
322 errnoEnforce(false, "Could not stat file "~filename);
323 }
324
325 if (prot & PROT_WRITE && size > statbuf.st_size)
326 {
327 // Need to make the file size bytes big
328 .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
329 char c = 0;
330 core.sys.posix.unistd.write(fd, &c, 1);
331 }
332 else if (prot & PROT_READ && size == 0)
333 size = statbuf.st_size;
334 }
335 else
336 {
337 fd = -1;
338 flags |= MAP_ANON;
339 }
340 this.size = size;
341 size_t initial_map = (window && 2*window<size)
342 ? 2*window : cast(size_t) size;
343 p = mmap(address, initial_map, prot, flags, fd, 0);
344 if (p == MAP_FAILED)
345 {
346 if (fd != -1)
347 {
348 .close(fd);
349 fd = -1;
350 }
351 errnoEnforce(false, "Could not map file "~filename);
352 }
353
354 data = p[0 .. initial_map];
355 }
356 else
357 {
358 static assert(0);
359 }
360 }
361
362 /**
363 * Flushes pending output and closes the memory mapped file.
364 */
365 ~this()
366 {
367 debug (MMFILE) printf("MmFile.~this()\n");
368 unmap();
369 data = null;
370 version (Windows)
371 {
372 wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
373 "Could not close file handle");
374 hFileMap = null;
375
376 wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
377 || CloseHandle(hFile) == TRUE,
378 "Could not close handle");
379 hFile = INVALID_HANDLE_VALUE;
380 }
381 else version (Posix)
382 {
383 version (linux)
384 {
385 if (file !is File.init)
386 {
387 // The File destructor will close the file,
388 // if it is the only remaining reference.
389 return;
390 }
391 }
392 errnoEnforce(fd == -1 || fd <= 2
393 || .close(fd) != -1,
394 "Could not close handle");
395 fd = -1;
396 }
397 else
398 {
399 static assert(0);
400 }
401 }
402
403 /* Flush any pending output.
404 */
405 void flush()
406 {
407 debug (MMFILE) printf("MmFile.flush()\n");
408 version (Windows)
409 {
410 FlushViewOfFile(data.ptr, data.length);
411 }
412 else version (Posix)
413 {
414 int i;
415 i = msync(cast(void*) data, data.length, MS_SYNC); // sys/mman.h
416 errnoEnforce(i == 0, "msync failed");
417 }
418 else
419 {
420 static assert(0);
421 }
422 }
423
424 /**
425 * Gives size in bytes of the memory mapped file.
426 */
427 @property ulong length() const
428 {
429 debug (MMFILE) printf("MmFile.length()\n");
430 return size;
431 }
432
433 /**
434 * Forwards `length`.
435 */
436 alias opDollar = length;
437
438 /**
439 * Read-only property returning the file mode.
440 */
441 Mode mode()
442 {
443 debug (MMFILE) printf("MmFile.mode()\n");
444 return mMode;
445 }
446
447 /**
448 * Returns entire file contents as an array.
449 */
450 void[] opSlice()
451 {
452 debug (MMFILE) printf("MmFile.opSlice()\n");
453 return opSlice(0,size);
454 }
455
456 /**
457 * Returns slice of file contents as an array.
458 */
459 void[] opSlice(ulong i1, ulong i2)
460 {
461 debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
462 ensureMapped(i1,i2);
463 size_t off1 = cast(size_t)(i1-start);
464 size_t off2 = cast(size_t)(i2-start);
465 return data[off1 .. off2];
466 }
467
468 /**
469 * Returns byte at index i in file.
470 */
471 ubyte opIndex(ulong i)
472 {
473 debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
474 ensureMapped(i);
475 size_t off = cast(size_t)(i-start);
476 return (cast(ubyte[]) data)[off];
477 }
478
479 /**
480 * Sets and returns byte at index i in file to value.
481 */
482 ubyte opIndexAssign(ubyte value, ulong i)
483 {
484 debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
485 ensureMapped(i);
486 size_t off = cast(size_t)(i-start);
487 return (cast(ubyte[]) data)[off] = value;
488 }
489
490
491 // return true if the given position is currently mapped
492 private int mapped(ulong i)
493 {
494 debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
495 data.length);
496 return i >= start && i < start+data.length;
497 }
498
499 // unmap the current range
500 private void unmap()
501 {
502 debug (MMFILE) printf("MmFile.unmap()\n");
503 version (Windows)
504 {
505 wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
506 }
507 else
508 {
509 errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
510 "munmap failed");
511 }
512 data = null;
513 }
514
515 // map range
516 private void map(ulong start, size_t len)
517 {
518 debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
519 void* p;
520 if (start+len > size)
521 len = cast(size_t)(size-start);
522 version (Windows)
523 {
524 uint hi = cast(uint)(start >> 32);
525 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
526 wenforce(p, "MapViewOfFileEx");
527 }
528 else
529 {
530 p = mmap(address, len, prot, flags, fd, cast(off_t) start);
531 errnoEnforce(p != MAP_FAILED);
532 }
533 data = p[0 .. len];
534 this.start = start;
535 }
536
537 // ensure a given position is mapped
538 private void ensureMapped(ulong i)
539 {
540 debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
541 if (!mapped(i))
542 {
543 unmap();
544 if (window == 0)
545 {
546 map(0,cast(size_t) size);
547 }
548 else
549 {
550 ulong block = i/window;
551 if (block == 0)
552 map(0,2*window);
553 else
554 map(window*(block-1),3*window);
555 }
556 }
557 }
558
559 // ensure a given range is mapped
560 private void ensureMapped(ulong i, ulong j)
561 {
562 debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
563 if (!mapped(i) || !mapped(j-1))
564 {
565 unmap();
566 if (window == 0)
567 {
568 map(0,cast(size_t) size);
569 }
570 else
571 {
572 ulong iblock = i/window;
573 ulong jblock = (j-1)/window;
574 if (iblock == 0)
575 {
576 map(0,cast(size_t)(window*(jblock+2)));
577 }
578 else
579 {
580 map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
581 }
582 }
583 }
584 }
585
586 private:
587 string filename;
588 void[] data;
589 ulong start;
590 size_t window;
591 ulong size;
592 Mode mMode;
593 void* address;
594 version (linux) File file;
595
596 version (Windows)
597 {
598 HANDLE hFile = INVALID_HANDLE_VALUE;
599 HANDLE hFileMap = null;
600 uint dwDesiredAccess;
601 }
602 else version (Posix)
603 {
604 int fd;
605 int prot;
606 int flags;
607 int fmode;
608 }
609 else
610 {
611 static assert(0);
612 }
613
614 // Report error, where errno gives the error number
615 // void errNo()
616 // {
617 // version (Windows)
618 // {
619 // throw new FileException(filename, GetLastError());
620 // }
621 // else version (linux)
622 // {
623 // throw new FileException(filename, errno);
624 // }
625 // else
626 // {
627 // static assert(0);
628 // }
629 // }
630 }
631
632 @system unittest
633 {
634 import core.memory : GC;
635 import std.file : deleteme;
636
637 const size_t K = 1024;
638 size_t win = 64*K; // assume the page size is 64K
639 version (Windows)
640 {
641 /+ these aren't defined in core.sys.windows.windows so let's use default
642 SYSTEM_INFO sysinfo;
643 GetSystemInfo(&sysinfo);
644 win = sysinfo.dwAllocationGranularity;
645 +/
646 }
647 else version (Posix)
648 {
649 import core.sys.posix.unistd;
650 win = cast(size_t) sysconf(_SC_PAGESIZE);
651 }
652 string test_file = std.file.deleteme ~ "-testing.txt";
653 MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew,
654 100*K,null,win);
655 ubyte[] str = cast(ubyte[])"1234567890";
656 ubyte[] data = cast(ubyte[]) mf[0 .. 10];
657 data[] = str[];
658 assert( mf[0 .. 10] == str );
659 data = cast(ubyte[]) mf[50 .. 60];
660 data[] = str[];
661 assert( mf[50 .. 60] == str );
662 ubyte[] data2 = cast(ubyte[]) mf[20*K .. 60*K];
663 assert( data2.length == 40*K );
664 assert( data2[$-1] == 0 );
665 mf[100*K-1] = cast(ubyte)'b';
666 data2 = cast(ubyte[]) mf[21*K .. 100*K];
667 assert( data2.length == 79*K );
668 assert( data2[$-1] == 'b' );
669
670 destroy(mf);
671 GC.free(&mf);
672
673 std.file.remove(test_file);
674 // Create anonymous mapping
675 auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null);
676 }
677
678 version (linux)
679 @system unittest // https://issues.dlang.org/show_bug.cgi?id=14868
680 {
681 import std.file : deleteme;
682 import std.typecons : scoped;
683
684 // Test retaining ownership of File/fd
685
686 auto fn = std.file.deleteme ~ "-testing.txt";
687 scope(exit) std.file.remove(fn);
688 File(fn, "wb").writeln("Testing!");
689 scoped!MmFile(File(fn));
690
691 // Test that unique ownership of File actually leads to the fd being closed
692
693 auto f = File(fn);
694 auto fd = f.fileno;
695 {
696 auto mf = scoped!MmFile(f);
697 f = File.init;
698 }
699 assert(.close(fd) == -1);
700 }
701
702 // https://issues.dlang.org/show_bug.cgi?id=14994
703 // https://issues.dlang.org/show_bug.cgi?id=14995
704 @system unittest
705 {
706 import std.file : deleteme;
707 import std.typecons : scoped;
708
709 // Zero-length map may or may not be valid on OSX and NetBSD
710 version (OSX)
711 import std.exception : verifyThrown = collectException;
712 version (NetBSD)
713 import std.exception : verifyThrown = collectException;
714 else
715 import std.exception : verifyThrown = assertThrown;
716
717 auto fn = std.file.deleteme ~ "-testing.txt";
718 scope(exit) std.file.remove(fn);
719 verifyThrown(scoped!MmFile(fn, MmFile.Mode.readWrite, 0, null));
720 }
721
722 @system unittest
723 {
724 MmFile shar = new MmFile(null, MmFile.Mode.readWrite, 10, null, 0);
725 void[] output = shar[0 .. $];
726 }
The D Programming Language's Unofficial Docs
beta