Added read-ahead buffering when reading zip index files.

git-svn-id: svn://svn.code.sf.net/p/sc2/code/trunk@3086 8092fc87-c524-0410-9efc-e669fe64eaf9

sc2/ChangeLog

@@ -1,4 +1,5 @@
 Changes towards version 0.7:
+- Added read-ahead buffering when reading zip index files. - SvdB
 - Added support for packed ani and font files - Mika
 - DrawTracedText abstraction (bug #1029), from Nic
 - Experimental support for Symbian S60 3rd edition - Mika & SvdB

sc2/src/sc2code/libs/uio/fileblock.c

@@ -18,6 +18,8 @@
  *
  */
 
+#define uio_INTERNAL_FILEBLOCK
+
 #include "iointrn.h"
 #include "fileblock.h"
 #include "uioport.h"
@@ -25,10 +27,41 @@
 #include <errno.h>
 
 static uio_FileBlock *uio_FileBlock_new(uio_Handle *handle, int flags,
-		off_t offset, size_t blockSize, char *buffer, size_t bufSize);
+		off_t offset, size_t blockSize, char *buffer, size_t bufSize,
+		off_t bufOffset, size_t bufFill, size_t readAheadBufSize);
 static inline uio_FileBlock *uio_FileBlock_alloc(void);
 static void uio_FileBlock_free(uio_FileBlock *block);
 
+// caller should uio_Handle_ref(handle) (unless it doesn't need it's own
+// reference anymore).
+static uio_FileBlock *
+uio_FileBlock_new(uio_Handle *handle, int flags, off_t offset,
+		size_t blockSize, char *buffer, size_t bufSize, off_t bufOffset,
+		size_t bufFill, size_t readAheadBufSize) {
+	uio_FileBlock *result;
+	
+	result = uio_FileBlock_alloc();
+	result->handle = handle;
+	result->flags = flags;
+	result->offset = offset;
+	result->blockSize = blockSize;
+	result->buffer = buffer;
+	result->bufSize = bufSize;
+	result->bufOffset = bufOffset;
+	result->bufFill = bufFill;
+	result->readAheadBufSize = readAheadBufSize;
+	return result;
+}
+
+static inline uio_FileBlock *
+uio_FileBlock_alloc(void) {
+	return uio_malloc(sizeof (uio_FileBlock));
+}
+
+static inline void
+uio_FileBlock_free(uio_FileBlock *block) {
+	uio_free(block);
+}
 
 uio_FileBlock *
 uio_openFileBlock(uio_Handle *handle) {
@@ -44,7 +77,7 @@ uio_openFileBlock(uio_Handle *handle) {
 		return NULL;
 	}
 	uio_Handle_ref(handle);
-	return uio_FileBlock_new(handle, 0, 0, statBuf.st_size, NULL, 0);
+	return uio_FileBlock_new(handle, 0, 0, statBuf.st_size, NULL, 0, 0, 0, 0);
 }
 
 uio_FileBlock *
@@ -58,52 +91,153 @@ uio_openFileBlock2(uio_Handle *handle, off_t offset, size_t size) {
 		// errno is set
 		return NULL;
 	}
-	if (statBuf.st_size > 
+	if (statBuf.st_size > offset || (statBuf.st_size - offset > size)) {
+		// NOT: 'if (statBuf.st_size > offset + size)', to protect
+		// against overflow.
+
+	}
 #endif
 	uio_Handle_ref(handle);
-	return uio_FileBlock_new(handle, 0, offset, size, NULL, 0);
+	return uio_FileBlock_new(handle, 0, offset, size, NULL, 0, 0, 0, 0);
+}
+
+static inline ssize_t
+uio_accessFileBlockMmap(uio_FileBlock *block, off_t offset, size_t length,
+		char **buffer) {
+	// TODO
+	errno = ENOSYS;
+	(void) block;
+	(void) offset;
+	(void) length;
+	(void) buffer;
+	return -1;
+}
+
+static inline ssize_t
+uio_accessFileBlockNoMmap(uio_FileBlock *block, off_t offset, size_t length,
+		char **buffer) {
+
+	// TODO: add read-ahead buffering
+	ssize_t numRead;
+	off_t start;
+	off_t end;
+	size_t bufSize;
+	char *oldBuffer;
+	//size_t oldBufSize;
+
+	// Don't go beyond the end of the block.
+	if (offset > (off_t) block->blockSize) {
+		*buffer = block->buffer;
+		return 0;
+	}
+	if (length > block->blockSize - offset)
+		length = block->blockSize - offset;
+
+	if (block->buffer != NULL) {
+		// Check whether the requested data is already in the buffer.
+		if (offset >= block->bufOffset &&
+				(offset - block->bufOffset) + length < block->bufFill) {
+			*buffer = block->buffer + (offset - block->bufOffset);
+			return length;
+		}
+	}
+
+	if (length < block->readAheadBufSize &&
+			(block->flags & uio_FB_USAGE_MASK) != 0) {
+		// We can buffer more data.
+		switch (block->flags & uio_FB_USAGE_MASK) {
+			case uio_FB_USAGE_FORWARD:
+				// Read extra data after the requested data.
+				start = offset;
+				end = (block->readAheadBufSize > block->blockSize - offset) ?
+						block->blockSize : offset + block->readAheadBufSize;
+				break;
+			case uio_FB_USAGE_BACKWARD:
+				// Read extra data before the requested data.
+				end = offset + length;
+				start = (end <= (off_t) block->blockSize) ?
+						0 : end - block->bufSize;
+				break;
+			case uio_FB_USAGE_FORWARD | uio_FB_USAGE_BACKWARD: {
+				// Read extra data both before and after the requested data.
+				off_t extraBefore = (block->readAheadBufSize - length) / 2;
+				start = (offset < extraBefore) ? 0 : offset - extraBefore;
+
+				end = (block->readAheadBufSize > block->blockSize - start) ?
+						block->blockSize : start + block->readAheadBufSize;
+				break;
+			}
+		}
+	} else {
+		start = offset;
+		end = offset + length;
+	}
+	bufSize = (length > block->readAheadBufSize) ?
+			length : block->readAheadBufSize;
+
+	// Start contains the start index in the block of the data we're going
+	// to read.
+	// End contains the end index.
+	// bufSize contains the size of the buffer. bufSize >= end - start.
+
+	oldBuffer = block->buffer;
+	//oldBufSize = block->bufSize;
+	if (block->buffer != NULL || block->bufSize != bufSize) {
+		// We don't have a buffer, or we have one, but of the wrong size.
+		block->buffer = uio_malloc(bufSize);
+		block->bufSize = bufSize;
+	}
+
+	if (oldBuffer != NULL) {
+		// TODO: If we have part of the data still in the old buffer, we
+		// can keep that.
+		// memmove(...)
+
+		if (oldBuffer != block->buffer)
+			uio_free(oldBuffer);
+	}
+	block->bufFill = 0;
+	block->bufOffset = start;
+
+	// TODO: lock handle
+	if (uio_lseek(block->handle, block->offset + start, SEEK_SET) ==
+			(off_t) -1) {
+		// errno is set
+		return -1;
+	}
+	
+	numRead = uio_read(block->handle, block->buffer, end - start);
+	if (numRead == -1) {
+		// errno is set
+		// TODO: unlock handle
+		return -1;
+	}
+	// TODO: unlock handle
+
+	block->bufFill = numRead;
+	*buffer = block->buffer + (offset - block->bufOffset);
+	if (numRead <= (offset - block->bufOffset))
+		return 0;
+	if ((size_t) numRead >= length)
+		return length;
+	return numRead - offset;
 }
 
 // block remains usable until the next call to uio_accessFileBlock
 // with the same block as argument, or until uio_closeFileBlock with
 // that block as argument.
+// The 'offset' parameter is wrt. the start of the block.
+// Requesting access to data beyond the file block is not an error. The
+// FileBlock is meant to be used as a replacement of seek() and read(), and
+// as with those functions, trying to go beyond the end of a file just
+// goes to the end. The return value is the number of bytes in the buffer.
 ssize_t
 uio_accessFileBlock(uio_FileBlock *block, off_t offset, size_t length,
 		char **buffer) {
 	if (block->flags & uio_FB_USE_MMAP) {
-		// TODO
+		return uio_accessFileBlockMmap(block, offset, length, buffer);
-		errno = ENOSYS;
-		return -1;
 	} else {
-		// TODO: add read-ahead buffering
+		return uio_accessFileBlockNoMmap(block, offset, length, buffer);
-		ssize_t numRead;
-	
-		if (length > block->blockSize - offset)
-			length = block->blockSize - offset;
-
-		if (block->buffer != NULL && length != block->bufSize) {
-			uio_free(block->buffer);
-			block->buffer = NULL;
-		}
-		if (block->buffer == NULL)
-			block->buffer = uio_malloc(length);
-
-		// TODO: lock handle
-		if (uio_lseek(block->handle, block->offset + offset, SEEK_SET) ==
-				(off_t) -1) {
-			// errno is set
-			return -1;
-		}
-		
-		numRead = uio_read(block->handle, block->buffer, length);
-		if (numRead == -1) {
-			// errno is set
-			// TODO: unlock handle
-			return -1;
-		}
-		// TODO: unlock handle
-		*buffer = block->buffer;
-		return numRead;
 	}
 }
 
@@ -115,11 +249,32 @@ uio_copyFileBlock(uio_FileBlock *block, off_t offset, char *buffer,
 		errno = ENOSYS;
 		return -1;
 	} else {
-		ssize_t numRead;
+		ssize_t numCopied = 0;
+		ssize_t readResult;
 
+		// Don't go beyond the end of the block.
+		if (offset > (off_t) block->blockSize)
+			return 0;
 		if (length > block->blockSize - offset)
 			length = block->blockSize - offset;
 		
+		// Check whether (part of) the requested data is already in our
+		// own buffer.
+		if (block->buffer != NULL && offset >= block->bufOffset
+				&& offset < block->bufOffset + (off_t) block->bufFill) {
+			size_t toCopy = block->bufFill - offset;
+			if (toCopy > length)
+				toCopy = length;
+			memcpy(buffer, block->buffer + (offset - block->bufOffset),
+					toCopy);
+			numCopied += toCopy;
+			length -= toCopy;
+			if (length == 0)
+				return numCopied;
+			buffer += toCopy;
+			offset += toCopy;
+		}
+
 		// TODO: lock handle
 		if (uio_lseek(block->handle, block->offset + offset, SEEK_SET) ==
 				(off_t) -1) {
@@ -127,14 +282,15 @@ uio_copyFileBlock(uio_FileBlock *block, off_t offset, char *buffer,
 			return -1;
 		}
 		
-		numRead = uio_read(block->handle, buffer, length);
+		readResult = uio_read(block->handle, buffer, length);
-		if (numRead == -1) {
+		// TODO: unlock handle
+		if (readResult == -1) {
 			// errno is set
-			// TODO: unlock handle
 			return -1;
 		}
-		// TODO: unlock handle
+		numCopied += readResult;
-		return numRead;
+
+		return numCopied;
 	}
 }
 
@@ -154,31 +310,25 @@ uio_closeFileBlock(uio_FileBlock *block) {
 	return 0;
 }
 
-// caller should uio_Handle_ref(handle) (unless it doesn't need it's own
+// Usage is the or'ed value of zero or more of uio_FB_USAGE_FORWARD,
-// reference anymore).
+// and uio_FB_USAGE_BACKWARD.
-static uio_FileBlock *
+void
-uio_FileBlock_new(uio_Handle *handle, int flags, off_t offset,
+uio_setFileBlockUsageHint(uio_FileBlock *block, int usage,
-		size_t blockSize, char *buffer, size_t bufSize) {
+		size_t readAheadBufSize) {
-	uio_FileBlock *result;
+	block->flags = (block->flags & ~uio_FB_USAGE_MASK) |
-	
+			(usage & uio_FB_USAGE_MASK);
-	result = uio_FileBlock_alloc();
+	block->readAheadBufSize = readAheadBufSize;
-	result->handle = handle;
-	result->flags = flags;
-	result->offset = offset;
-	result->blockSize = blockSize;
-	result->buffer = buffer;
-	result->bufSize = bufSize;
-	return result;
 }
 
-static inline uio_FileBlock *
+// Call if you want the memory used by the fileblock to be released, but
-uio_FileBlock_alloc(void) {
+// still want to use the fileblock later. If you don't need the fileblock,
-	return uio_malloc(sizeof (uio_FileBlock));
+// call uio_closeFileBlock() instead.
-}
+void
-
+uio_clearFileBlockBuffers(uio_FileBlock *block) {
-static void
+	if (block->buffer != NULL) {
-uio_FileBlock_free(uio_FileBlock *block) {
+		uio_free(block->buffer);
-	uio_free(block);
+		block->buffer = NULL;
+	}
 }
 
 

sc2/src/sc2code/libs/uio/fileblock.h

@@ -21,22 +21,54 @@
 #ifndef _FILEBLOCK_H
 #define _FILEBLOCK_H
 
+typedef struct uio_FileBlock uio_FileBlock;
+
 #include "io.h"
 #include "uioport.h"
 
 #include <sys/types.h>
 
-typedef struct uio_FileBlock {
+#define uio_FB_USAGE_FORWARD  1
+#define uio_FB_USAGE_BACKWARD 2
+#define uio_FB_USAGE_MASK  (uio_FB_USAGE_FORWARD | uio_FB_USAGE_BACKWARD)
+
+#ifdef uio_INTERNAL_FILEBLOCK
+
+// A fileblock represents a contiguous block of data from a file.
+// It's purpose is to avoid needless copying of data, while enabling
+// buffering.
+
+struct uio_FileBlock {
 	uio_Handle *handle;
 	int flags;
-#define uio_FB_USE_MMAP 1
+			// See above for uio_FB_USAGE_FORWARD, uio_FB_USAGE_BACKWARD.
+#define uio_FB_USE_MMAP       4
 	off_t offset;
 			// Offset to the start of the block in the file.
 	size_t blockSize;
+			// Size of the block of data represented by this FileBlock.
 	char *buffer;
 			// either allocated buffer, or buffer to mmap'ed area.
 	size_t bufSize;
-} uio_FileBlock;
+			// Size of the buffer.
+	off_t bufOffset;
+			// Offset of the start of the buffer into the block.
+	size_t bufFill;
+			// Part of 'buffer' which is in use.
+	size_t readAheadBufSize;
+			// Try to read up to this many bytes at a time, even when less
+			// is immediately needed.
+};
+// INV: The FileBlock represents 'fileData[offset..(offset + blockSize - 1)]'
+// where 'fileData' is the contents of the file.
+// INV: If buf != NULL then:
+//     bufFill <= bufSize
+//     bufFill <= blockSize
+//     buffer[0..bufFill - 1] == fileData[
+//             (offset + bufOffset)..(offset + bufOffset + bufFill - 1)]
+
+
+#endif  /* uio_INTERNAL_FILEBLOCK */
 
 uio_FileBlock *uio_openFileBlock(uio_Handle *handle);
 uio_FileBlock *uio_openFileBlock2(uio_Handle *handle, off_t offset,
@@ -46,6 +78,10 @@ ssize_t uio_accessFileBlock(uio_FileBlock *block, off_t offset, size_t length,
 int uio_copyFileBlock(uio_FileBlock *block, off_t offset, char *buffer,
 		size_t length);
 int uio_closeFileBlock(uio_FileBlock *block);
+#define uio_FB_READAHEAD_BUFSIZE_MAX ((size_t) -1)
+void uio_setFileBlockUsageHint(uio_FileBlock *block, int usage,
+		size_t readAheadBufSize);
+void uio_clearFileBlockBuffers(uio_FileBlock *block);
 
 #endif  /* _FILEBLOCK_H */
 

sc2/src/sc2code/libs/uio/zip/zip.c

@@ -43,6 +43,8 @@
 #endif
 
 
+#define DIR_STRUCTURE_READ_BUFSIZE 0x10000
+
 static int zip_badFile(zip_GPFileData *gPFileData, char *fileName);
 static int zip_fillDirStructure(uio_GPDir *top, uio_Handle *handle);
 #if zip_USE_HEADERS == zip_USE_LOCAL_HEADERS
@@ -583,7 +585,7 @@ zip_mount(uio_Handle *handle, int flags) {
 		int savedErrno = errno;
 #ifdef DEBUG
 		fprintf(stderr, "Error: failed to read the zip directory "
-				"structure - %d.\n", errno);
+				"structure - %s.\n", strerror(errno));
 #endif
 		uio_GPRoot_umount(result);
 		errno = savedErrno;
@@ -648,6 +650,10 @@ zip_fillDirStructureCentral(uio_GPDir *top, uio_Handle *handle) {
 
 	startCentralDir = makeUInt32(buf[16], buf[17], buf[18], buf[19]);
 
+	// Enable read-ahead buffering, for speed.
+	uio_setFileBlockUsageHint(fileBlock, uio_FB_USAGE_FORWARD,
+			DIR_STRUCTURE_READ_BUFSIZE);
+
 	pos = startCentralDir;
 	while (numEntries--) {
 		if (zip_fillDirStructureCentralProcessEntry(top, fileBlock, &pos)