- Change disk format to add magic number to all pages

- Change blockfile magic number to reflect new format
- Cleanups and javadocs

core/java/src/net/metanotion/io/block/BlockFile.java

@@ -51,10 +51,20 @@ import net.metanotion.util.skiplist.SkipList;
 import net.i2p.I2PAppContext;
 import net.i2p.util.Log;
 
-class CorruptFileException extends IOException { }
+/**
-class BadFileFormatException extends IOException { }
+ * On-disk format:
-class BadVersionException extends IOException { }
+ *    Magic number (6 bytes)
-
+ *    Version major/minor (2 bytes)
+ *    file length (long)
+ *    free list start (unsigned int)
+ *    is mounted (unsigned short) 0 = no, 1 = yes
+ *    span size (unsigned short)
+ *
+ * Metaindex skiplist is on page 2
+ *
+ * Pages are 1 KB and are numbered starting from 1.
+ * e.g. the Metaindex skiplist is at offset 1024 bytes
+ */
 public class BlockFile {
 	public static final long PAGESIZE = 1024;
 	public static final long OFFSET_MOUNTED = 20;
@@ -62,7 +72,13 @@ public class BlockFile {
 
 	public RandomAccessInterface file;
 
-	private long magicBytes = 0x3141deadbeef0100L;
+	private static final int MAJOR = 0x01;
+	private static final int MINOR = 0x01;
+	// I2P changed magic number, format changed, magic numbers now on all pages
+	private static final long MAGIC_BASE = 0x3141de4932500000L;   // 0x3141de I 2 P 00 00
+	private static final long MAGIC = MAGIC_BASE | (MAJOR << 8) | MINOR;
+	private long magicBytes = MAGIC;
+	public static final int MAGIC_CONT = 0x434f4e54;   // "CONT"
 	private long fileLen = PAGESIZE * 2;
 	private int freeListStart = 0;
 	private int mounted = 0;
@@ -123,14 +139,17 @@ public class BlockFile {
 				if(curNextPage==0) {
 					curNextPage = this.allocPage();
 					BlockFile.pageSeek(this.file, curNextPage);
+					this.file.writeInt(MAGIC_CONT);
 					this.file.writeInt(0);
 					BlockFile.pageSeek(this.file, curPage);
+					this.file.skipBytes(4);   // skip magic
 					this.file.writeInt(curNextPage);
 				}
 				BlockFile.pageSeek(this.file, curNextPage);
 				curPage = curNextPage;
+				this.file.skipBytes(4);   // skip magic
 				curNextPage = this.file.readUnsignedInt();
-				pageCounter = 4;
+				pageCounter = 8;
 				len = ((int) BlockFile.PAGESIZE) - pageCounter;
 			}
 			this.file.write(data, dct, Math.min(len, data.length - dct));
@@ -152,9 +171,12 @@ public class BlockFile {
 			int len = ((int) BlockFile.PAGESIZE) - pageCounter;
 			if(len <= 0) {
 				BlockFile.pageSeek(this.file, curNextPage);
+				int magic = this.file.readInt();
+				if (magic != MAGIC_CONT)
+					throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
 				curPage = curNextPage;
 				curNextPage = this.file.readUnsignedInt();
-				pageCounter = 4;
+				pageCounter = 8;
 				len = ((int) BlockFile.PAGESIZE) - pageCounter;
 			}
 			res = this.file.read(arr, dct, Math.min(len, arr.length - dct));
@@ -184,16 +206,19 @@ public class BlockFile {
 		}
 
 		readSuperBlock();
-		if(magicBytes != 0x3141deadbeef0100L) {
+		if(magicBytes != MAGIC) {
-			if((magicBytes & 0x3141deadbeef0000L) == 0x3141deadbeef0000L) {
+			if((magicBytes & MAGIC_BASE) == MAGIC_BASE) {
-				throw new BadVersionException();
+				throw new IOException("Expected " + MAJOR + '.' + MINOR +
+				                      " but got " + (magicBytes >> 8 & 0xff) + '.' + (magicBytes & 0xff));
 			} else {
-				throw new BadFileFormatException();
+				throw new IOException("Bad magic number");
 			}
 		}
 		if (mounted != 0)
 			log.warn("Warning - file was not previously closed");
-		if(fileLen != file.length()) { throw new CorruptFileException(); }
+		if(fileLen != file.length())
+			throw new IOException("Expected file length " + fileLen +
+		                              " but actually " + file.length());
 		mount();
 
 		metaIndex = new BSkipList(spanSize, this, 2, new StringBytes(), new IntBytes());

core/java/src/net/metanotion/io/block/index/BSkipLevels.java

@@ -38,12 +38,14 @@ import net.metanotion.util.skiplist.SkipSpan;
 
 /**
  * On-disk format:
+ *    Magic number (long)
  *    max height (unsigned short)
  *    non-null height (unsigned short)
  *    span page (unsigned int)
  *    height number of level pages (unsigned ints)
  */
 public class BSkipLevels extends SkipLevels {
+	private static final long MAGIC = 0x42534c6576656c73l;  // "BSLevels"
 	public final int levelPage;
 	public final int spanPage;
 	public final BlockFile bf;
@@ -53,6 +55,9 @@ public class BSkipLevels extends SkipLevels {
 		this.bf = bf;
 
 		BlockFile.pageSeek(bf.file, levelPage);
+		long magic = bf.file.readLong();
+		if (magic != MAGIC)
+			throw new IOException("Bad SkipLevels magic number 0x" + Long.toHexString(magic) + " on page " + levelPage);
 
 		bsl.levelHash.put(new Integer(this.levelPage), this);
 
@@ -87,6 +92,7 @@ public class BSkipLevels extends SkipLevels {
 
 	public static void init(BlockFile bf, int page, int spanPage, int maxHeight) throws IOException {
 		BlockFile.pageSeek(bf.file, page);
+		bf.file.writeLong(MAGIC);
 		bf.file.writeShort((short) maxHeight);
 		bf.file.writeShort(0);
 		bf.file.writeInt(spanPage);
@@ -95,6 +101,7 @@ public class BSkipLevels extends SkipLevels {
 	public void flush() {
 		try {
 			BlockFile.pageSeek(bf.file, levelPage);
+			bf.file.writeLong(MAGIC);
 			bf.file.writeShort((short) levels.length);
 			int i = 0;
 			for( ; i < levels.length; i++) {

core/java/src/net/metanotion/io/block/index/BSkipList.java

@@ -37,7 +37,16 @@ import net.metanotion.io.Serializer;
 import net.metanotion.io.block.BlockFile;
 import net.metanotion.util.skiplist.*;
 
+/**
+ * On-disk format:
+ *    Magic number (long)
+ *    first span page (unsigned int)
+ *    first level page (unsigned int)
+ *    size (unsigned int)
+ *    spans (unsigned int)
+ */
 public class BSkipList extends SkipList {
+	private static final long MAGIC = 0x536b69704c697374l;  // "SkipList"
 	public int firstSpanPage = 0;
 	public int firstLevelPage = 0;
 	public int skipPage = 0;
@@ -59,6 +68,9 @@ public class BSkipList extends SkipList {
 		this.bf = bf;
 
 		BlockFile.pageSeek(bf.file, skipPage);
+		long magic = bf.file.readLong();
+		if (magic != MAGIC)
+			throw new IOException("Bad SkipList magic number 0x" + Long.toHexString(magic) + " on page " + skipPage);
 		firstSpanPage = bf.file.readUnsignedInt();
 		firstLevelPage = bf.file.readUnsignedInt();
 		size = bf.file.readUnsignedInt();
@@ -84,6 +96,7 @@ public class BSkipList extends SkipList {
 	public void flush() {
 		try {
 			BlockFile.pageSeek(bf.file, skipPage);
+			bf.file.writeLong(MAGIC);
 			bf.file.writeInt(firstSpanPage);
 			bf.file.writeInt(firstLevelPage);
 			bf.file.writeInt(size);
@@ -114,6 +127,7 @@ public class BSkipList extends SkipList {
 		int firstSpan = bf.allocPage();
 		int firstLevel = bf.allocPage();
 		BlockFile.pageSeek(bf.file, page);
+		bf.file.writeLong(MAGIC);
 		bf.file.writeInt(firstSpan);
 		bf.file.writeInt(firstLevel);
 		bf.file.writeInt(0);

core/java/src/net/metanotion/io/block/index/BSkipSpan.java

@@ -36,8 +36,30 @@ import net.metanotion.io.block.BlockFile;
 import net.metanotion.util.skiplist.SkipList;
 import net.metanotion.util.skiplist.SkipSpan;
 
+/**
+ * On-disk format:
+ *
+ *   First Page:
+ *     Magic number (int)
+ *     overflow page (unsigned int)
+ *     previous page (unsigned int)
+ *     next page (unsigned int)
+ *     max keys (unsigned short)
+ *     number of keys (unsigned short)
+ *     for each key:
+ *         key length (unsigned short)
+ *         value length (unsigned short)
+ *         key data
+ *         value data
+ *
+ *   Overflow pages:
+ *     Magic number (int)
+ *     next overflow page (unsigned int)
+ */
 public class BSkipSpan extends SkipSpan {
-
+	protected static final int MAGIC = 0x5370616e;  // "Span"
+	protected static final int HEADER_LEN = 20;
+	protected static final int CONT_HEADER_LEN = 8;
 	protected BlockFile bf;
 	protected int page;
 	protected int overflowPage;
@@ -52,6 +74,7 @@ public class BSkipSpan extends SkipSpan {
 
 	public static void init(BlockFile bf, int page, int spanSize) throws IOException {
 		BlockFile.pageSeek(bf.file, page);
+		bf.file.writeInt(MAGIC);
 		bf.file.writeInt(0);
 		bf.file.writeInt(0);
 		bf.file.writeInt(0);
@@ -73,6 +96,7 @@ public class BSkipSpan extends SkipSpan {
 			int next;
 			while(curPage != 0) {
 				BlockFile.pageSeek(bf.file, curPage);
+				bf.file.skipBytes(4);   // skip magic
 				next = bf.file.readUnsignedInt();
 				bf.freePage(curPage);
 				curPage = next;
@@ -91,6 +115,7 @@ public class BSkipSpan extends SkipSpan {
 	private void fflush() {
 		try {
 			BlockFile.pageSeek(bf.file, page);
+			bf.file.writeInt(MAGIC);
 			bf.file.writeInt(overflowPage);
 			bf.file.writeInt((prev != null) ? ((BSkipSpan) prev).page : 0);
 			bf.file.writeInt((next != null) ? ((BSkipSpan) next).page : 0);
@@ -102,7 +127,7 @@ public class BSkipSpan extends SkipSpan {
 			int[] curNextPage = new int[1];
 			curNextPage[0] = this.overflowPage;
 			int[] pageCounter = new int[1];
-			pageCounter[0] = 16;
+			pageCounter[0] = HEADER_LEN;
 			byte[] keyData;
 			byte[] valData;
 
@@ -111,14 +136,17 @@ public class BSkipSpan extends SkipSpan {
 					if(curNextPage[0] == 0) {
 						curNextPage[0] = bf.allocPage();
 						BlockFile.pageSeek(bf.file, curNextPage[0]);
+						bf.file.writeInt(BlockFile.MAGIC_CONT);
 						bf.file.writeInt(0);
 						BlockFile.pageSeek(bf.file, curPage);
+						bf.file.skipBytes(4);  // skip magic
 						bf.file.writeInt(curNextPage[0]);
 					}
 					BlockFile.pageSeek(bf.file, curNextPage[0]);
 					curPage = curNextPage[0];
+					bf.file.skipBytes(4);  // skip magic
 					curNextPage[0] = bf.file.readUnsignedInt();
-					pageCounter[0] = 4;
+					pageCounter[0] = CONT_HEADER_LEN;
 				}
 				// Drop bad entry without throwing exception
 				if (keys[i] == null || vals[i] == null) {
@@ -144,7 +172,9 @@ public class BSkipSpan extends SkipSpan {
 				curPage = bf.writeMultiPageData(keyData, curPage, pageCounter, curNextPage);
 				curPage = bf.writeMultiPageData(valData, curPage, pageCounter, curNextPage);
 			}
+			// FIXME why seek and rescan the overflow page?
 			BlockFile.pageSeek(bf.file, this.page);
+			bf.file.skipBytes(4);  // skip magic
 			this.overflowPage = bf.file.readUnsignedInt();
 		} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
 		// FIXME can't get there from here
@@ -171,6 +201,9 @@ public class BSkipSpan extends SkipSpan {
 
 		BlockFile.pageSeek(bf.file, spanPage);
 
+		int magic = bf.file.readInt();
+		if (magic != MAGIC)
+			throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + spanPage);
 		bss.overflowPage = bf.file.readUnsignedInt();
 		bss.prevPage = bf.file.readUnsignedInt();
 		bss.nextPage = bf.file.readUnsignedInt();
@@ -200,15 +233,18 @@ public class BSkipSpan extends SkipSpan {
 		int[] curNextPage = new int[1];
 		curNextPage[0] = this.overflowPage;
 		int[] pageCounter = new int[1];
-		pageCounter[0] = 16;
+		pageCounter[0] = HEADER_LEN;
 //		System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
 		int fail = 0;
 		for(int i=0;i<this.nKeys;i++) {
 			if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
 				BlockFile.pageSeek(this.bf.file, curNextPage[0]);
 				curPage = curNextPage[0];
+				int magic = bf.file.readInt();
+				if (magic != BlockFile.MAGIC_CONT)
+					throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
 				curNextPage[0] = this.bf.file.readUnsignedInt();
-				pageCounter[0] = 4;
+				pageCounter[0] = CONT_HEADER_LEN;
 			}
 			ksz = this.bf.file.readUnsignedShort();
 			vsz = this.bf.file.readUnsignedShort();

core/java/src/net/metanotion/io/block/index/IBSkipSpan.java

@@ -114,7 +114,7 @@ public class IBSkipSpan extends BSkipSpan {
 		int[] curNextPage = new int[1];
 		curNextPage[0] = this.overflowPage;
 		int[] pageCounter = new int[1];
-		pageCounter[0] = 16;
+		pageCounter[0] = HEADER_LEN;
 		ksz = this.bf.file.readUnsignedShort();
 		this.bf.file.skipBytes(2);  //vsz
 		pageCounter[0] +=4;
@@ -134,8 +134,11 @@ public class IBSkipSpan extends BSkipSpan {
 	 */
 	private void seekData() throws IOException {
 		BlockFile.pageSeek(this.bf.file, this.page);
+		int magic = bf.file.readInt();
+		if (magic != MAGIC)
+			throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + this.page);
 		// 3 ints and 2 shorts
-		this.bf.file.skipBytes(16);
+		this.bf.file.skipBytes(HEADER_LEN - 4);
 	}
 
 	/**
@@ -157,15 +160,18 @@ public class IBSkipSpan extends BSkipSpan {
 		int[] curNextPage = new int[1];
 		curNextPage[0] = this.overflowPage;
 		int[] pageCounter = new int[1];
-		pageCounter[0] = 16;
+		pageCounter[0] = HEADER_LEN;
 		int fail = 0;
 		//System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
 		for(int i=0;i<this.nKeys;i++) {
 			if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
 				BlockFile.pageSeek(this.bf.file, curNextPage[0]);
 				curPage = curNextPage[0];
+				int magic = bf.file.readInt();
+				if (magic != BlockFile.MAGIC_CONT)
+					throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
 				curNextPage[0] = this.bf.file.readUnsignedInt();
-				pageCounter[0] = 4;
+				pageCounter[0] = CONT_HEADER_LEN;
 			}
 			ksz = this.bf.file.readUnsignedShort();
 			vsz = this.bf.file.readUnsignedShort();