I2P_Developers/i2p.android.base
3
0
Fork 2
Code Issues 39 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

- Change disk format to add magic number to all pages

Browse Source 
- Change blockfile magic number to reflect new format
- Cleanups and javadocs
This commit is contained in:
zzz
2011-03-26 05:27:34 +00:00
parent 8f9f102baf
commit 312534b635
5 changed files with 109 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
core/java/src/net/metanotion/io/block/BlockFile.java
View File

@ -51,10 +51,20 @@ import net.metanotion.util.skiplist.SkipList;
import net.i2p.I2PAppContext;
import net.i2p.util.Log;
class CorruptFileException extends IOException { }
class BadFileFormatException extends IOException { }
class BadVersionException extends IOException { }
/**
* On-disk format:
* Magic number (6 bytes)
* Version major/minor (2 bytes)
* file length (long)
* free list start (unsigned int)
* is mounted (unsigned short) 0 = no, 1 = yes
* span size (unsigned short)
*
* Metaindex skiplist is on page 2
*
* Pages are 1 KB and are numbered starting from 1.
* e.g. the Metaindex skiplist is at offset 1024 bytes
*/
public class BlockFile {
public static final long PAGESIZE = 1024;
public static final long OFFSET_MOUNTED = 20;
@ -62,7 +72,13 @@ public class BlockFile {
public RandomAccessInterface file;
private long magicBytes = 0x3141deadbeef0100L;
private static final int MAJOR = 0x01;
private static final int MINOR = 0x01;
// I2P changed magic number, format changed, magic numbers now on all pages
private static final long MAGIC_BASE = 0x3141de4932500000L; // 0x3141de I 2 P 00 00
private static final long MAGIC = MAGIC_BASE | (MAJOR << 8) | MINOR;
private long magicBytes = MAGIC;
public static final int MAGIC_CONT = 0x434f4e54; // "CONT"
private long fileLen = PAGESIZE * 2;
private int freeListStart = 0;
private int mounted = 0;
@ -123,14 +139,17 @@ public class BlockFile {
if(curNextPage==0) {
curNextPage = this.allocPage();
BlockFile.pageSeek(this.file, curNextPage);
this.file.writeInt(MAGIC_CONT);
this.file.writeInt(0);
BlockFile.pageSeek(this.file, curPage);
this.file.skipBytes(4); // skip magic
this.file.writeInt(curNextPage);
}
BlockFile.pageSeek(this.file, curNextPage);
curPage = curNextPage;
this.file.skipBytes(4); // skip magic
curNextPage = this.file.readUnsignedInt();
pageCounter = 4;
pageCounter = 8;
len = ((int) BlockFile.PAGESIZE) - pageCounter;
}
this.file.write(data, dct, Math.min(len, data.length - dct));
@ -152,9 +171,12 @@ public class BlockFile {
int len = ((int) BlockFile.PAGESIZE) - pageCounter;
if(len <= 0) {
BlockFile.pageSeek(this.file, curNextPage);
int magic = this.file.readInt();
if (magic != MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
curPage = curNextPage;
curNextPage = this.file.readUnsignedInt();
pageCounter = 4;
pageCounter = 8;
len = ((int) BlockFile.PAGESIZE) - pageCounter;
}
res = this.file.read(arr, dct, Math.min(len, arr.length - dct));
@ -184,16 +206,19 @@ public class BlockFile {
}
readSuperBlock();
if(magicBytes != 0x3141deadbeef0100L) {
if((magicBytes & 0x3141deadbeef0000L) == 0x3141deadbeef0000L) {
throw new BadVersionException();
if(magicBytes != MAGIC) {
if((magicBytes & MAGIC_BASE) == MAGIC_BASE) {
throw new IOException("Expected " + MAJOR + '.' + MINOR +
" but got " + (magicBytes >> 8 & 0xff) + '.' + (magicBytes & 0xff));
} else {
throw new BadFileFormatException();
throw new IOException("Bad magic number");
}
}
if (mounted != 0)
log.warn("Warning - file was not previously closed");
if(fileLen != file.length()) { throw new CorruptFileException(); }
if(fileLen != file.length())
throw new IOException("Expected file length " + fileLen +
" but actually " + file.length());
mount();
metaIndex = new BSkipList(spanSize, this, 2, new StringBytes(), new IntBytes());

7
core/java/src/net/metanotion/io/block/index/BSkipLevels.java
View File

@ -38,12 +38,14 @@ import net.metanotion.util.skiplist.SkipSpan;
/**
* On-disk format:
* Magic number (long)
* max height (unsigned short)
* non-null height (unsigned short)
* span page (unsigned int)
* height number of level pages (unsigned ints)
*/
public class BSkipLevels extends SkipLevels {
private static final long MAGIC = 0x42534c6576656c73l; // "BSLevels"
public final int levelPage;
public final int spanPage;
public final BlockFile bf;
@ -53,6 +55,9 @@ public class BSkipLevels extends SkipLevels {
this.bf = bf;
BlockFile.pageSeek(bf.file, levelPage);
long magic = bf.file.readLong();
if (magic != MAGIC)
throw new IOException("Bad SkipLevels magic number 0x" + Long.toHexString(magic) + " on page " + levelPage);
bsl.levelHash.put(new Integer(this.levelPage), this);
@ -87,6 +92,7 @@ public class BSkipLevels extends SkipLevels {
public static void init(BlockFile bf, int page, int spanPage, int maxHeight) throws IOException {
BlockFile.pageSeek(bf.file, page);
bf.file.writeLong(MAGIC);
bf.file.writeShort((short) maxHeight);
bf.file.writeShort(0);
bf.file.writeInt(spanPage);
@ -95,6 +101,7 @@ public class BSkipLevels extends SkipLevels {
public void flush() {
try {
BlockFile.pageSeek(bf.file, levelPage);
bf.file.writeLong(MAGIC);
bf.file.writeShort((short) levels.length);
int i = 0;
for( ; i < levels.length; i++) {

14
core/java/src/net/metanotion/io/block/index/BSkipList.java
View File

@ -37,7 +37,16 @@ import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.*;
/**
* On-disk format:
* Magic number (long)
* first span page (unsigned int)
* first level page (unsigned int)
* size (unsigned int)
* spans (unsigned int)
*/
public class BSkipList extends SkipList {
private static final long MAGIC = 0x536b69704c697374l; // "SkipList"
public int firstSpanPage = 0;
public int firstLevelPage = 0;
public int skipPage = 0;
@ -59,6 +68,9 @@ public class BSkipList extends SkipList {
this.bf = bf;
BlockFile.pageSeek(bf.file, skipPage);
long magic = bf.file.readLong();
if (magic != MAGIC)
throw new IOException("Bad SkipList magic number 0x" + Long.toHexString(magic) + " on page " + skipPage);
firstSpanPage = bf.file.readUnsignedInt();
firstLevelPage = bf.file.readUnsignedInt();
size = bf.file.readUnsignedInt();
@ -84,6 +96,7 @@ public class BSkipList extends SkipList {
public void flush() {
try {
BlockFile.pageSeek(bf.file, skipPage);
bf.file.writeLong(MAGIC);
bf.file.writeInt(firstSpanPage);
bf.file.writeInt(firstLevelPage);
bf.file.writeInt(size);
@ -114,6 +127,7 @@ public class BSkipList extends SkipList {
int firstSpan = bf.allocPage();
int firstLevel = bf.allocPage();
BlockFile.pageSeek(bf.file, page);
bf.file.writeLong(MAGIC);
bf.file.writeInt(firstSpan);
bf.file.writeInt(firstLevel);
bf.file.writeInt(0);

46
core/java/src/net/metanotion/io/block/index/BSkipSpan.java
View File

@ -36,8 +36,30 @@ import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipSpan;
/**
* On-disk format:
*
* First Page:
* Magic number (int)
* overflow page (unsigned int)
* previous page (unsigned int)
* next page (unsigned int)
* max keys (unsigned short)
* number of keys (unsigned short)
* for each key:
* key length (unsigned short)
* value length (unsigned short)
* key data
* value data
*
* Overflow pages:
* Magic number (int)
* next overflow page (unsigned int)
*/
public class BSkipSpan extends SkipSpan {
protected static final int MAGIC = 0x5370616e; // "Span"
protected static final int HEADER_LEN = 20;
protected static final int CONT_HEADER_LEN = 8;
protected BlockFile bf;
protected int page;
protected int overflowPage;
@ -52,6 +74,7 @@ public class BSkipSpan extends SkipSpan {
public static void init(BlockFile bf, int page, int spanSize) throws IOException {
BlockFile.pageSeek(bf.file, page);
bf.file.writeInt(MAGIC);
bf.file.writeInt(0);
bf.file.writeInt(0);
bf.file.writeInt(0);
@ -73,6 +96,7 @@ public class BSkipSpan extends SkipSpan {
int next;
while(curPage != 0) {
BlockFile.pageSeek(bf.file, curPage);
bf.file.skipBytes(4); // skip magic
next = bf.file.readUnsignedInt();
bf.freePage(curPage);
curPage = next;
@ -91,6 +115,7 @@ public class BSkipSpan extends SkipSpan {
private void fflush() {
try {
BlockFile.pageSeek(bf.file, page);
bf.file.writeInt(MAGIC);
bf.file.writeInt(overflowPage);
bf.file.writeInt((prev != null) ? ((BSkipSpan) prev).page : 0);
bf.file.writeInt((next != null) ? ((BSkipSpan) next).page : 0);
@ -102,7 +127,7 @@ public class BSkipSpan extends SkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
pageCounter[0] = 16;
pageCounter[0] = HEADER_LEN;
byte[] keyData;
byte[] valData;
@ -111,14 +136,17 @@ public class BSkipSpan extends SkipSpan {
if(curNextPage[0] == 0) {
curNextPage[0] = bf.allocPage();
BlockFile.pageSeek(bf.file, curNextPage[0]);
bf.file.writeInt(BlockFile.MAGIC_CONT);
bf.file.writeInt(0);
BlockFile.pageSeek(bf.file, curPage);
bf.file.skipBytes(4); // skip magic
bf.file.writeInt(curNextPage[0]);
}
BlockFile.pageSeek(bf.file, curNextPage[0]);
curPage = curNextPage[0];
bf.file.skipBytes(4); // skip magic
curNextPage[0] = bf.file.readUnsignedInt();
pageCounter[0] = 4;
pageCounter[0] = CONT_HEADER_LEN;
}
// Drop bad entry without throwing exception
if (keys[i] == null || vals[i] == null) {
@ -144,7 +172,9 @@ public class BSkipSpan extends SkipSpan {
curPage = bf.writeMultiPageData(keyData, curPage, pageCounter, curNextPage);
curPage = bf.writeMultiPageData(valData, curPage, pageCounter, curNextPage);
}
// FIXME why seek and rescan the overflow page?
BlockFile.pageSeek(bf.file, this.page);
bf.file.skipBytes(4); // skip magic
this.overflowPage = bf.file.readUnsignedInt();
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
// FIXME can't get there from here
@ -171,6 +201,9 @@ public class BSkipSpan extends SkipSpan {
BlockFile.pageSeek(bf.file, spanPage);
int magic = bf.file.readInt();
if (magic != MAGIC)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + spanPage);
bss.overflowPage = bf.file.readUnsignedInt();
bss.prevPage = bf.file.readUnsignedInt();
bss.nextPage = bf.file.readUnsignedInt();
@ -200,15 +233,18 @@ public class BSkipSpan extends SkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
pageCounter[0] = 16;
pageCounter[0] = HEADER_LEN;
// System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
int fail = 0;
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = 4;
pageCounter[0] = CONT_HEADER_LEN;
}
ksz = this.bf.file.readUnsignedShort();
vsz = this.bf.file.readUnsignedShort();

14
core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
View File

@ -114,7 +114,7 @@ public class IBSkipSpan extends BSkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
pageCounter[0] = 16;
pageCounter[0] = HEADER_LEN;
ksz = this.bf.file.readUnsignedShort();
this.bf.file.skipBytes(2); //vsz
pageCounter[0] +=4;
@ -134,8 +134,11 @@ public class IBSkipSpan extends BSkipSpan {
*/
private void seekData() throws IOException {
BlockFile.pageSeek(this.bf.file, this.page);
int magic = bf.file.readInt();
if (magic != MAGIC)
throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + this.page);
// 3 ints and 2 shorts
this.bf.file.skipBytes(16);
this.bf.file.skipBytes(HEADER_LEN - 4);
}
/**
@ -157,15 +160,18 @@ public class IBSkipSpan extends BSkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
pageCounter[0] = 16;
pageCounter[0] = HEADER_LEN;
int fail = 0;
//System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
int magic = bf.file.readInt();
if (magic != BlockFile.MAGIC_CONT)
throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = 4;
pageCounter[0] = CONT_HEADER_LEN;
}
ksz = this.bf.file.readUnsignedShort();
vsz = this.bf.file.readUnsignedShort();