Mercurial > hg > bitcoin
changeset 3647:fdd92a9e5196 draft
Multiple blocks per file
Change the block storage layer again, this time with multiple files
per block, but tracked by txindex.dat database entries. The file
format is exactly the same as the earlier blk00001.dat, but with
smaller files (128 MiB for now).
The database entries track how many bytes each block file already
uses, how many blocks are in it, which range of heights is present
and which range of dates.
| author | Pieter Wuille <pieter.wuille@gmail.com> |
|---|---|
| date | Mon, 13 Aug 2012 19:11:05 +0200 |
| parents | 4314bbfbfee5 |
| children | 4e4ec61eb33d |
| files | src/db.cpp src/db.h src/main.cpp src/main.h |
| diffstat | 4 files changed, 265 insertions(+), 102 deletions(-) [+] |
line wrap: on
line diff
--- a/src/db.cpp +++ b/src/db.cpp @@ -546,6 +546,22 @@ return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); } +bool CTxDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { + return Write(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { + return Read(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::WriteLastBlockFile(int nFile) { + return Write(string("lastblockfile"), nFile); +} + +bool CTxDB::ReadLastBlockFile(int &nFile) { + return Read(string("lastblockfile"), nFile); +} + bool CTxDB::ReadHashBestChain(uint256& hashBestChain) { return Read(string("hashBestChain"), hashBestChain); @@ -609,6 +625,12 @@ pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); } + // Load block file info + ReadLastBlockFile(nLastBlockFile); + printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); + if (ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); + // Load hashBestChain pointer to end of best chain if (!ReadHashBestChain(hashBestChain)) { @@ -788,7 +810,8 @@ pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); pindexNew->nHeight = diskindex.nHeight; - pindexNew->nAlternative = diskindex.nAlternative; + pindexNew->pos = diskindex.pos; + pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime;
--- a/src/db.h +++ b/src/db.h @@ -339,6 +339,10 @@ bool WriteHashBestChain(uint256 hashBestChain); bool ReadBestInvalidWork(CBigNum& bnBestInvalidWork); bool WriteBestInvalidWork(CBigNum bnBestInvalidWork); + bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo); + bool WriteBlockFileInfo(int nFile, const CBlockFileInfo &fileinfo); + bool ReadLastBlockFile(int &nFile); + bool WriteLastBlockFile(int nFile); bool LoadBlockIndex(); private: bool LoadBlockIndexGuts();
--- a/src/main.cpp +++ b/src/main.cpp @@ -1399,6 +1399,8 @@ return true; } +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); + bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) { // Check it again in case a previous version let a bad block in @@ -1430,7 +1432,7 @@ // Since we're just checking the block and not actually connecting it, it might not (and probably shouldn't) be on the disk to get the transaction from nTxPos = 1; else - nTxPos = ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); + nTxPos = pindex->pos.nPos + ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); CBlockUndo blockundo; @@ -1507,6 +1509,17 @@ return error("ConnectBlock() : UpdateTxIndex failed"); } + // Write undo information to disk + if (pindex->GetUndoPos().IsNull() && pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) + { + CDiskBlockPos pos; + if (!FindUndoPos(txdb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + return error("ConnectBlock() : FindUndoPos failed"); + if (!blockundo.WriteToDisk(pos)) + return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); + pindex->nUndoPos = pos.nPos; + } + // Update block index on disk without changing it in memory. // The memory index structure will be changed after the db commits. if (pindex->pprev) @@ -1517,13 +1530,6 @@ return error("ConnectBlock() : WriteBlockIndex failed"); } - // Write undo information to disk - if (pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) - { - CAutoFile fileUndo(fopen(pindex->GetBlockPos().GetUndoFile(GetDataDir()).string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); - fileUndo << blockundo; - } - // Watch for transactions paying to me BOOST_FOREACH(CTransaction& tx, vtx) SyncWithWallets(tx, this, true); @@ -1788,8 +1794,8 @@ pindexNew->nHeight = pindexNew->pprev->nHeight + 1; } pindexNew->bnChainWork = (pindexNew->pprev ? pindexNew->pprev->bnChainWork : 0) + pindexNew->GetBlockWork(); - assert(pos.nHeight == pindexNew->nHeight); - pindexNew->nAlternative = pos.nAlternative; + pindexNew->pos = pos; + pindexNew->nUndoPos = 0; CTxDB txdb; if (!txdb.TxnBegin()) @@ -1819,6 +1825,57 @@ +bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +{ + bool fUpdatedLast = false; + + LOCK(cs_LastBlockFile); + + while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + nLastBlockFile++; + infoLastBlockFile.SetNull(); + txdb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + fUpdatedLast = true; + } + + pos.nFile = nLastBlockFile; + pos.nPos = infoLastBlockFile.nSize; + infoLastBlockFile.nSize += nAddSize; + infoLastBlockFile.AddBlock(nHeight, nTime); + + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindBlockPos() : cannot write updated block info"); + if (fUpdatedLast) + txdb.WriteLastBlockFile(nLastBlockFile); + + return true; +} + +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + if (nFile == nLastBlockFile) { + pos.nPos = infoLastBlockFile.nUndoSize; + infoLastBlockFile.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindUndoPos() : cannot write updated block info"); + return true; + } + + CBlockFileInfo info; + if (!txdb.ReadBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot read block info"); + pos.nPos = info.nUndoSize; + info.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot write updated block info"); + return true; +} + bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const { @@ -1928,9 +1985,15 @@ } // Write block to history file - CDiskBlockPos blockPos = CDiskBlockPos(nHeight); + unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) return error("AcceptBlock() : out of disk space"); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, nHeight, nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!WriteToDisk(blockPos)) return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) @@ -2067,18 +2130,39 @@ return true; } -FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode) +CCriticalSection cs_LastBlockFile; +CBlockFileInfo infoLastBlockFile; +int nLastBlockFile = 0; + +FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { - boost::filesystem::path path = pos.GetFileName(GetDataDir()); - boost::filesystem::create_directories(path.parent_path()); if (pos.IsNull() || pos.IsMemPool()) return NULL; - FILE* file = fopen(path.string().c_str(), pszMode); + boost::filesystem::path path = GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + boost::filesystem::create_directories(path.parent_path()); + FILE* file = fopen(path.string().c_str(), "rb+"); + if (!file && !fReadOnly) + file = fopen(path.string().c_str(), "wb+"); if (!file) return NULL; + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + printf("Unable to seek to position %u of %s\n", pos.nPos, path.string().c_str()); + fclose(file); + return NULL; + } + } return file; } +FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "blk", fReadOnly); +} + +FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "rev", fReadOnly); +} + bool LoadBlockIndex(bool fAllowNew) { if (fTestNet) @@ -2146,7 +2230,13 @@ assert(hash == hashGenesisBlock); // Start new block file - CDiskBlockPos blockPos(0); + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, 0, block.nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!block.WriteToDisk(blockPos)) return error("LoadBlockIndex() : writing genesis block to disk failed"); if (!block.AddToBlockIndex(blockPos)) @@ -2203,9 +2293,9 @@ // print item CBlock block; block.ReadFromDisk(pindex); - printf("%d (%s) %s tx %"PRIszu"", + printf("%d (blk%05u.dat:0x%lx) %s tx %"PRIszu"", pindex->nHeight, - pindex->GetBlockPos().GetFileName("").string().c_str(), + pindex->GetBlockPos().nFile, pindex->GetBlockPos().nPos, DateTimeStrFormat("%x %H:%M:%S", block.GetBlockTime()).c_str(), block.vtx.size());
--- a/src/main.h +++ b/src/main.h @@ -28,6 +28,7 @@ static const unsigned int MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50; static const unsigned int MAX_ORPHAN_TRANSACTIONS = MAX_BLOCK_SIZE/100; static const unsigned int MAX_INV_SZ = 50000; +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB static const int64 MIN_TX_FEE = 50000; static const int64 MIN_RELAY_TX_FEE = 10000; static const int64 MAX_MONEY = 21000000 * COIN; @@ -87,7 +88,8 @@ void SyncWithWallets(const CTransaction& tx, const CBlock* pblock = NULL, bool fUpdate = false); bool ProcessBlock(CNode* pfrom, CBlock* pblock); bool CheckDiskSpace(uint64 nAdditionalBytes=0); -FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode="rb"); +FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); bool LoadBlockIndex(bool fAllowNew=true); void PrintBlockTree(); CBlockIndex* FindBlockByHeight(int nHeight); @@ -121,86 +123,27 @@ class CDiskBlockPos { public: - int nHeight; - int nAlternative; - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nHeightIn, int nAlternativeIn = 0) { - nHeight = nHeightIn; - nAlternative = nAlternativeIn; - } - - std::string GetAlternative() const { - char c[9]={0,0,0,0,0,0,0,0,0}; - char *cp = &c[8]; - unsigned int n = nAlternative; - while (n > 0 && cp>c) { - n--; - *(--cp) = 'a' + (n % 26); - n /= 26; - } - return std::string(cp); - } - - boost::filesystem::path GetDirectory(const boost::filesystem::path &base) const { - assert(nHeight != -1); - return base / strprintf("era%02u", nHeight / 210000) / - strprintf("cycle%04u", nHeight / 2016); - } + int nFile; + unsigned int nPos; - boost::filesystem::path GetFileName(const boost::filesystem::path &base) const { - return GetDirectory(base) / strprintf("%08u%s.blk", nHeight, GetAlternative().c_str()); - } - - boost::filesystem::path GetUndoFile(const boost::filesystem::path &base) const { - return GetDirectory(base) / strprintf("%08u%s.und", nHeight, GetAlternative().c_str()); - } - - // TODO: make thread-safe (lockfile, atomic file creation, ...?) - void MakeUnique(const boost::filesystem::path &base) { - while (boost::filesystem::exists(GetFileName(base))) - nAlternative++; - } - - IMPLEMENT_SERIALIZE(({ - CDiskBlockPos *me = const_cast<CDiskBlockPos*>(this); - if (!fRead) { - unsigned int nCode = (nHeight + 1) * 2 + (nAlternative > 0); - READWRITE(VARINT(nCode)); - if (nAlternative > 0) { - unsigned int nAlt = nAlternative - 1; - READWRITE(VARINT(nAlt)); - } - } else { - unsigned int nCode = 0; - READWRITE(VARINT(nCode)); - me->nHeight = (nCode / 2) - 1; - if (nCode & 1) { - unsigned int nAlt = 0; - READWRITE(VARINT(nAlt)); - me->nAlternative = 1 + nAlt; - } else { - me->nAlternative = 0; - } - } - });) + IMPLEMENT_SERIALIZE( + READWRITE(VARINT(nFile)); + READWRITE(VARINT(nPos)); + ) friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return ((a.nHeight == b.nHeight) && (a.nAlternative == b.nAlternative)); + return (a.nFile == b.nFile && a.nPos == b.nPos); } friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { return !(a == b); } - void SetNull() { nHeight = -1; nAlternative = 0; } - bool IsNull() const { return ((nHeight == -1) && (nAlternative == 0)); } + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } - void SetMemPool() { nHeight = -1; nAlternative = -1; } - bool IsMemPool() const { return ((nHeight == -1) && (nAlternative == -1)); } + void SetMemPool() { nFile = -2; nPos = 0; } + bool IsMemPool() const { return (nFile == -2); } }; /** Position on disk for a particular transaction. */ @@ -225,7 +168,7 @@ ) void SetNull() { blockPos.SetNull(); nTxPos = 0; } - bool IsNull() const { return blockPos.IsNull(); } + bool IsNull() const { return (nTxPos == 0); } bool IsMemPool() const { return blockPos.IsMemPool(); } friend bool operator==(const CDiskTxPos& a, const CDiskTxPos& b) @@ -246,7 +189,7 @@ else if (blockPos.IsMemPool()) return "mempool"; else - return strprintf("(%s, nTxPos=%u)", blockPos.GetFileName("").string().c_str(), nTxPos); + return strprintf("\"blk%05i.dat:0x%x\"", blockPos.nFile, nTxPos); } void print() const @@ -632,7 +575,7 @@ bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL) { - CAutoFile filein = CAutoFile(OpenBlockFile(pos.blockPos, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION); + CAutoFile filein = CAutoFile(OpenBlockFile(pos.blockPos, pfileRet==NULL), SER_DISK, CLIENT_VERSION); if (!filein) return error("CTransaction::ReadFromDisk() : OpenBlockFile failed"); @@ -823,6 +766,33 @@ IMPLEMENT_SERIALIZE( READWRITE(vtxundo); ) + + bool WriteToDisk(CDiskBlockPos &pos) + { + // Open history file to append + CAutoFile fileout = CAutoFile(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (!fileout) + return error("CBlockUndo::WriteToDisk() : OpenUndoFile failed"); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(*this); + fileout << FLATDATA(pchMessageStart) << nSize; + + // Write undo data + long fileOutPos = ftell(fileout); + if (fileOutPos < 0) + return error("CBlock::WriteToDisk() : ftell failed"); + pos.nPos = (unsigned int)fileOutPos; + fileout << *this; + + // Flush stdio buffers and commit to disk before returning + fflush(fileout); + if (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) + FileCommit(fileout); + + return true; + } + }; /** pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -1316,12 +1286,19 @@ bool WriteToDisk(CDiskBlockPos &pos) { // Open history file to append - pos.MakeUnique(GetDataDir()); - CAutoFile fileout = CAutoFile(OpenBlockFile(pos, "ab"), SER_DISK, CLIENT_VERSION); + CAutoFile fileout = CAutoFile(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); if (!fileout) - return error("CBlock::WriteToDisk() : AppendBlockFile failed"); + return error("CBlock::WriteToDisk() : OpenBlockFile failed"); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(*this); + fileout << FLATDATA(pchMessageStart) << nSize; // Write block + long fileOutPos = ftell(fileout); + if (fileOutPos < 0) + return error("CBlock::WriteToDisk() : ftell failed"); + pos.nPos = (unsigned int)fileOutPos; fileout << *this; // Flush stdio buffers and commit to disk before returning @@ -1337,7 +1314,7 @@ SetNull(); // Open history file to read - CAutoFile filein = CAutoFile(OpenBlockFile(pos, "rb"), SER_DISK, CLIENT_VERSION); + CAutoFile filein = CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); if (!filein) return error("CBlock::ReadFromDisk() : OpenBlockFile failed"); if (!fReadTransactions) @@ -1397,6 +1374,62 @@ +class CBlockFileInfo +{ +public: + unsigned int nBlocks; // number of blocks stored in file + unsigned int nSize; // number of used bytes of block file + unsigned int nUndoSize; // number of used bytes in the undo file + unsigned int nHeightFirst; // lowest height of block in file + unsigned int nHeightLast; // highest height of block in file + uint64 nTimeFirst; // earliest time of block in file + uint64 nTimeLast; // latest time of block in file + + IMPLEMENT_SERIALIZE( + READWRITE(VARINT(nBlocks)); + READWRITE(VARINT(nSize)); + READWRITE(VARINT(nUndoSize)); + READWRITE(VARINT(nHeightFirst)); + READWRITE(VARINT(nHeightLast)); + READWRITE(VARINT(nTimeFirst)); + READWRITE(VARINT(nTimeLast)); + ) + + void SetNull() { + nBlocks = 0; + nSize = 0; + nUndoSize = 0; + nHeightFirst = 0; + nHeightLast = 0; + nTimeFirst = 0; + nTimeLast = 0; + } + + CBlockFileInfo() { + SetNull(); + } + + std::string ToString() const { + return strprintf("CBlockFileInfo(blocks=%u, size=%lu, heights=%u..%u, time=%s..%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst).c_str(), DateTimeStrFormat("%Y-%m-%d", nTimeLast).c_str()); + } + + // update statistics (does not update nSize) + void AddBlock(unsigned int nHeightIn, uint64 nTimeIn) { + if (nBlocks==0 || nHeightFirst > nHeightIn) + nHeightFirst = nHeightIn; + if (nBlocks==0 || nTimeFirst > nTimeIn) + nTimeFirst = nTimeIn; + nBlocks++; + if (nHeightIn > nHeightFirst) + nHeightLast = nHeightIn; + if (nTimeIn > nTimeLast) + nTimeLast = nTimeIn; + } +}; + +extern CCriticalSection cs_LastBlockFile; +extern CBlockFileInfo infoLastBlockFile; +extern int nLastBlockFile; /** The block chain is a tree shaped structure starting with the * genesis block at the root, with each block potentially having multiple @@ -1412,7 +1445,8 @@ CBlockIndex* pprev; CBlockIndex* pnext; int nHeight; - unsigned int nAlternative; + CDiskBlockPos pos; + unsigned int nUndoPos; CBigNum bnChainWork; // block header @@ -1429,8 +1463,9 @@ pprev = NULL; pnext = NULL; nHeight = 0; + pos.SetNull(); + nUndoPos = (unsigned int)(-1); bnChainWork = 0; - nAlternative = 0; nVersion = 0; hashMerkleRoot = 0; @@ -1445,8 +1480,9 @@ pprev = NULL; pnext = NULL; nHeight = 0; + pos.SetNull(); + nUndoPos = 0; bnChainWork = 0; - nAlternative = 0; nVersion = block.nVersion; hashMerkleRoot = block.hashMerkleRoot; @@ -1456,7 +1492,16 @@ } CDiskBlockPos GetBlockPos() const { - return CDiskBlockPos(nHeight, nAlternative); + return pos; + } + + CDiskBlockPos GetUndoPos() const { + CDiskBlockPos ret = pos; + if (nUndoPos == (unsigned int)(-1)) + ret.SetNull(); + else + ret.nPos = nUndoPos; + return ret; } CBlock GetBlockHeader() const @@ -1578,7 +1623,8 @@ READWRITE(hashNext); READWRITE(nHeight); - READWRITE(nAlternative); + READWRITE(pos); + READWRITE(nUndoPos); // block header READWRITE(this->nVersion);