d0s3t3ch
/
wownero
mirror of https://git.wownero.com/wownero/wownero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merges #36

This commit is contained in:
Thomas Winget 2015-03-25 05:49:29 -04:00
parent 38c54b79fd dbdcf11778
commit a96134113e
No known key found for this signature in database
GPG Key ID: 58131A160789E630
6 changed files with 141 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/blockchain_converter/README.md Normal file
View File

@ -0,0 +1,54 @@
For importing into the LMDB database, compile with `DATABASE=lmdb`
e.g.
`DATABASE=lmdb make release`
This is also the default compile setting on the blockchain branch.
By default, the exporter will use the original in-memory database (blockchain.bin) as its source.
This default is to make migrating to an LMDB database easy, without having to recompile anything.
To change the source, adjust `SOURCE_DB` in `src/blockchain_converter/blockchain_export.h` according to the comments.
# Usage:
See also each utility's "--help" option.
## Export an existing in-memory database
`$ blockchain_export`
This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw`
## Import the exported file
`$ blockchain_import`
This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database.
Defaults: `--batch on`, `--batch size 20000`, `--verify on`
Batch size refers to number of blocks and can be adjusted for performance based on available RAM.
Verification should only be turned off if importing from a trusted blockchain.
```bash
# use default settings to import blockchain.raw into database
$ blockchain_import
# fast import with large batch size, verification off
$ blockchain_import --batch-size 100000 --verify off
# LMDB flags can be set by appending them to the database type:
# flags: nosync, nometasync, writemap, mapasync
$ blockchain_import --database lmdb#nosync
$ blockchain_import --database lmdb#nosync,nometasync
```
## Blockchain converter with batching
`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate.
```bash
$ blockchain_converter --batch on --batch-size 20000
```

47
src/blockchain_converter/blockchain_converter.cpp
View File

@ -44,14 +44,19 @@
#include "version.h"
#include <iostream>
namespace
{
// CONFIG
static bool opt_batch = true;
static bool opt_testnet = false;
bool opt_batch = true;
bool opt_resume = true;
bool opt_testnet = false;
// number of blocks per batch transaction
// adjustable through command-line argument according to available RAM
static uint64_t db_batch_size = 20000;
uint64_t db_batch_size = 20000;
}
namespace po = boost::program_options;
@ -113,11 +118,14 @@ int main(int argc, char* argv[])
const command_line::arg_descriptor<bool> arg_batch = {"batch",
"Batch transactions for faster import", true};
const command_line::arg_descriptor<bool> arg_resume = {"resume",
"Resume from current height if output database already exists", true};
// call add_options() directly for these arguments since command_line helpers
// support only boolean switch, not boolean argument
desc_cmd_sett.add_options()
(arg_batch.name, make_semantic(arg_batch), arg_batch.description)
(arg_batch.name, make_semantic(arg_batch), arg_batch.description)
(arg_resume.name, make_semantic(arg_resume), arg_resume.description)
;
po::options_description desc_options("Allowed options");
@ -137,6 +145,7 @@ int main(int argc, char* argv[])
int log_level = command_line::get_arg(vm, arg_log_level);
opt_batch = command_line::get_arg(vm, arg_batch);
opt_resume = command_line::get_arg(vm, arg_resume);
db_batch_size = command_line::get_arg(vm, arg_batch_size);
if (command_line::get_arg(vm, command_line::arg_help))
@ -178,23 +187,41 @@ int main(int argc, char* argv[])
{
LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha);
}
LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha);
LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
fake_core c(src_folder, opt_testnet);
height = c.m_storage.get_current_blockchain_height();
if (! num_blocks || num_blocks > height)
end_block = height - 1;
else
end_block = start_block + num_blocks - 1;
BlockchainDB *blockchain;
blockchain = new BlockchainLMDB(opt_batch);
dest_folder /= blockchain->get_db_name();
LOG_PRINT_L0("Source blockchain: " << src_folder);
LOG_PRINT_L0("Dest blockchain: " << dest_folder.string());
LOG_PRINT_L0("Opening LMDB: " << dest_folder.string());
LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")");
blockchain->open(dest_folder.string());
LOG_PRINT_L0("Source blockchain height: " << height);
LOG_PRINT_L0("Dest blockchain height: " << blockchain->height());
if (opt_resume)
// next block number to add is same as current height
start_block = blockchain->height();
if (! num_blocks || (start_block + num_blocks > height))
end_block = height - 1;
else
end_block = start_block + num_blocks - 1;
LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " <<
end_block+1);
if (start_block > end_block)
{
LOG_PRINT_L0("Finished: no blocks to add");
delete blockchain;
return 0;
}
if (opt_batch)
blockchain->batch_start();
@ -247,7 +274,7 @@ int main(int argc, char* argv[])
catch (const std::exception& e)
{
std::cout << ENDL;
std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL;
std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL;
delete blockchain;
return 2;
}

36
src/blockchain_converter/blockchain_export.cpp
View File

@ -148,7 +148,7 @@ void BlockchainExport::write_block(block& block)
#if SOURCE_DB == DB_MEMORY
const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash);
#else
transaction cb_tx_full = m_blockchain_storage->get_db()->get_tx(coinbase_tx_hash);
transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash);
#endif
#if SOURCE_DB == DB_MEMORY
@ -167,7 +167,7 @@ void BlockchainExport::write_block(block& block)
#if SOURCE_DB == DB_MEMORY
const transaction* tx = m_blockchain_storage->get_tx(tx_id);
#else
transaction tx = m_blockchain_storage->get_db()->get_tx(tx_id);
transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
#endif
#if SOURCE_DB == DB_MEMORY
@ -205,18 +205,18 @@ void BlockchainExport::write_block(block& block)
#if SOURCE_DB == DB_MEMORY
size_t block_size = m_blockchain_storage->get_block_size(block_height);
#else
size_t block_size = m_blockchain_storage->get_db()->get_block_size(block_height);
size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height);
#endif
#if SOURCE_DB == DB_MEMORY
difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height);
#else
difficulty_type cumulative_difficulty = m_blockchain_storage->get_db()->get_block_cumulative_difficulty(block_height);
difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
#endif
#if SOURCE_DB == DB_MEMORY
uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height);
#else
// TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number
uint64_t coins_generated = m_blockchain_storage->get_db()->get_block_already_generated_coins(block_height);
uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
#endif
*m_raw_archive << block_size;
@ -239,14 +239,16 @@ bool BlockchainExport::BlockchainExport::close()
#if SOURCE_DB == DB_MEMORY
bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
#else
bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
#endif
{
uint64_t use_block_height2 = 0;
uint64_t block_height = 0;
m_blockchain_storage = _blockchain_storage;
m_tx_pool = _tx_pool;
uint64_t progress_interval = 100;
std::string refresh_string = "\r \r";
LOG_PRINT_L0("Storing blocks raw data...");
if (!BlockchainExport::open(output_dir))
{
@ -255,15 +257,15 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_
}
block b;
LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height());
LOG_PRINT_L0("requested block height: " << use_block_height);
if ((use_block_height > 0) && (use_block_height < m_blockchain_storage->get_current_blockchain_height()))
use_block_height2 = use_block_height;
LOG_PRINT_L0("requested block height: " << requested_block_height);
if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height()))
block_height = requested_block_height;
else
{
use_block_height2 = m_blockchain_storage->get_current_blockchain_height();
LOG_PRINT_L0("using block height: " << use_block_height2);
block_height = m_blockchain_storage->get_current_blockchain_height();
LOG_PRINT_L0("Using block height of source blockchain: " << block_height);
}
for (height=0; height < use_block_height2; ++height)
for (height=0; height < block_height; ++height)
{
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height);
m_blockchain_storage->get_block_by_hash(hash, b);
@ -271,11 +273,17 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_
if (height % NUM_BLOCKS_PER_CHUNK == 0) {
flush_chunk();
}
if (height % progress_interval == 0) {
std::cout << refresh_string;
std::cout << "height " << height << "/" << block_height << std::flush;
}
}
if (height % NUM_BLOCKS_PER_CHUNK != 0)
{
flush_chunk();
}
std::cout << refresh_string;
std::cout << "height " << height << "/" << block_height << ENDL;
LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes");
return BlockchainExport::close();

41
src/blockchain_converter/blockchain_import.cpp
View File

@ -57,6 +57,8 @@ static bool opt_testnet = true;
// adjustable through command-line argument according to available RAM
static uint64_t db_batch_size = 20000;
static std::string refresh_string = "\r \r";
namespace po = boost::program_options;
@ -146,7 +148,7 @@ int count_blocks(std::string& import_file_path)
int chunk_size;
import_file.read(buffer1, STR_LENGTH_OF_INT);
if (!import_file) {
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L1("End of import file reached");
quit = true;
break;
@ -154,7 +156,7 @@ int count_blocks(std::string& import_file_path)
h += NUM_BLOCKS_PER_CHUNK;
if (h % progress_interval == 0)
{
std::cout << "\r \r" << "block height: " << h <<
std::cout << refresh_string << "block height: " << h <<
std::flush;
}
bytes_read += STR_LENGTH_OF_INT;
@ -162,32 +164,33 @@ int count_blocks(std::string& import_file_path)
chunk_size = atoi(buffer1);
if (chunk_size > BUFFER_SIZE)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
<< " height: " << h);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > 100000)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: "
<< h);
}
else if (chunk_size <= 0) {
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h);
throw std::runtime_error("Aborting");
}
// skip to next expected block size value
import_file.seekg(chunk_size, std::ios_base::cur);
if (! import_file) {
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: "
<< import_file.gcount() << " of chunk_size " << chunk_size);
throw std::runtime_error("Aborting");
}
bytes_read += chunk_size;
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L3("Total bytes scanned: " << bytes_read);
}
@ -214,7 +217,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
// Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block.
// This aligns internal db counts with importer counts.
simple_core.m_storage.get_db()->reset_stats();
simple_core.m_storage.get_db().reset_stats();
}
#endif
boost::filesystem::path raw_file_path(import_file_path);
@ -285,7 +288,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
int chunk_size;
import_file.read(buffer1, STR_LENGTH_OF_INT);
if (! import_file) {
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("End of import file reached");
quit = 1;
break;
@ -340,7 +343,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
h++;
if (h % display_interval == 0)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L0("loading block height " << h);
}
else
@ -352,7 +355,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
}
catch (const std::exception& e)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h);
quit = 1;
break;
@ -361,7 +364,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
if (h % progress_interval == 0)
{
std::cout << "\r \r" << "block " << h-1
std::cout << refresh_string << "block " << h-1
<< std::flush;
}
@ -374,7 +377,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
}
catch (const std::exception& e)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h);
quit = 1;
break;
@ -469,7 +472,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
a >> cumulative_difficulty;
a >> coins_generated;
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_L2("block_size: " << block_size);
LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty);
LOG_PRINT_L2("coins_generated: " << coins_generated);
@ -480,7 +483,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
}
catch (const std::exception& e)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what());
quit = 2; // make sure we don't commit partial block data
break;
@ -491,13 +494,13 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
{
if (h % db_batch_size == 0)
{
std::cout << "\r \r";
std::cout << refresh_string;
std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL;
simple_core.batch_stop();
simple_core.batch_start();
std::cout << ENDL;
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
simple_core.m_storage.get_db()->show_stats();
simple_core.m_storage.get_db().show_stats();
#endif
}
}
@ -505,7 +508,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
}
catch (const std::exception& e)
{
std::cout << "\r \r";
std::cout << refresh_string;
LOG_PRINT_RED_L0("exception while reading from import file, height=" << h);
return 2;
}
@ -525,7 +528,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
simple_core.batch_stop();
}
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
simple_core.m_storage.get_db()->show_stats();
simple_core.m_storage.get_db().show_stats();
#endif
if (h > 0)
LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1);

8
src/blockchain_converter/fake_core.h
View File

@ -55,7 +55,7 @@ struct fake_core_lmdb
m_pool.init(path.string());
m_storage.init(path.string(), use_testnet, mdb_flags);
if (do_batch)
m_storage.get_db()->set_batch_transactions(do_batch);
m_storage.get_db().set_batch_transactions(do_batch);
support_batch = true;
support_add_block = true;
}
@ -71,17 +71,17 @@ struct fake_core_lmdb
, const std::vector<transaction>& txs
)
{
return m_storage.get_db()->add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
}
void batch_start()
{
m_storage.get_db()->batch_start();
m_storage.get_db().batch_start();
}
void batch_stop()
{
m_storage.get_db()->batch_stop();
m_storage.get_db().batch_stop();
}
};

4
src/cryptonote_core/blockchain.h
View File

@ -145,9 +145,9 @@ namespace cryptonote
void set_enforce_dns_checkpoints(bool enforce);
bool update_checkpoints(const std::string& file_path, bool check_dns);
BlockchainDB* get_db()
BlockchainDB& get_db()
{
return m_db;
return *m_db;
}
private: