Skip to content

Commit

Permalink
sync.js fixes (#1)
Browse files Browse the repository at this point in the history 
* Testing commit to new branch
* Correct function variable token
* config.js to hold file read/write configs.
* forgot require for testing.
* fs include for config.js
* call web3
* fix
* require
* test commit ignore
* test
* test
* exported syncChain function
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test new sync memory json file
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test for 0
* test
* test
* test
* test
* test
* test
* Last doub insert test
* test
* test
* New vars
* broekn func
* rest
* block data
* limit test
* minus dwa
* minus jeden
* quite stuff
* fix brackets
* fix
* test
* quite
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* test
* Patcher test
* test
* patch test
* test
* patcher test
* test
* test
* test
* fix startBlock config
* patcher test
* test
* test
* test
* FInal patcher test
* Patcher Test #2,000,000
* patch test #2,000,001
* test # two million seven hundred thousand four hundred seventy five
* test # two million seven hundred thousand four hundred seventy six
* test # two million seven hundred thousand four hundred seventy seve
* test # two million seven hundred thousand four hundred seventy eight
* test # two million seven hundred thousand four hundred seventy nine
* test # two million seven hundred thousand four hundred eighty
* test # two million seven hundred thousand four hundred eighty one
* test # two million seven hundred thousand four hundred eighty two
* test # two million seven hundred thousand four hundred eighty three
* Default conf
  • Loading branch information
Ethernode.io authored Apr 23, 2018
1 parent 447fb89 commit fb66875
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 126 deletions.
23 changes: 12 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,25 +31,26 @@ Configuration file: `/conf.json`
Basic settings:
```json
{
"nodeAddr": "localhost",
"gethPort": 8545,
"startBlock": 0,
"endBlock": 0,
"lastSynced": 0,
"syncAll": true,
"patch": false
"quiet": false,
"nodeAddr": "localhost",
"gethPort": 8545,
"startBlock": 0,
"endBlock": "latest",
"quiet": true,
"syncAll": true,
"patch": false,
"patchBlocks": 0
}
```

```nodeAddr``` Your node API RPC address.
```gethPort``` Your node API RPC port.
```startBLock``` This is the start block of the blockchain, should always be 0 if you want to sync the whole ETC blockchain.
```endBlock```` This is usually the 'latest'/'newest' block in the blockchain, this value gets updated automatically, and will be used to patch missing blocks if the whole app goes down.
```lastSynced``` If `syncAll` is set to true, the sync will record the last block it has written to the db, this will be automatically updated and used when sync.js prematurely exits.
```syncAll``` If this is set to true at the start of the app, the sync will start syncing all blocks from lastSync, and if lastSync is 0 it will start from whatever the endBlock or latest block in the blockchain is.
```patch``` This is not used yet but it will be directly corolated with the startBlock param.
```quiet``` Prints out the log of what it is doing, Might not work for all messages in this release
```syncAll``` If this is set to true at the start of the app, the sync will start syncing all blocks from lastSync, and if lastSync is 0 it will start from whatever the endBlock or latest block in the blockchain is.
```patch``` If set to true and below value is set, sync will iterated through the # of blocks specified
```patchBlocks``` If `patch` is set to true, the amount of block specified will be check from the latest one.


### Run:
If you run
Expand Down
6 changes: 3 additions & 3 deletions conf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
"gethPort": 8545,
"startBlock": 0,
"endBlock": "latest",
"lastSynced": 0,
"quiet": false,
"quiet": true,
"syncAll": true,
"patch": false
"patch": false,
"patchBlocks": 0
}
239 changes: 127 additions & 112 deletions tools/sync.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
Name: Ethereum Blockchain syncer
Version: .0.0.1
Version: .0.0.2
This file will start syncing the blockchain from the node address you provide in the conf.json file.
Please read the README in the root directory that explains the parameters of this code
*/
Expand All @@ -19,90 +19,84 @@ var Transaction = mongoose.model( 'Transaction' );
//Just listen for latest blocks and sync from the start of the app.
**/
var listenBlocks = function(config) {
var options = {
logIndex: "Number",
fromBlock: "latest",
};
var newBlocks = web3.eth.filter("latest");
newBlocks.watch(function (error, blockHashOrNumber) {
newBlocks.watch(function (error,latestBlock) {
if(error) {
console.log('Error: ' + error);
} else if (blockHashOrNumber == null) {
} else if (latestBlock == null) {
console.log('Warning: null block hash');
} else {
console.log('Found new block: ' + blockHashOrNumber);
updatedEndBlock(config,blockHashOrNumber);
console.log('Found new block: ' + latestBlock);
if(web3.isConnected()) {
web3.eth.getBlock(blockHashOrNumber, true, function(error, blockData) {
web3.eth.getBlock(latestBlock, true, function(error,blockData) {
if(error) {
console.log('Warning: error on getting block with hash/number: ' + blockHashOrNumber + ': ' + error);
console.log('Warning: error on getting block with hash/number: ' + latestBlock + ': ' + error);
}else if(blockData == null) {
console.log('Warning: null block data received from the block with hash/number: ' + blockHashOrNumber);
console.log('Warning: null block data received from the block with hash/number: ' + latestBlock);
}else{
writeBlockToDB(config, blockData);
writeTransactionsToDB(config, blockData);
return;
}
});
}else{
console.log('Error: Web3 connection time out trying to get block ' + blockHashOrNumber + ' retrying connection now');
return;
console.log('Error: Web3 connection time out trying to get block ' + latestBlock + ' retrying connection now');
listenBlocks(config);
}
}
});
// Starts full sync when set to true in config
if (config.syncAll === true){
console.log('Starting Full Sync');
getOldestBlockDB(config);
}
// Starts full sync when set to true in config
if (config.patch === true){
console.log('Checking for missing blocks');
runPatcher(config);
}
}
/**
If full sync is checked this function will start syncing the block chain from lastSynced param see README
**/
var syncChain = function(config, web3, blockHashOrNumber) {
if(blockHashOrNumber == undefined) {
blockHashOrNumber = config.endBlock
}
var syncChain = function(config,web3,nextBlock){
if(web3.isConnected()) {
web3.eth.getBlock(blockHashOrNumber, true, function(error, blockData) {
web3.eth.getBlock(nextBlock, true, function(error,blockData) {
if(error) {
console.log('Warning: error on getting block with hash/number: ' + blockHashOrNumber + ': ' + error);
console.log('Warning: error on getting block with hash/number: ' + nextBlock + ': ' + error);
getOldestBlockDB();
}else if(blockData == null) {
console.log('Warning: null block data received from the block with hash/number: ' + blockHashOrNumber);
}else{
if(config.lastSynced === 0){
console.log('No last full sync record found, start from block: latest');
writeBlockToDB(config, blockData);
writeTransactionsToDB(config, blockData);
var lastSync = blockData.number;
updateLastSynced(config, lastSync);
}else{
console.log('Found last full sync record: ' + config.lastSynced);
writeBlockToDB(config, blockData);
writeTransactionsToDB(config, blockData);
var lastSync = config.lastSynced - 1;
updateLastSynced(config, lastSync);
}
console.log('Warning: null block data received from the block with hash/number: ' + nextBlock);
getOldestBlockDB();
}else{
writeBlockToDB(config, blockData);
writeTransactionsToDB(config, blockData);
}
});
}else{
console.log('Error: Web3 connection time out trying to get block ' + blockHashOrNumber + ' retrying connection now');
syncChain(config, web3, blockHashOrNumber);
console.log('Error: Web3 connection time out trying to get block ' + nextBlock + ' retrying connection now');
syncChain(config,web3,nextBlock);
}
};
}
/**
Write the whole block object to DB
**/
var writeBlockToDB = function(config, blockData) {
return new Block(blockData).save( function( err, block, count ){
if ( typeof err !== 'undefined' && err ) {
if (err.code == 11000) {
if(!('quiet' in config && config.quiet === true)) {
console.log('Skip: Duplicate key ' + blockData.number.toString() + ': ' + err);
}
} else {
console.log('Error: Aborted due to error on ' + 'block number ' + blockData.number.toString() + ': ' + err);
process.exit(9);
}
} else {
if (err.code == 11000) {
if(!('quiet' in config && config.quiet === true)) {
console.log('DB successfully written for block number ' + blockData.number.toString());
console.log('Skip: Duplicate key ' + blockData.number.toString() + ': ' +err);
}
}else{
console.log('Error: Aborted due to error on ' + 'block number ' + blockData.number.toString() + ': ' + err);
process.exit(9);
}
}else{
console.log('DB successfully written for block number ' + blockData.number.toString() );
// continues sync if flag is still true
if (config.syncAll === true){
getOldestBlockDB(config);
}
}
});
}
Expand All @@ -121,98 +115,119 @@ var writeTransactionsToDB = function(config, blockData) {
Transaction.collection.insert(bulkOps, function( err, tx ){
if ( typeof err !== 'undefined' && err ) {
if (err.code == 11000) {
console.log('Skip: Duplicate key ' + err);
} else {
console.log('Error: Aborted due to error: ' + err);
process.exit(9);
}
} else if(!('quiet' in config && config.quiet === true)) {
console.log(blockData.transactions.length.toString() + ' transactions recorded for Block# ' + blockData.number.toString());
if(!('quiet' in config && config.quiet === true)) {
console.log('Skip: Duplicate key ' + err);
}
}else{
console.log('Error: Aborted due to error: ' + err);
getOldestBlockDB();
//process.exit(9);
}
}else{
console.log(blockData.transactions.length.toString() + ' transactions recorded for Block# ' + blockData.number.toString());
// continues sync if flag is still true
if (config.syncAll === true){
getOldestBlockDB(config);
}
}
});
}
}
var checkBlockDBExistsThenWrite = function(config, blockData) {
Block.find({number: blockData.number}, function (err, b) {
if (!b.length){
writeBlockToDB(config, blockData);
writeTransactionsToDB(config, blockData);
}else if(!('quiet' in config && config.quiet === true)) {
console.log('Block number: ' + blockData.number.toString() + ' already exists in DB.');
listenBlocks(config);
}
});
};
var updatedEndBlock = function(config,lastBlock){
var configFile = '../conf.json';
var file = require(configFile);

file.endBlock = lastBlock;

fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
if (err) return console.log(err);
//console.log('Wirting new Synced Block ' + lastBlock + ' to ' + configFile);
});
};
/**
Take the last block the grabber exited on and update the param 'end' in the config.JSON
//Check oldest block in db and start sync from tehre
**/
var updateLastSynced = function(config,lastSync){
var configFile = '../conf.json';
var file = require(configFile);

file.lastSynced = lastSync;
config.lastSynced = lastSync;

fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
if (err) return console.log(err);
//console.log('writing block ' + lastSync + ' to ' + configFile);

if (config.lastSynced === config.startBlock){
config.syncAll = false;
file.syncAll = false;
fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
if (err) return console.log(err);
});
var getOldestBlockDB = function() {
var oldBlockFind = Block.find({}, "number").lean(true).sort('number').limit(1);
oldBlockFind.exec(function (err, docs) {
if(docs.length < 1){
console.log('nothing here starting from latest');
}else{
syncChain(config, web3, config.lastSynced);
var nextBlock = (docs[0].number - 1);
if(nextBlock == config.startBlock){
console.log('Sync Finsihed');
config.syncAll = false;
return;
}else{
syncChain(config,web3,nextBlock);
}
}
});
}
/**
Block Patcher(experimental)
**/
var runPatcher = function(config) {
currentBlock = web3.eth.blockNumber;
patchBlock = currentBlock - config.patchBlocks;
console.log('Starting patching from block: '+patchBlock);
while(config.patchBlocks > 0){
config.patchBlocks--;
patchBlock++;
if(!('quiet' in config && config.quiet === true)) {
console.log('Patching Block: '+patchBlock)
}
web3.eth.getBlock(patchBlock, true, function(error,patchData) {
if(error) {
console.log('Warning: error on getting block with hash/number: ' + patchBlock + ': ' + error);
//getOldestBlockDB();
}else if(patchData == null) {
console.log('Warning: null block data received from the block with hash/number: ' + patchBlock);
//getOldestBlockDB();
}else{
checkBlockDBExistsThenWrite(config,patchData)
}
});
if (config.patchBlocks == 0){
config.patch = false;
console.log('Block Patching Complete')
}
}
}
/**
This will be used for the patcher(experimental)
**/
var checkBlockDBExistsThenWrite = function(config,patchData) {
Block.find({number: patchData.number}, function (err, b) {
if (!b.length){
writeBlockToDB(config,patchData);
writeTransactionsToDB(config,patchData);
}else if(!('quiet' in config && config.quiet === true)) {
console.log('Block number: ' +patchData.number.toString() + ' already exists in DB.');
}
});
};
/**
Start config for node connection and sync
**/
var config = {};
// set the default NODE address to localhost if it's not provided
if (!('nodeAddr' in config) || !(config.nodeAddr)) {
config.nodeAddr = 'localhost'; // default
config.nodeAddr = 'localhost'; // default
}
// set the default geth port if it's not provided
if (!('gethPort' in config) || (typeof config.gethPort) !== 'number') {
config.gethPort = 8545; // default
config.gethPort = 8545; // default
}
// set the default output directory if it's not provided
if (!('output' in config) || (typeof config.output) !== 'string') {
config.output = '.'; // default this directory
config.output = '.'; // default this directory
}
//Look for config.json file if not
try {
var configContents = fs.readFileSync('conf.json');
config = JSON.parse(configContents);
console.log('CONFIG FOUND: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
// Sets address for RPC WEb3 to connect to, usually your node address defaults ot localhost
var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString()));
if (config.syncAll === true){
syncChain(config,web3);
}
}
catch (error) {
if (error.code === 'ENOENT') {
console.log('No config file found. Using default configuration: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
}
else {
throw error;
process.exit(1);
}
if (error.code === 'ENOENT') {
console.log('No config file found. Using default configuration: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
}
else {
throw error;
process.exit(1);
}
}
// Sets address for RPC WEB3 to connect to, usually your node IP address defaults ot localhost
var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString()));
// Start listening for latest blocks
listenBlocks(config);

0 comments on commit fb66875

Please sign in to comment.