Fix/1805

src/burnchains/bitcoin/indexer.rs

@@ -30,22 +30,27 @@ use crate::burnchains::bitcoin::blocks::BitcoinHeaderIPC;
 use crate::burnchains::bitcoin::messages::BitcoinMessageHandler;
 use crate::burnchains::bitcoin::spv::*;
 use crate::burnchains::bitcoin::Error as btc_error;
+use crate::burnchains::db::BurnchainHeaderReader;
 use crate::burnchains::indexer::BurnchainIndexer;
 use crate::burnchains::indexer::*;
 use crate::burnchains::Burnchain;
+use crate::util_lib::db::Error as DBError;
 
 use crate::burnchains::bitcoin::blocks::{BitcoinBlockDownloader, BitcoinBlockParser};
 use crate::burnchains::bitcoin::BitcoinNetworkType;
 
+use crate::burnchains::BurnchainBlockHeader;
 use crate::burnchains::Error as burnchain_error;
 use crate::burnchains::MagicBytes;
 use crate::burnchains::BLOCKSTACK_MAGIC_MAINNET;
 use crate::types::chainstate::BurnchainHeaderHash;
 
-use stacks_common::deps_common::bitcoin::blockdata::block::LoneBlockHeader;
+use stacks_common::deps_common::bitcoin::blockdata::block::{BlockHeader, LoneBlockHeader};
+use stacks_common::deps_common::bitcoin::network::encodable::VarInt;
 use stacks_common::deps_common::bitcoin::network::message::NetworkMessage;
 use stacks_common::deps_common::bitcoin::network::serialize::BitcoinHash;
 use stacks_common::deps_common::bitcoin::network::serialize::Error as btc_serialization_err;
+use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash;
 use stacks_common::util::get_epoch_time_secs;
 use stacks_common::util::log;
 
@@ -54,7 +59,7 @@ use crate::core::{
 };
 use std::convert::TryFrom;
 
-pub const USER_AGENT: &'static str = "Stacks/2.0";
+pub const USER_AGENT: &'static str = "Stacks/2.1";
 
 pub const BITCOIN_MAINNET: u32 = 0xD9B4BEF9;
 pub const BITCOIN_TESTNET: u32 = 0x0709110B;
@@ -154,6 +159,22 @@ impl BitcoinIndexerConfig {
         }
     }
 
+    pub fn default_regtest(spv_headers_path: String) -> BitcoinIndexerConfig {
+        BitcoinIndexerConfig {
+            peer_host: "127.0.0.1".to_string(),
+            peer_port: 18444,
+            rpc_port: 18443,
+            rpc_ssl: false,
+            username: Some("blockstack".to_string()),
+            password: Some("blockstacksystem".to_string()),
+            timeout: 30,
+            spv_headers_path: spv_headers_path,
+            first_block: 0,
+            magic_bytes: BLOCKSTACK_MAGIC_MAINNET.clone(),
+            epochs: None,
+        }
+    }
+
     #[cfg(test)]
     pub fn test_default(spv_headers_path: String) -> BitcoinIndexerConfig {
         BitcoinIndexerConfig {
@@ -197,6 +218,34 @@ impl BitcoinIndexer {
         }
     }
 
+    #[cfg(test)]
+    pub fn new_unit_test(working_dir: &str) -> BitcoinIndexer {
+        let mut working_dir_path = PathBuf::from(working_dir);
+        if fs::metadata(&working_dir_path).is_err() {
+            fs::create_dir_all(&working_dir_path).unwrap();
+        }
+
+        working_dir_path.push("headers.sqlite");
+
+        // instantiate headers DB
+        let _ = SpvClient::new(
+            &working_dir_path.to_str().unwrap().to_string(),
+            0,
+            None,
+            BitcoinNetworkType::Regtest,
+            true,
+            false,
+        )
+        .unwrap();
+
+        BitcoinIndexer {
+            config: BitcoinIndexerConfig::default_regtest(
+                working_dir_path.to_str().unwrap().to_string(),
+            ),
+            runtime: BitcoinIndexerRuntime::new(BitcoinNetworkType::Regtest),
+        }
+    }
+
     pub fn dup(&self) -> BitcoinIndexer {
         BitcoinIndexer {
             config: self.config.clone(),
@@ -797,6 +846,47 @@ impl BitcoinIndexer {
         Ok(new_tip)
     }
 
+    #[cfg(test)]
+    pub fn raw_store_header(&mut self, header: BurnchainBlockHeader) -> Result<(), btc_error> {
+        let mut spv_client = SpvClient::new(
+            &self.config.spv_headers_path,
+            self.config.first_block,
+            None,
+            self.runtime.network_id,
+            true,
+            false,
+        )?;
+        spv_client.disable_check_txcount();
+
+        let hdr = LoneBlockHeader {
+            header: BitcoinIndexer::mock_bitcoin_header(
+                &header.parent_block_hash,
+                header.timestamp as u32,
+            ),
+            tx_count: VarInt(header.num_txs),
+        };
+
+        assert!(header.block_height > 0);
+        let start_height = header.block_height - 1;
+        spv_client.insert_block_headers_after(start_height, vec![hdr])?;
+        Ok(())
+    }
+
+    #[cfg(test)]
+    pub fn mock_bitcoin_header(
+        parent_block_hash: &BurnchainHeaderHash,
+        timestamp: u32,
+    ) -> BlockHeader {
+        BlockHeader {
+            bits: 0,
+            merkle_root: Sha256dHash([0u8; 32]),
+            nonce: 0,
+            prev_blockhash: parent_block_hash.to_bitcoin_hash(),
+            time: timestamp,
+            version: 0x20000000,
+        }
+    }
+
     /// Verify that the last block header we have is within 2 hours of now.
     /// Return burnchain_error::TrySyncAgain if not, and delete the offending header
     pub fn check_chain_tip_timestamp(&mut self) -> Result<(), burnchain_error> {
@@ -843,7 +933,7 @@ impl BurnchainIndexer for BitcoinIndexer {
 
     /// Connect to the Bitcoin peer network.
     /// Use the peer host and peer port given in the config file,
-    /// and loaded in on setup.  Don't call this before init().
+    /// and loaded in on setup.
     fn connect(&mut self) -> Result<(), burnchain_error> {
         self.reconnect_peer().map_err(burnchain_error::Bitcoin)
     }
@@ -1042,6 +1132,39 @@ impl BurnchainIndexer for BitcoinIndexer {
     fn parser(&self) -> BitcoinBlockParser {
         BitcoinBlockParser::new(self.runtime.network_id, self.config.magic_bytes)
     }
+
+    fn reader(&self) -> BitcoinIndexer {
+        self.dup()
+    }
+}
+
+impl BurnchainHeaderReader for BitcoinIndexer {
+    fn read_burnchain_headers(
+        &self,
+        start_height: u64,
+        end_height: u64,
+    ) -> Result<Vec<BurnchainBlockHeader>, DBError> {
+        let hdrs = self
+            .read_headers(start_height, end_height)
+            .map_err(|e| DBError::Other(format!("Burnchain error: {:?}", &e)))?;
+
+        Ok(hdrs
+            .into_iter()
+            .map(|hdr| BurnchainBlockHeader {
+                block_height: hdr.block_height,
+                block_hash: BurnchainHeaderHash::from_bitcoin_hash(&Sha256dHash(hdr.header_hash())),
+                parent_block_hash: BurnchainHeaderHash::from_bitcoin_hash(
+                    &hdr.block_header.header.prev_blockhash,
+                ),
+                num_txs: hdr.block_header.tx_count.0,
+                timestamp: hdr.block_header.header.time as u64,
+            })
+            .collect())
+    }
+    fn get_burnchain_headers_height(&self) -> Result<u64, DBError> {
+        self.get_headers_height()
+            .map_err(|e| DBError::Other(format!("Burnchain error: {:?}", &e)))
+    }
 }
 
 #[cfg(test)]

src/burnchains/bitcoin/spv.rs

@@ -38,6 +38,8 @@ use crate::burnchains::bitcoin::BitcoinNetworkType;
 use crate::burnchains::bitcoin::Error as btc_error;
 use crate::burnchains::bitcoin::PeerMessage;
 
+use stacks_common::types::chainstate::BurnchainHeaderHash;
+
 use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
 use rusqlite::OptionalExtension;
 use rusqlite::Row;
@@ -54,15 +56,15 @@ use stacks_common::util::log;
 
 const BLOCK_HEADER_SIZE: u64 = 81;
 
-const BITCOIN_GENESIS_BLOCK_HASH_MAINNET: &'static str =
+pub const BITCOIN_GENESIS_BLOCK_HASH_MAINNET: &'static str =
     "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f";
-const BITCOIN_GENESIS_BLOCK_MERKLE_ROOT_MAINNET: &'static str =
+pub const BITCOIN_GENESIS_BLOCK_MERKLE_ROOT_MAINNET: &'static str =
     "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b";
 
-const BITCOIN_GENESIS_BLOCK_HASH_TESTNET: &'static str =
+pub const BITCOIN_GENESIS_BLOCK_HASH_TESTNET: &'static str =
     "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943";
 
-const BITCOIN_GENESIS_BLOCK_HASH_REGTEST: &'static str =
+pub const BITCOIN_GENESIS_BLOCK_HASH_REGTEST: &'static str =
     "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206";
 
 pub const BLOCK_DIFFICULTY_CHUNK_SIZE: u64 = 2016;
@@ -105,6 +107,7 @@ pub struct SpvClient {
     readwrite: bool,
     reverse_order: bool,
     headers_db: DBConn,
+    check_txcount: bool,
 }
 
 impl FromColumn<Sha256dHash> for Sha256dHash {
@@ -142,6 +145,7 @@ impl SpvClient {
         readwrite: bool,
         reverse_order: bool,
     ) -> Result<SpvClient, btc_error> {
+        let exists = fs::metadata(headers_path).is_ok();
         let conn = SpvClient::db_open(headers_path, readwrite, true)?;
         let mut client = SpvClient {
             headers_path: headers_path.to_owned(),
@@ -152,9 +156,10 @@ impl SpvClient {
             readwrite: readwrite,
             reverse_order: reverse_order,
             headers_db: conn,
+            check_txcount: true,
         };
 
-        if readwrite {
+        if readwrite && !exists {
             client.init_block_headers(true)?;
         }
 
@@ -180,6 +185,7 @@ impl SpvClient {
             readwrite: readwrite,
             reverse_order: reverse_order,
             headers_db: conn,
+            check_txcount: true,
         };
 
         if readwrite {
@@ -189,6 +195,11 @@ impl SpvClient {
         Ok(client)
     }
 
+    #[cfg(test)]
+    pub fn disable_check_txcount(&mut self) {
+        self.check_txcount = false;
+    }
+
     pub fn conn(&self) -> &DBConn {
         &self.headers_db
     }
@@ -487,15 +498,21 @@ impl SpvClient {
     fn validate_header_integrity(
         start_height: u64,
         headers: &Vec<LoneBlockHeader>,
+        check_txcount: bool,
     ) -> Result<(), btc_error> {
         if headers.len() == 0 {
             return Ok(());
         }
 
-        for i in 0..headers.len() {
-            if headers[i].tx_count != VarInt(0) {
-                warn!("Non-zero tx count on header offset {}", i);
-                return Err(btc_error::InvalidReply);
+        if check_txcount {
+            for i in 0..headers.len() {
+                if headers[i].tx_count != VarInt(0) {
+                    warn!(
+                        "Non-zero tx count on header offset {}: {:?}",
+                        i, &headers[i].tx_count
+                    );
+                    return Err(btc_error::InvalidReply);
+                }
             }
         }
 
@@ -694,6 +711,13 @@ impl SpvClient {
             &header.nonce,
             &u64_to_sql(height)?,
         ];
+
+        test_debug!(
+            "SPV: insert header {} {}: {:?}",
+            height,
+            &header.bitcoin_hash(),
+            &header
+        );
         tx.execute(sql, args)
             .map_err(|e| btc_error::DBError(db_error::SqliteError(e)))
             .and_then(|_x| Ok(()))
@@ -833,7 +857,7 @@ impl SpvClient {
 
     /// Write a run of continuous headers to a particular location.
     /// Does _not_ check for continuity!
-    fn write_block_headers(
+    pub fn write_block_headers(
         &mut self,
         height: u64,
         headers: Vec<LoneBlockHeader>,
@@ -886,10 +910,11 @@ impl SpvClient {
             start_height
         );
 
-        SpvClient::validate_header_integrity(start_height, &block_headers).map_err(|e| {
-            error!("Received invalid headers: {:?}", &e);
-            e
-        })?;
+        SpvClient::validate_header_integrity(start_height, &block_headers, self.check_txcount)
+            .map_err(|e| {
+                error!("Received invalid headers: {:?}", &e);
+                e
+            })?;
 
         let parent_header = match self.read_block_header(start_height)? {
             Some(header) => header,
@@ -939,10 +964,11 @@ impl SpvClient {
             end_height
         );
 
-        SpvClient::validate_header_integrity(start_height, &block_headers).map_err(|e| {
-            error!("Received invalid headers: {:?}", &e);
-            e
-        })?;
+        SpvClient::validate_header_integrity(start_height, &block_headers, self.check_txcount)
+            .map_err(|e| {
+                error!("Received invalid headers: {:?}", &e);
+                e
+            })?;
 
         match self.read_block_header(end_height)? {
             Some(child_header) => {