diff --git a/Cargo.lock b/Cargo.lock index c5361cc3..f099a9c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3222,9 +3222,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "packx" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47bfbeb8a641f959d048c4adf9d66d602e35791061ee003920b6b41e2253ecc1" +checksum = "542e591f348a33c1b32aa12ad32f4f132d1e0045bf8ccec285d4a61d0861b39b" dependencies = [ "blake3", "bytemuck", @@ -6761,6 +6761,7 @@ dependencies = [ "mime_guess", "num_cpus", "num_enum", + "packx", "reqwest 0.12.15", "serde_json", "solana-client", diff --git a/Cargo.toml b/Cargo.toml index cdae84a3..c99a83e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ tape-network = { version = "0.2.1", path = "./network" } tape-program = { version = "0.2.1", path = "./program" } crankx = { version = "0.2.2", features = ["solana"] } -packx = { version = "0.3.1", features = ["solana"] } +packx = { version = "0.4.1", features = ["solana"] } # program dependencies bytemuck = "1.14.3" @@ -87,3 +87,12 @@ http-body-util = "0.1.3" [profile.release] overflow-checks = true + +[profile.dev] +overflow-checks = true +opt-level = 3 +debug = false + +[profile.test] +opt-level = 3 +debug = false diff --git a/api/src/consts.rs b/api/src/consts.rs index 01a1db00..f052b725 100644 --- a/api/src/consts.rs +++ b/api/src/consts.rs @@ -46,12 +46,12 @@ pub const TAPE_PROOF_LEN: usize = TAPE_TREE_HEIGHT; /// Segment size in bytes pub const SEGMENT_SIZE: usize = 128; /// Packed Segment size in bytes -pub const PACKED_SEGMENT_SIZE: usize = 152; // packx::SOLUTION_SIZE +pub const PACKED_SEGMENT_SIZE: usize = 145; // packx::SOLUTION_SIZE /// Maximum number of segments in a tape -pub const MAX_SEGMENTS_PER_TAPE: usize = 1 << SEGMENT_TREE_HEIGHT - 1; +pub const MAX_SEGMENTS_PER_TAPE: usize = 1 << (SEGMENT_TREE_HEIGHT - 1); /// Maximum number of tapes in a spool -pub const MAX_TAPES_PER_SPOOL: usize = 1 << TAPE_TREE_HEIGHT - 1; +pub const MAX_TAPES_PER_SPOOL: usize = 1 << (TAPE_TREE_HEIGHT - 1); // ==================================================================== // Token Economics diff --git a/api/src/types.rs b/api/src/types.rs index b17577c4..6eabaf7b 100644 --- a/api/src/types.rs +++ b/api/src/types.rs @@ -31,7 +31,7 @@ impl PoW { #[derive(Clone, Copy, Debug, Pod, Zeroable)] /// Proof-of-access solution for the tape segment, cryptographically tied to the miner using PackX. pub struct PoA { - pub bump: [u8; 8], + pub bump: u8, pub seed: [u8; 16], pub nonce: [u8; 128], pub path: ProofPath, @@ -48,7 +48,7 @@ impl PoA { } pub fn as_solution(&self) -> packx::Solution { - packx::Solution::new(self.seed, self.nonce, self.bump) + packx::Solution::new(self.bump, self.seed, self.nonce) } } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 992732b0..f9c5bb86 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,6 +19,7 @@ path = "src/main.rs" tape-api.workspace = true tape-client.workspace = true tape-network.workspace = true +packx.workspace = true anyhow.workspace = true chrono.workspace = true diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 6c82bb60..6861c99b 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -135,7 +135,10 @@ pub enum SnapshotCommands { Resync { #[arg(help = "Tape account public key to re-sync")] - tape: String, + tape_address: String, + + #[arg(help = "Miner account public key", short = 'm', long = "miner")] + miner_address: Option, }, Create { @@ -150,21 +153,27 @@ pub enum SnapshotCommands { GetTape { #[arg(help = "Tape account public key")] - tape: String, + tape_address: String, #[arg(short = 'o', long = "output", help = "Output file")] output: Option, #[arg(short = 'r', long = "raw", help = "Output raw segments instead of decoded tape")] raw: bool, + + #[arg(help = "Miner account public key", short = 'm', long = "miner")] + miner_address: Option, }, GetSegment { #[arg(help = "Tape account public key")] - tape: String, + tape_address: String, #[arg(help = "Segment index (0 to tape size - 1)")] index: u32, + + #[arg(help = "Miner account public key", short = 'm', long = "miner")] + miner_address: Option, }, } diff --git a/cli/src/commands/admin.rs b/cli/src/commands/admin.rs index dae74b01..39950337 100644 --- a/cli/src/commands/admin.rs +++ b/cli/src/commands/admin.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use dialoguer::{theme::ColorfulTheme, Confirm}; use solana_sdk::signer::Signer; use crate::cli::{Cli, Context, Commands}; @@ -14,15 +13,6 @@ use tape_client::{ pub async fn handle_admin_commands(cli:Cli, context: Context) -> Result<()> { log::print_divider(); - let proceed = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("→ Are you sure?") - .default(false) - .interact() - .map_err(|e| anyhow::anyhow!("Failed to get user input: {}", e))?; - if !proceed { - log::print_error("Write operation cancelled"); - return Ok(()); - } match cli.command { Commands::Init {} => { diff --git a/cli/src/commands/network.rs b/cli/src/commands/network.rs index 9ca765da..2632609e 100644 --- a/cli/src/commands/network.rs +++ b/cli/src/commands/network.rs @@ -1,14 +1,12 @@ use anyhow::{bail, Result}; use std::str::FromStr; use std::sync::Arc; -use dialoguer::{theme::ColorfulTheme, Confirm}; use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{signature::Keypair, signer::Signer, pubkey::Pubkey}; use tape_api::prelude::*; use tape_client::{register::register_miner, get_miner_account}; use tape_network::{ - //archive::archive_loop, archive, mine::mine_loop, web::web_loop, @@ -114,16 +112,6 @@ pub async fn handle_register( let (miner_address, _) = miner_pda(context.payer().pubkey(), to_name(&name)); - let proceed = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("→ Are you sure?") - .default(false) - .interact() - .map_err(|e| anyhow::anyhow!("Failed to get user input: {}", e))?; - if !proceed { - log::print_error("Write operation cancelled"); - return Ok(()); - } - register_miner(context.rpc(), context.payer(), &name).await?; log::print_section_header("Miner Registered"); diff --git a/cli/src/commands/snapshot.rs b/cli/src/commands/snapshot.rs index 82496344..b53ce11f 100644 --- a/cli/src/commands/snapshot.rs +++ b/cli/src/commands/snapshot.rs @@ -1,6 +1,7 @@ use std::env; use std::io::{self, Write}; use std::str::FromStr; +use std::sync::Arc; use anyhow::Result; use chrono::Utc; @@ -17,7 +18,9 @@ use tapedrive::{decode_tape, MimeType, TapeHeader}; use crate::cli::{Cli, Commands, Context, SnapshotCommands}; use crate::log; +use crate::network::get_or_create_miner; use crate::utils::write_output; +use packx; pub async fn handle_snapshot_commands(cli: Cli, context: Context) -> Result<()> { if let Commands::Snapshot(snapshot) = cli.command { @@ -25,8 +28,8 @@ pub async fn handle_snapshot_commands(cli: Cli, context: Context) -> Result<()> SnapshotCommands::Stats {} => { handle_stats(context)? } - SnapshotCommands::Resync { tape } => { - handle_resync(context, &tape).await? + SnapshotCommands::Resync { tape_address, miner_address } => { + handle_resync(context, tape_address, miner_address).await? } SnapshotCommands::Create { output } => { handle_create(context, output)? @@ -34,11 +37,11 @@ pub async fn handle_snapshot_commands(cli: Cli, context: Context) -> Result<()> SnapshotCommands::Load { input } => { handle_load(&input)? } - SnapshotCommands::GetTape { tape, output, raw } => { - handle_get_tape(context, &tape, output, raw).await? + SnapshotCommands::GetTape { tape_address, miner_address, output, raw } => { + handle_get_tape(context, tape_address, miner_address, output, raw).await? } - SnapshotCommands::GetSegment { tape, index } => { - handle_get_segment(context, &tape, index).await? + SnapshotCommands::GetSegment { tape_address, miner_address, index } => { + handle_get_segment(context, tape_address, miner_address, index).await? } } } @@ -55,13 +58,36 @@ fn handle_stats(context: Context) -> Result<()> { Ok(()) } -async fn handle_resync(context: Context, tape: &str) -> Result<()> { - let tape_pubkey: Pubkey = FromStr::from_str(tape)?; +async fn handle_resync( + context: Context, + tape_address: String, + miner_address: Option, + ) -> Result<()> { + let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?; + let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?; let starting_slot = tape_account.tail_slot; - let store = context.open_primary_store_conn()?; - log::print_message(&format!("Re-syncing tape: {tape}, please wait")); - sync_from_block(&store, context.rpc(), &tape_pubkey, starting_slot).await?; + let store = Arc::new(context.open_primary_store_conn()?); + + let miner_pubkey = get_or_create_miner( + context.rpc(), + context.payer(), + miner_address, + None, + false + ).await?; + log::print_message(&format!("Using miner address: {miner_pubkey}")); + + log::print_message(&format!("Re-syncing tape: {tape_address}, please wait")); + + sync_from_block( + &store, + context.rpc(), + &tape_pubkey, + &miner_pubkey, + starting_slot + ).await?; + log::print_message("Done"); Ok(()) } @@ -84,20 +110,34 @@ fn handle_load(input: &str) -> Result<()> { async fn handle_get_tape( context: Context, - tape: &str, + tape_address: String, + miner_address: Option, output: Option, raw: bool, ) -> Result<()> { - let tape_pubkey: Pubkey = FromStr::from_str(tape)?; + let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?; let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?; + let total_segments = tape_account.total_segments; let store = context.open_read_only_store_conn()?; + + let miner_pubkey = get_or_create_miner( + context.rpc(), + context.payer(), + miner_address, + None, + false + ).await?; + let miner_bytes = miner_pubkey.to_bytes(); + let mut data: Vec = Vec::with_capacity((total_segments as usize) * SEGMENT_SIZE); let mut missing: Vec = Vec::new(); for seg_idx in 0..total_segments { match store.get_segment(&tape_pubkey, seg_idx) { - Ok(seg) => { - data.extend_from_slice(&seg); + Ok(segment_data) => { + let solution = packx::Solution::from_bytes(&segment_data.try_into().unwrap()); + let segment = solution.unpack(&miner_bytes); + data.extend_from_slice(&segment); } Err(StoreError::SegmentNotFoundForAddress(..)) => { data.extend_from_slice(&[0u8; SEGMENT_SIZE]); @@ -130,8 +170,14 @@ async fn handle_get_tape( Ok(()) } -async fn handle_get_segment(context: Context, tape: &str, index: u32) -> Result<()> { - let tape_pubkey: Pubkey = FromStr::from_str(tape)?; +async fn handle_get_segment( + context: Context, + tape_address: String, + miner_address: Option, + index: u32 +) -> Result<()> { + + let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?; let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?; if (index as u64) >= tape_account.total_segments { anyhow::bail!( @@ -143,10 +189,22 @@ async fn handle_get_segment(context: Context, tape: &str, index: u32) -> Result< let store = context.open_read_only_store_conn()?; + let miner_pubkey = get_or_create_miner( + context.rpc(), + context.payer(), + miner_address, + None, + false + ).await?; + let miner_bytes = miner_pubkey.to_bytes(); + match store.get_segment(&tape_pubkey, index as u64) { - Ok(data) => { + Ok(segment_data) => { + let solution = packx::Solution::from_bytes(&segment_data.try_into().unwrap()); + let segment = solution.unpack(&miner_bytes); + let mut stdout = io::stdout(); - stdout.write_all(&data)?; + stdout.write_all(&segment)?; stdout.flush()?; } Err(StoreError::SegmentNotFoundForAddress(..)) => { diff --git a/cli/src/commands/write.rs b/cli/src/commands/write.rs index 275657cd..fff7d046 100644 --- a/cli/src/commands/write.rs +++ b/cli/src/commands/write.rs @@ -52,7 +52,7 @@ pub async fn handle_write_command(cli: Cli, context: Context) -> Result<()> { let mut header = TapeHeader::new(mime_type, compression_algo, encryption_algo, flags); let encoded = encode_tape(&data, &mut header)?; - let num_segments = (encoded.len() + SEGMENT_SIZE - 1) / SEGMENT_SIZE; + let num_segments = encoded.len().div_ceil(SEGMENT_SIZE); let chunks: Vec<_> = encoded.chunks(SAFE_SIZE).map(|c| c.to_vec()).collect(); let chunks_len = chunks.len(); diff --git a/client/src/utils/block.rs b/client/src/utils/block.rs index 5057a8cb..ceb1f562 100644 --- a/client/src/utils/block.rs +++ b/client/src/utils/block.rs @@ -150,6 +150,7 @@ fn merge_events_and_instructions( // Iterate over events and instructions in parallel for (event, instruction) in tape_block.events.iter().zip(&tape_block.instructions) { + match (event, instruction) { (EventData::Write(write_event), InstructionData::Write { address, data }) => { merge_write(write_event, address, data, &mut merged)?; @@ -160,6 +161,7 @@ fn merge_events_and_instructions( } (EventData::Finalize(finalize_event), InstructionData::Finalize { address }) => { + log::debug!("Merging event: {:?} with instruction: {:?}", event, instruction); merge_finalize(finalize_event, address, &mut merged)?; } diff --git a/network/benches/disk_size_bench.rs b/network/benches/disk_size_bench.rs index 77beb38a..032210cb 100644 --- a/network/benches/disk_size_bench.rs +++ b/network/benches/disk_size_bench.rs @@ -138,7 +138,7 @@ fn main() { let mut key = Vec::with_capacity(40); key.extend_from_slice(&hash); key.extend_from_slice(&suffix); - store.db.put_cf(&cf_handle, &key, &value).expect("Put failed"); + store.db.put_cf(&cf_handle, &key, value).expect("Put failed"); } } } else { @@ -146,7 +146,7 @@ fn main() { let key: [u8; 8] = rng.gen(); let mut value = [0u8; SEGMENT_SIZE]; rng.fill_bytes(&mut value); - store.db.put_cf(&cf_handle, &key, &value).expect("Put failed"); + store.db.put_cf(&cf_handle, key, value).expect("Put failed"); } } diff --git a/network/src/archive/helpers.rs b/network/src/archive/helpers.rs index 82338a66..8e56626e 100644 --- a/network/src/archive/helpers.rs +++ b/network/src/archive/helpers.rs @@ -9,25 +9,10 @@ pub fn sync_needed( tape_address: &Pubkey, total_segments: u64, ) -> Result { - let sector_count = store.get_sector_count(tape_address).unwrap_or(0); - let upper_bound = sector_count as u64 * SECTOR_LEAVES as u64; - if total_segments > upper_bound { - // More sectors are needed - Ok(true) - } else if total_segments > 0 && total_segments <= upper_bound { - // Check if the last sector has enough segments in it - let last_sector_index = sector_count.saturating_sub(1) as u64; - if let Ok(last_sector) = store.get_sector(tape_address, last_sector_index) { - let segments_in_last_sector = last_sector.count_segments() as u64; - let total_stored_segments = segments_in_last_sector + (last_sector_index * SECTOR_LEAVES as u64); - Ok(total_stored_segments < total_segments) - } else { - // Last sector doesn't exist, so we need to sync - Ok(true) - } - } else { - // No segments are needed - Ok(false) - } + let current_count = store + .get_segment_count(tape_address) + .unwrap_or(0); + + Ok(current_count < total_segments as usize) } diff --git a/network/src/archive/pack.rs b/network/src/archive/pack.rs index 11fe95ec..b0c232c1 100644 --- a/network/src/archive/pack.rs +++ b/network/src/archive/pack.rs @@ -5,9 +5,10 @@ use brine_tree::{Leaf, Hash, MerkleTree}; use tape_api::prelude::*; use tape_client::get_epoch_account; use solana_client::nonblocking::rpc_client::RpcClient; +use packx::{solve_with_memory, build_memory, SolverMemory}; use crate::store::*; -use super::queue::{Rx, SegmentJob}; +use super::queue::Rx; type CanopyTree = MerkleTree<{ SEGMENT_TREE_HEIGHT - SECTOR_TREE_HEIGHT }>; @@ -15,14 +16,38 @@ type CanopyTree = MerkleTree<{ SEGMENT_TREE_HEIGHT - SECTOR_TREE_HEIGHT }>; pub async fn run(rpc: Arc, mut rx: Rx, miner: Pubkey, store: Arc) -> Result<()> { let epoch = get_epoch_account(&rpc).await?.0; let packing_difficulty = epoch.packing_difficulty; + let miner_bytes = miner.to_bytes(); + let mem = Arc::new(build_memory(&miner_bytes)); + + // TODO: scale the thread count based on job queue depth (but also don't get in the way of the + // miner threads) while let Some(job) = rx.recv().await { let store = store.clone(); + let mem = mem.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - log::info!("packx: tape={} seg={}", job.tape, job.seg_no); + log::info!("packx: tape={} seg={} diff={}", job.tape, job.seg_no, packing_difficulty); + + pack_segment( + &store, + &mem, + &miner, + &job.tape, + job.data, + job.seg_no, + packing_difficulty + )?; - process_poa_job(&store, miner, packing_difficulty, job) + // TODO: only update the canopy if this segment is the last for this sector + update_merkle_canopy_for_segment( + &store, + &miner, + &job.tape, + job.seg_no, + )?; + + Ok(()) }) .await??; } @@ -30,42 +55,92 @@ pub async fn run(rpc: Arc, mut rx: Rx, miner: Pubkey, store: Arc, - miner: Pubkey, - packing_difficulty: u64, - job: SegmentJob, + mem: &Arc, + miner_address: &Pubkey, + tape_address: &Pubkey, + segment_data: Vec, + segment_number: u64, + difficulty: u64, ) -> anyhow::Result<()> { - let sector_number = job.seg_no / SECTOR_LEAVES as u64; - let local_idx = (job.seg_no % SECTOR_LEAVES as u64) as usize; - // TODO: rate limit or find a way to determine if this is the last segment in a sector - // (might not be the last segment index for tapes that don't fill the entire last sector). + let miner_bytes = miner_address.to_bytes(); + let segment_bytes = padded_array::(&segment_data); + + // Pack the segment into a miner-specific solution + let solution = solve_with_memory( + &segment_bytes, + mem, + difficulty as u32 + ).ok_or_else(|| anyhow!("Failed to find solution"))?; + + // Verify the solution before storing + if !packx::verify( + &miner_bytes, + &segment_bytes, + &solution, + difficulty as u32 + ) { + return Err(anyhow!("Solution verification failed")); + } - let mut sector = get_or_create_sector(store.as_ref(), &job.tape, sector_number)?; + let packed_segment = solution.to_bytes(); - // Pack the segment, and update the PackedTapeLayer sector root - let solved = pack_segment(&miner, &job.data, packing_difficulty)?; - sector.set_segment(local_idx, &solved); + store.put_segment( + tape_address, + segment_number, + packed_segment.to_vec() + )?; - let empty_hashes = get_or_create_empty_hashes(store, &job.tape)?; + Ok(()) +} + +/// Updates the Merkle canopy for the sector containing the specified segment. +pub fn update_merkle_canopy_for_segment( + store: &Arc, + miner_address: &Pubkey, + tape_address: &Pubkey, + segment_number: u64, +) -> anyhow::Result<()> { + let sector_number = segment_number / SECTOR_LEAVES as u64; + + update_merkle_canopy_for_sector( + store, + miner_address, + tape_address, + sector_number, + ) +} + +/// Updates the Merkle canopy for the provided sector number. +pub fn update_merkle_canopy_for_sector( + store: &Arc, + miner_address: &Pubkey, + tape_address: &Pubkey, + sector_number: u64, +) -> anyhow::Result<()> { + + let empty_hashes = get_or_create_empty_hashes(store, tape_address)?; let empty_leaf = empty_hashes.first().unwrap().as_leaf(); - let miner_bytes = miner.to_bytes(); let leaves_unpacked = compute_sector_leaves_unpacked( - §or, + store, + miner_address, + tape_address, sector_number, - &miner_bytes, empty_leaf, )?; + let leaves_packed = compute_sector_leaves_packed( - §or, + store, + tape_address, sector_number, empty_leaf, )?; - // Compute both roots with the same zero vector let root_unpacked = compute_sector_root(&leaves_unpacked, &empty_hashes)?; let root_packed = compute_sector_root(&leaves_packed, &empty_hashes)?; @@ -74,7 +149,7 @@ fn process_poa_job( sector_number, root_unpacked, MerkleCacheKey::UnpackedTapeLayer { - address: job.tape, + address: *tape_address, layer: SECTOR_TREE_HEIGHT as u8, }, )?; @@ -84,40 +159,20 @@ fn process_poa_job( sector_number, root_packed, MerkleCacheKey::PackedTapeLayer { - address: job.tape, + address: *tape_address, layer: SECTOR_TREE_HEIGHT as u8, }, )?; - // Finally, add the newly packed segment into the sector itself. - store.put_sector(&job.tape, sector_number, §or)?; - Ok(()) } -/// Packs a segment using the packx algorithm. -/// Can be quite CPU intensive if the difficulty is high. -pub fn pack_segment(miner_address: &Pubkey, segment: &[u8], packing_difficulty: u64) -> Result> { - let miner_address: [u8; 32] = miner_address.to_bytes(); - let canonical_segment = padded_array::(segment); - - let solution = packx::solve(&miner_address, &canonical_segment, packing_difficulty as u32) - .ok_or_else(|| anyhow!("Failed to find solution"))?; - - if !packx::verify(&miner_address, &canonical_segment, &solution, packing_difficulty as u32) { - return Err(anyhow!("Solution verification failed")); - } - - let segment_bytes = solution.to_bytes(); - Ok(segment_bytes.to_vec()) -} /// Computes the Merkle root for the entire tape by using a cached canopy of sector roots. pub fn get_tape_root( store: &Arc, tape_address: &Pubkey, ) -> Result { - // canopy height = number of levels above the sector layer up to the root const CANOPY_HEIGHT: usize = SEGMENT_TREE_HEIGHT - SECTOR_TREE_HEIGHT; // All zero values for the full-height segment tree @@ -161,37 +216,47 @@ pub fn get_tape_root( /// Computes packed leaves (stored solution bytes). pub fn compute_sector_leaves_packed( - sector: &Sector, + store: &Arc, + tape_address: &Pubkey, sector_number: u64, empty_leaf: Leaf, ) -> Result> { let mut leaves = vec![empty_leaf; SECTOR_LEAVES]; + for i in 0..SECTOR_LEAVES { - if let Some(packed) = sector.get_segment(i) { + let segment_number = (sector_number * SECTOR_LEAVES as u64) + i as u64; + if let Ok(packed) = store.get_segment(tape_address, segment_number) { let segment_id = (sector_number * SECTOR_LEAVES as u64) + i as u64; leaves[i] = Leaf::new(&[ &segment_id.to_le_bytes(), - packed, + &packed, ]); } } + Ok(leaves) } /// Computes unpacked leaves (reconstructed 128-byte data from solutions). pub fn compute_sector_leaves_unpacked( - sector: &Sector, + store: &Arc, + miner_address: &Pubkey, + tape_address: &Pubkey, sector_number: u64, - miner_bytes: &[u8; 32], empty_leaf: Leaf, ) -> Result> { + let miner_bytes = miner_address.to_bytes(); + + let mut data = [0u8; PACKED_SEGMENT_SIZE]; let mut leaves = vec![empty_leaf; SECTOR_LEAVES]; + for i in 0..SECTOR_LEAVES { - if let Some(packed) = sector.get_segment(i) { - let mut arr = [0u8; PACKED_SEGMENT_SIZE]; - arr.copy_from_slice(&packed[..PACKED_SEGMENT_SIZE]); - let sol = packx::Solution::from_bytes(&arr); - let data_unpacked = sol.unpack(miner_bytes); + let segment_number = (sector_number * SECTOR_LEAVES as u64) + i as u64; + if let Ok(packed) = store.get_segment(tape_address, segment_number) { + data.copy_from_slice(&packed[..PACKED_SEGMENT_SIZE]); + + let solution = packx::Solution::from_bytes(&data); + let data_unpacked = solution.unpack(&miner_bytes); let segment_id = (sector_number * SECTOR_LEAVES as u64) + i as u64; leaves[i] = Leaf::new(&[ @@ -200,6 +265,7 @@ pub fn compute_sector_leaves_unpacked( ]); } } + Ok(leaves) } @@ -300,17 +366,6 @@ pub fn get_or_create_empty_hashes( Ok(empty_hashes) } -fn get_or_create_sector( - store: &TapeStore, - tape_address: &Pubkey, - sector_number: u64, -) -> Result { - match store.get_sector(tape_address, sector_number) { - Ok(s) => Ok(s), - Err(StoreError::SectorNotFoundForAddress(_, _)) => Ok(Sector::new()), - Err(e) => Err(e), - } -} #[cfg(test)] mod tests { @@ -325,14 +380,14 @@ mod tests { Ok((Arc::new(store), temp_dir)) } - fn create_segment_data(marker: u8, miner: &Pubkey) -> Vec { + fn create_segment_data(marker: u8, mem: &packx::SolverMemory) -> Vec { const TEST_DIFFICULTY: u32 = 0; let data = &[marker; SEGMENT_SIZE]; let canonical_segment = padded_array::(data); - let solution = packx::solve( - &miner.to_bytes(), + let solution = packx::solve_with_memory( &canonical_segment, + mem, TEST_DIFFICULTY ).expect("Failed to pack segment"); @@ -346,6 +401,9 @@ mod tests { // Setup our tape and miner let miner_address = Pubkey::new_unique(); let tape_address = Pubkey::new_unique(); + + let mem = packx::build_memory(&miner_address.to_bytes()); + let mut tape_tree = SegmentTree::new(&[tape_address.as_ref()]); let mut leaves = vec![]; @@ -359,13 +417,13 @@ mod tests { let leaf_count = (SECTOR_LEAVES as f64 * 2.5) as usize; for i in 0..leaf_count { let segment_id = i as u64; - let segment_data_packed = create_segment_data(1, &miner_address); + let segment_data_packed = create_segment_data(1, &mem); // For the in-memory expected tree, use UNPACKED bytes - let mut sol_bytes = [0u8; PACKED_SEGMENT_SIZE]; - sol_bytes.copy_from_slice(&segment_data_packed[..PACKED_SEGMENT_SIZE]); - let sol = packx::Solution::from_bytes(&sol_bytes); - let data_unpacked = sol.unpack(&miner_bytes); + let mut data = [0u8; PACKED_SEGMENT_SIZE]; + data.copy_from_slice(&segment_data_packed[..PACKED_SEGMENT_SIZE]); + let solution = packx::Solution::from_bytes(&data); + let data_unpacked = solution.unpack(&miner_bytes); let leaf = Leaf::new(&[ &segment_id.to_le_bytes(), @@ -383,28 +441,9 @@ mod tests { .map(|h| h.to_bytes()) .collect::>(); - // Calculate sector roots and update the stored canopy values - for sector_number in 0..=2 { - let sector = store.get_sector(&tape_address, sector_number)?; - - let leaves_unpacked = compute_sector_leaves_unpacked( - §or, - sector_number, - &miner_bytes, - empty_leaf - )?; - let root_unpacked = compute_sector_root(&leaves_unpacked, &empty_values)?; - - update_sector_canopy_with_key( - &store, - sector_number, - root_unpacked, - MerkleCacheKey::UnpackedTapeLayer { - address: tape_address, - layer: SECTOR_TREE_HEIGHT as u8, - }, - )?; - } + update_merkle_canopy_for_sector(&store, &miner_address, &tape_address, 0)?; + update_merkle_canopy_for_sector(&store, &miner_address, &tape_address, 1)?; + update_merkle_canopy_for_sector(&store, &miner_address, &tape_address, 2)?; let actual_canopy = store.get_merkle_cache( &MerkleCacheKey::UnpackedTapeLayer { @@ -418,8 +457,6 @@ mod tests { // Lets try a Merkle proof for a segment let segment_number : usize = 1234; - let sector_number = (segment_number as u64) / SECTOR_LEAVES as u64; - let sector = store.get_sector(&tape_address, sector_number)?; let expected_proof = tape_tree.get_proof(&leaves, segment_number); // <- Requires all leaves in memory (bad) let actual_proof = get_cached_merkle_proof( // <- Only one sector in memory (good) @@ -428,20 +465,19 @@ mod tests { SECTOR_TREE_HEIGHT, &expected_vals, |i| { - let local_idx = i % SECTOR_LEAVES; - match sector.get_segment(local_idx) { // <- Look ma, no store access here! - Some(packed) => { - let mut arr = [0u8; PACKED_SEGMENT_SIZE]; - arr.copy_from_slice(&packed[..PACKED_SEGMENT_SIZE]); - let sol = packx::Solution::from_bytes(&arr); - let data_unpacked = sol.unpack(&miner_bytes); + match store.get_segment(&tape_address, i as u64) { + Ok(packed) => { + let mut data = [0u8; PACKED_SEGMENT_SIZE]; + data.copy_from_slice(&packed[..PACKED_SEGMENT_SIZE]); + let solution = packx::Solution::from_bytes(&data); + let data_unpacked = solution.unpack(&miner_bytes); Some(Leaf::new(&[ &(i as u64).to_le_bytes(), &data_unpacked ])) } - None => Some(empty_leaf), + _ => Some(empty_leaf), } } ); @@ -462,7 +498,7 @@ mod tests { let _ = std::thread::Builder::new() .name("larger_stack".into()) .stack_size(4 * 1024 * 1024) - .spawn(|| test_with_larger_stack()) + .spawn(test_with_larger_stack) .unwrap() .join() .unwrap(); diff --git a/network/src/archive/queue.rs b/network/src/archive/queue.rs index 9d87b88a..6daf66cb 100644 --- a/network/src/archive/queue.rs +++ b/network/src/archive/queue.rs @@ -1,7 +1,7 @@ use tokio::sync::mpsc; use solana_sdk::pubkey::Pubkey; -pub const QUEUE_CAP: usize = 1_000; +pub const QUEUE_CAP: usize = 10_000; #[derive(Debug)] pub struct SegmentJob { diff --git a/network/src/archive/sync.rs b/network/src/archive/sync.rs index 356c5690..aba5648e 100644 --- a/network/src/archive/sync.rs +++ b/network/src/archive/sync.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -//use reqwest::Client as HttpClient; use solana_client::nonblocking::rpc_client::RpcClient; use solana_transaction_status_client_types::TransactionDetails; use solana_sdk::pubkey::Pubkey; @@ -7,6 +6,7 @@ use std::sync::Arc; use std::collections::HashSet; use tokio::task::JoinSet; +use tape_api::{SEGMENT_SIZE, state::TapeState}; use tape_client::{ get_block_by_number, get_archive_account, get_tape_account, find_tape_account, init_read, process_next_block, get_epoch_account @@ -185,14 +185,21 @@ pub async fn sync_addresses_from_solana( /// Syncs block data for a specific tape address starting from a given slot. pub async fn sync_from_block( - store: &TapeStore, + store: &Arc, client: &Arc, tape_address: &Pubkey, + miner_address: &Pubkey, starting_slot: u64, ) -> Result<()> { let mut visited: HashSet = HashSet::new(); let mut stack: Vec = Vec::new(); + // Ensure the tape address is stored if finalized + let (tape, _) = get_tape_account(client, tape_address).await?; + if tape.state == u64::from(TapeState::Finalized) { + store.put_tape_address(tape.number, tape_address)?; + } + stack.push(starting_slot); while let Some(current_slot) = stack.pop() { @@ -200,6 +207,8 @@ pub async fn sync_from_block( continue; // Skip if already visited } + log::debug!("Processing slot: {}", current_slot); + let block = get_block_by_number(client, current_slot, TransactionDetails::Full).await?; let ProcessedBlock { segment_writes, @@ -207,6 +216,18 @@ pub async fn sync_from_block( finalized_tapes, } = process_block(block, current_slot)?; + log::debug!( + "Slot: {}, Finalized Tapes: {}, Segment Writes: {} ({} bytes)", + slot, + finalized_tapes.len(), + segment_writes.len(), + SEGMENT_SIZE * segment_writes.len() + ); + + // Note: we won't usually hit this during a manual re-sync, the tail_slot in the tape + // account doesn't include the finalized slot. This is here to catch any other tape + // finalizations that may have occurred (not for the current tape) + if finalized_tapes.is_empty() && segment_writes.is_empty() { continue; // Skip empty blocks } @@ -235,14 +256,23 @@ pub async fn sync_from_block( .await .map_err(|e| anyhow!("Failed to get epoch account: {}", e))?.0; - let packing_difficulty = epoch.packing_difficulty; + let miner_bytes = miner_address.to_bytes(); + let mem = Arc::new(packx::build_memory(&miner_bytes)); for (key, data) in segment_writes { if key.address != *tape_address { continue; } - let processed_segment = pack_segment(&key.address, &data, packing_difficulty)?; - store.put_segment(&key.address, key.segment_number, processed_segment)?; + + pack_segment( + store, + &mem, + miner_address, + &key.address, + data, + key.segment_number, + epoch.packing_difficulty, + )?; } for parent in parents { diff --git a/network/src/mine.rs b/network/src/mine.rs index c5743335..f210aec3 100644 --- a/network/src/mine.rs +++ b/network/src/mine.rs @@ -6,6 +6,7 @@ use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{signature::Keypair, pubkey::Pubkey}; use tape_client::mine::mine::perform_mining; use tokio::time::{sleep, Duration}; +use brine_tree::{Hash, Leaf, get_cached_merkle_proof}; use crankx::equix::SolverMemory; use crankx::{ @@ -128,43 +129,60 @@ async fn try_mine_iteration( tape.total_segments ); - // Unpack the whole tape (temporary code for now...) - // (todo: this could be up to 32Mb and not really trival with ~262k segments) - - let segments = store.get_tape_segments(&tape_address)?; - if segments.len() != tape.total_segments as usize { - // TODO: we need to refetch the tape segments from the network - return Err(anyhow!("Local store is missing some segments for tape number {}: expected {}, got {}", - tape_address, tape.total_segments, segments.len())); - } - - let mut leaves = Vec::new(); - let mut packed_segment = [0; PACKED_SEGMENT_SIZE]; - let mut unpacked_segment = [0; SEGMENT_SIZE]; + let canopy_values = store.get_merkle_cache( + &MerkleCacheKey::UnpackedTapeLayer { + address: tape_address, + layer: SECTOR_TREE_HEIGHT as u8 + } + )?; - for (segment_id, packed_data) in segments.iter() { - let mut data = [0u8; PACKED_SEGMENT_SIZE]; - data.copy_from_slice(&packed_data[..PACKED_SEGMENT_SIZE]); + let canopy_hashes: Vec<_> = canopy_values + .into_iter() + .map(Hash::from) + .collect(); - let solution = packx::Solution::from_bytes(&data); - let segement_data = solution.unpack(&miner_address.to_bytes()); + let miner_bytes = miner_address.to_bytes(); + let merkle_tree = SegmentTree::new(&[tape_address.as_ref()]); - let leaf = compute_leaf( - *segment_id, - &segement_data, - ); + // This only fetches 1024 segments max, which is fine for now (~150kb in total) + let get_leaf = |i| { + match store.get_segment(&tape_address, i as u64) { + Ok(packed) => { + let mut data = [0u8; PACKED_SEGMENT_SIZE]; + data.copy_from_slice(&packed[..PACKED_SEGMENT_SIZE]); + let solution = packx::Solution::from_bytes(&data); + let data_unpacked = solution.unpack(&miner_bytes); + + Some(Leaf::new(&[ + &(i as u64).to_le_bytes(), + &data_unpacked + ])) + } + _ => Some(merkle_tree.get_empty_leaf()), + } + }; + + // Get the Merkle proof for the segment (using a pre-cached canopy) + let proof_nodes = get_cached_merkle_proof( + &merkle_tree, + segment_number as usize, + SECTOR_TREE_HEIGHT, + &canopy_hashes, + get_leaf + ); - leaves.push(leaf); + let proof_nodes: Vec<[u8; 32]> = proof_nodes + .into_iter() + .map(|h| h.to_bytes()) + .collect(); - if *segment_id == segment_number { - packed_segment.copy_from_slice(&data); - unpacked_segment.copy_from_slice(&segement_data); - } - } + let proof_path = ProofPath::from_slice(&proof_nodes).unwrap(); - if packed_segment == [0; PACKED_SEGMENT_SIZE] { - return Err(anyhow!("Segment number {} not found in tape {}", segment_number, tape_address)); - } + let segment = store.get_segment(&tape_address, segment_number)?; + let mut packed_segment = [0; PACKED_SEGMENT_SIZE]; + packed_segment.copy_from_slice(&segment[..PACKED_SEGMENT_SIZE]); + let solution = packx::Solution::from_bytes(&packed_segment); + let unpacked_segment = solution.unpack(&miner_address.to_bytes()); let poa_solution = packx::Solution::from_bytes(&packed_segment); let pow_solution = solve_challenge( @@ -175,15 +193,6 @@ async fn try_mine_iteration( debug_assert!(pow_solution.is_valid(&miner_challenge, &unpacked_segment).is_ok()); - let merkle_tree = SegmentTree::new(&[tape_address.as_ref()]); - let proof_nodes: Vec<[u8; 32]> = merkle_tree - .get_proof(&leaves, segment_number as usize) - .into_iter() - .map(|h| h.to_bytes()) - .collect(); - - let proof_path = ProofPath::from_slice(&proof_nodes).unwrap(); - let pow = PoW::from_solution(&pow_solution); let poa = PoA::from_solution(&poa_solution, proof_path); diff --git a/network/src/store/column/merkle.rs b/network/src/store/column/merkle.rs index 46f3cca7..fb34d126 100644 --- a/network/src/store/column/merkle.rs +++ b/network/src/store/column/merkle.rs @@ -208,7 +208,7 @@ mod tests { // Manually write an invalid-length value (not a multiple of 32) let cf = store.get_cf_handle(ColumnFamily::MerkleHashes)?; - store.db.put_cf(&cf, &key_vec, &[0u8; 5])?; + store.db.put_cf(&cf, &key_vec, [0u8; 5])?; let res = store.get_merkle_cache( &MerkleCacheKey::UnpackedTapeLayer { address: addr, layer: 3 }, diff --git a/network/src/store/column/mod.rs b/network/src/store/column/mod.rs index 9a124e24..b0ec34bc 100644 --- a/network/src/store/column/mod.rs +++ b/network/src/store/column/mod.rs @@ -1,4 +1,3 @@ -mod sector; mod segment; mod health; mod tape; @@ -8,6 +7,5 @@ mod stats; pub use health::{StoreStaticKeys, HealthOps}; pub use tape::TapeOps; pub use segment::SegmentOps; -pub use sector::{SectorOps, Sector}; pub use merkle::{MerkleOps, MerkleCacheKey}; pub use stats::{LocalStats, StatsOps}; diff --git a/network/src/store/column/sector.rs b/network/src/store/column/sector.rs deleted file mode 100644 index c59e587d..00000000 --- a/network/src/store/column/sector.rs +++ /dev/null @@ -1,328 +0,0 @@ -use bytemuck::{Pod, Zeroable, try_from_bytes, bytes_of}; -use solana_sdk::pubkey::Pubkey; -use tape_api::consts::*; -use crate::store::*; - -pub trait SectorOps { - fn get_sector(&self, tape_address: &Pubkey, sector_number: u64) -> Result; - fn put_sector(&self, tape_address: &Pubkey, sector_number: u64, sector: &Sector) -> Result<(), StoreError>; - fn get_sector_count(&self, tape_address: &Pubkey) -> Result; -} - -impl SectorOps for TapeStore { - fn get_sector(&self, tape_address: &Pubkey, sector_number: u64) -> Result { - let cf = self.get_cf_handle(ColumnFamily::Sectors)?; - let mut key = Vec::with_capacity(TAPE_STORE_SLOTS_KEY_SIZE); - key.extend_from_slice(&tape_address.to_bytes()); - key.extend_from_slice(§or_number.to_be_bytes()); - - let data = self - .db - .get_cf(&cf, &key)? - .ok_or_else(|| StoreError::SectorNotFoundForAddress(tape_address.to_string(), sector_number))?; - - if data.len() != SECTOR_HEADER_BYTES + SECTOR_LEAVES * PACKED_SEGMENT_SIZE { - return Err(StoreError::InvalidSectorSize(data.len())); - } - - Ok(*try_from_bytes(&data).map_err(|_| StoreError::InvalidSectorSize(data.len()))?) - } - - fn put_sector(&self, tape_address: &Pubkey, sector_number: u64, sector: &Sector) -> Result<(), StoreError> { - let cf = self.get_cf_handle(ColumnFamily::Sectors)?; - let mut key = Vec::with_capacity(TAPE_STORE_SLOTS_KEY_SIZE); - key.extend_from_slice(&tape_address.to_bytes()); - key.extend_from_slice(§or_number.to_be_bytes()); - - self.db.put_cf(&cf, &key, bytes_of(sector))?; - Ok(()) - } - - fn get_sector_count(&self, tape_address: &Pubkey) -> Result { - let cf = self.get_cf_handle(ColumnFamily::Sectors)?; - let prefix = tape_address.to_bytes().to_vec(); - let iter = self.db.prefix_iterator_cf(&cf, &prefix); - Ok(iter.count()) - } -} - -#[repr(transparent)] -#[derive(Clone, Copy, Debug)] -pub struct Sector(pub [u8; - SECTOR_HEADER_BYTES + SECTOR_LEAVES * PACKED_SEGMENT_SIZE]); - -unsafe impl Zeroable for Sector {} -unsafe impl Pod for Sector {} - -impl Default for Sector { - fn default() -> Self { - Self::new() - } -} - -impl Sector { - pub fn new() -> Self { - Self::zeroed() - } - - pub fn set_segment(&mut self, local_seg_idx: usize, data: &[u8]) -> bool { - if local_seg_idx >= SECTOR_LEAVES || data.len() != PACKED_SEGMENT_SIZE { - return false; - } - - let bitmap_idx = local_seg_idx / 8; - let bit_pos = local_seg_idx % 8; - let is_new_segment = (self.0[bitmap_idx] & (1 << bit_pos)) == 0; - - self.0[bitmap_idx] |= 1 << bit_pos; - let seg_start = SECTOR_HEADER_BYTES + local_seg_idx * PACKED_SEGMENT_SIZE; - self.0[seg_start..seg_start + PACKED_SEGMENT_SIZE].copy_from_slice(data); - is_new_segment - } - - pub fn get_segment(&self, local_seg_idx: usize) -> Option<&[u8]> { - if local_seg_idx >= SECTOR_LEAVES { - return None; - } - - let bitmap_idx = local_seg_idx / 8; - let bit_pos = local_seg_idx % 8; - - if (self.0[bitmap_idx] & (1 << bit_pos)) == 0 { - return None; - } - - let seg_start = SECTOR_HEADER_BYTES + local_seg_idx * PACKED_SEGMENT_SIZE; - Some(&self.0[seg_start..seg_start + PACKED_SEGMENT_SIZE]) - } - - pub fn count_segments(&self) -> usize { - let bitmap_len = SECTOR_LEAVES / 8; - self.0[..bitmap_len].iter().map(|byte| byte.count_ones() as usize).sum() - } - - pub fn get_last_index(&self) -> usize { - let bitmap_len = SECTOR_LEAVES / 8; - - // Iterate over bitmap bytes from right to left - for byte_idx in (0..bitmap_len).rev() { - let byte = self.0[byte_idx]; - if byte != 0 { - // Find the highest set bit in this byte - for bit_pos in (0..8).rev() { - if (byte & (1 << bit_pos)) != 0 { - let segment_idx = byte_idx * 8 + bit_pos; - if segment_idx < SECTOR_LEAVES { - return segment_idx; - } - } - } - } - } - 0 - } -} - -#[cfg(test)] -mod tests { - use super::*; - use solana_sdk::pubkey::Pubkey; - use tempdir::TempDir; - - fn setup_store() -> Result<(TapeStore, TempDir), StoreError> { - let temp_dir = TempDir::new("rocksdb_test").map_err(StoreError::IoError)?; - let store = TapeStore::new(temp_dir.path())?; - Ok((store, temp_dir)) - } - - fn make_data(marker: u8) -> Vec { - vec![marker; PACKED_SEGMENT_SIZE] - } - - fn setup_sector(indices: &[usize]) -> Sector { - let mut sector = Sector::new(); - let data = make_data(1); - for &idx in indices { - sector.set_segment(idx, &data); - } - sector - } - - #[test] - fn sector_set_segment() { - let mut sector = Sector::new(); - let data = make_data(1); - assert!(sector.set_segment(0, &data)); - assert_eq!(sector.get_segment(0), Some(data.as_slice())); - assert_eq!(sector.count_segments(), 1); - - // Invalid index - assert!(!sector.set_segment(SECTOR_LEAVES, &data)); - assert_eq!(sector.count_segments(), 1); - - // Invalid data size - let invalid_data = vec![1; PACKED_SEGMENT_SIZE + 1]; - assert!(!sector.set_segment(1, &invalid_data)); - assert_eq!(sector.count_segments(), 1); - } - - #[test] - fn sector_get_segment() { - let sector = setup_sector(&[0, 2]); - assert_eq!(sector.get_segment(0), Some(make_data(1).as_slice())); - assert_eq!(sector.get_segment(1), None); // Not set - assert_eq!(sector.get_segment(SECTOR_LEAVES), None); // Invalid index - } - - #[test] - fn sector_count_segments() { - let sector = setup_sector(&[0, 1, 8]); - assert_eq!(sector.count_segments(), 3); - let empty_sector = Sector::new(); - assert_eq!(empty_sector.count_segments(), 0); - } - - #[test] - fn sector_rightmost_index() { - let empty_sector = Sector::new(); - assert_eq!(empty_sector.get_last_index(), 0); - - let single = setup_sector(&[5]); - assert_eq!(single.get_last_index(), 5); - - let multiple = setup_sector(&[0, 2, 5, 8]); - assert_eq!(multiple.get_last_index(), 8); - - let last = setup_sector(&[SECTOR_LEAVES - 1]); - assert_eq!(last.get_last_index(), SECTOR_LEAVES - 1); - - let sparse = setup_sector(&[0, SECTOR_LEAVES / 2, SECTOR_LEAVES - 2]); - assert_eq!(sparse.get_last_index(), SECTOR_LEAVES - 2); - } - - #[test] - fn sector_default() { - let sector = Sector::default(); - assert_eq!(sector.0, Sector::new().0); - assert_eq!(sector.count_segments(), 0); - assert_eq!(sector.get_last_index(), 0); - } - - #[test] - fn get_sector() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - let sector_num = 0u64; - let sector = setup_sector(&[0]); - - store.put_sector(&address, sector_num, §or)?; - let retrieved = store.get_sector(&address, sector_num)?; - assert_eq!(retrieved.0, sector.0); - - let result = store.get_sector(&address, 1); - assert!(matches!(result, Err(StoreError::SectorNotFoundForAddress(_, 1)))); - - Ok(()) - } - - #[test] - fn get_sector_invalid_size() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - let sector_num = 0u64; - let cf = store.get_cf_handle(ColumnFamily::Sectors)?; - let mut key = Vec::with_capacity(TAPE_STORE_SLOTS_KEY_SIZE); - key.extend_from_slice(&address.to_bytes()); - key.extend_from_slice(§or_num.to_be_bytes()); - store.db.put_cf(&cf, &key, vec![0; 10])?; - - let result = store.get_sector(&address, sector_num); - assert!(matches!(result, Err(StoreError::InvalidSectorSize(10)))); - - Ok(()) - } - - #[test] - fn put_sector() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - let sector_num = 0u64; - let sector = setup_sector(&[0]); - - store.put_sector(&address, sector_num, §or)?; - let retrieved = store.get_sector(&address, sector_num)?; - assert_eq!(retrieved.0, sector.0); - - Ok(()) - } - - #[test] - fn fill_sector() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - - for i in 0..SECTOR_LEAVES as u64 { - store.put_segment(&address, i, make_data(i as u8))?; - } - - assert_eq!(store.get_sector_count(&address)?, 1); - - let sector = store.get_sector(&address, 0)?; - let bitmap_len = SECTOR_LEAVES / 8; - assert!(sector.0[..bitmap_len].iter().all(|&b| b == 0xFF)); - assert_eq!(sector.get_segment(0).unwrap(), make_data(0)); - assert_eq!(sector.get_segment(SECTOR_LEAVES - 1).unwrap(), make_data((SECTOR_LEAVES - 1) as u8)); - - Ok(()) - } - - #[test] - fn sector_boundary() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - let last_s0 = (SECTOR_LEAVES - 1) as u64; - let first_s1 = SECTOR_LEAVES as u64; - - store.put_segment(&address, last_s0, make_data(1))?; - store.put_segment(&address, first_s1, make_data(2))?; - - assert_eq!(store.get_sector_count(&address)?, 2); - assert_eq!(store.get_segment(&address, last_s0)?, make_data(1)); - assert_eq!(store.get_segment(&address, first_s1)?, make_data(2)); - - let s0 = store.get_sector(&address, 0)?; - let s1 = store.get_sector(&address, 1)?; - assert_eq!(s0.0[(SECTOR_LEAVES - 1) / 8] & (1 << 7), 1 << 7); - assert_eq!(s1.0[0] & 0x01, 0x01); - - Ok(()) - } - - #[test] - fn multi_sectors() -> Result<(), StoreError> { - let (store, _temp_dir) = setup_store()?; - let address = Pubkey::new_unique(); - let sectors = 100u64; - let stride = (SECTOR_LEAVES / 3).max(1) as u64; - - for s in 0..sectors { - let base = s * SECTOR_LEAVES as u64; - for k in 0..3 { - let idx = base + k * stride; - store.put_segment(&address, idx, make_data(idx as u8))?; - } - } - - assert_eq!(store.get_sector_count(&address)?, sectors as usize); - - for s in 0..sectors { - let sector = store.get_sector(&address, s)?; - for k in 0..3 { - let li = (k * stride) as usize; - assert_ne!(sector.0[li / 8] & (1 << (li % 8)), 0); - } - } - - Ok(()) - } -} diff --git a/network/src/store/column/segment.rs b/network/src/store/column/segment.rs index e3f68579..910fb579 100644 --- a/network/src/store/column/segment.rs +++ b/network/src/store/column/segment.rs @@ -1,85 +1,110 @@ use solana_sdk::pubkey::Pubkey; -use rocksdb::WriteBatch; -use bytemuck::bytes_of; use tape_api::consts::PACKED_SEGMENT_SIZE; use crate::store::*; use crate::metrics::inc_total_segments_written; pub trait SegmentOps { - fn get_segment(&self, tape_address: &Pubkey, global_seg_idx: u64) -> Result, StoreError>; fn put_segment(&self, tape_address: &Pubkey, global_seg_idx: u64, seg: Vec) -> Result<(), StoreError>; + fn get_segment(&self, tape_address: &Pubkey, global_seg_idx: u64) -> Result, StoreError>; + fn get_segment_range(&self, tape_address: &Pubkey, start: u64, end: u64) -> Result)>, StoreError>; fn get_tape_segments(&self, tape_address: &Pubkey) -> Result)>, StoreError>; + fn get_segment_count(&self, tape_address: &Pubkey) -> Result; } impl SegmentOps for TapeStore { - fn get_segment(&self, tape_address: &Pubkey, global_seg_idx: u64) -> Result, StoreError> { - let sector_number = global_seg_idx / SECTOR_LEAVES as u64; - let local_seg_idx = (global_seg_idx % SECTOR_LEAVES as u64) as usize; - - let sector = self.get_sector(tape_address, sector_number)?; - - // Check bitmap - let bitmap_idx = local_seg_idx / 8; - let bit_pos = local_seg_idx % 8; - if (sector.0[bitmap_idx] & (1 << bit_pos)) == 0 { - return Err(StoreError::SegmentNotFoundForAddress(tape_address.to_string(), global_seg_idx)); - } - - let seg_start = SECTOR_HEADER_BYTES + local_seg_idx * PACKED_SEGMENT_SIZE; - Ok(sector.0[seg_start..seg_start + PACKED_SEGMENT_SIZE].to_vec()) + fn get_segment(&self, tape_address: &Pubkey, segment_number: u64) -> Result, StoreError> { + let cf = self.get_cf_handle(ColumnFamily::Segments)?; + let mut key = Vec::with_capacity(40); + key.extend_from_slice(&tape_address.to_bytes()); + key.extend_from_slice(&segment_number.to_be_bytes()); + let segment_data = self + .db + .get_cf(&cf, &key)? + .ok_or(StoreError::SegmentNotFound(tape_address.to_string(), segment_number))?; + Ok(segment_data) } - fn put_segment(&self, tape_address: &Pubkey, global_seg_idx: u64, seg: Vec) -> Result<(), StoreError> { - if seg.len() != PACKED_SEGMENT_SIZE { - return Err(StoreError::InvalidSegmentSize(seg.len())); + fn put_segment(&self, tape_address: &Pubkey, segment_number: u64, data: Vec) -> Result<(), StoreError> { + if data.len() > PACKED_SEGMENT_SIZE { + return Err(StoreError::InvalidSegmentSize(PACKED_SEGMENT_SIZE)); } - - let sector_number = global_seg_idx / SECTOR_LEAVES as u64; - let local_seg_idx = (global_seg_idx % SECTOR_LEAVES as u64) as usize; - - let cf_sectors = self.get_cf_handle(ColumnFamily::Sectors)?; - - let mut sector = self.get_sector(tape_address, sector_number).unwrap_or_else(|_| Sector::new()); - sector.set_segment(local_seg_idx, &seg); - - let mut batch = WriteBatch::default(); - let mut key = Vec::with_capacity(TAPE_STORE_SLOTS_KEY_SIZE); + let cf = self.get_cf_handle(ColumnFamily::Segments)?; + let mut key = Vec::with_capacity(40); key.extend_from_slice(&tape_address.to_bytes()); - key.extend_from_slice(§or_number.to_be_bytes()); - batch.put_cf(&cf_sectors, &key, bytes_of(§or)); - - self.db.write(batch)?; + key.extend_from_slice(&segment_number.to_be_bytes()); + self.db.put_cf(&cf, &key, &data)?; inc_total_segments_written(); + Ok(()) } - fn get_tape_segments(&self, tape_address: &Pubkey) -> Result)>, StoreError> { - let cf = self.get_cf_handle(ColumnFamily::Sectors)?; + fn get_segment_range(&self, tape_address: &Pubkey, start: u64, end: u64) -> Result)>, StoreError> { + if start >= end { + return Ok(Vec::new()); + } + let cf = self.get_cf_handle(ColumnFamily::Segments)?; let prefix = tape_address.to_bytes().to_vec(); - let iter = self.db.prefix_iterator_cf(&cf, &prefix); + let mut segments = Vec::new(); + let iter = self.db.prefix_iterator_cf(&cf, &prefix); + for item in iter { + let (key, value) = item?; + if key.len() != 40 { + continue; + } + if !key.starts_with(&prefix) { + continue; + } + let segment_number = u64::from_be_bytes( + key[32..TAPE_STORE_SLOTS_KEY_SIZE] + .try_into() + .map_err(|_| StoreError::InvalidSegmentKey)?, + ); + if segment_number >= start && segment_number < end { + segments.push((segment_number, value.to_vec())); + } + } + // Since the iterator is sorted by key (and thus by segment_number), the segments are already in order + Ok(segments) + } + + fn get_tape_segments(&self, tape_address: &Pubkey) -> Result)>, StoreError> { + let cf = self.get_cf_handle(ColumnFamily::Segments)?; + let prefix = tape_address.to_bytes().to_vec(); + + let mut segments = Vec::new(); + let iter = self.db.prefix_iterator_cf(&cf, &prefix); for item in iter { - let (key, data) = item?; - if key.len() < TAPE_STORE_SLOTS_KEY_SIZE { + let (key, value) = item?; + if key.len() != 40 { continue; } - let sector_number = u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()); - - let sector: Sector = *bytemuck::try_from_bytes(&data) - .map_err(|_| StoreError::InvalidSectorSize(data.len()))?; - - for local_idx in 0..SECTOR_LEAVES { - if let Some(segment_data) = sector.get_segment(local_idx) { - let global_index = sector_number * SECTOR_LEAVES as u64 + local_idx as u64; - segments.push((global_index, segment_data.to_vec())); - } + if !key.starts_with(&prefix) { + continue; } + let segment_number = u64::from_be_bytes( + key[32..TAPE_STORE_SLOTS_KEY_SIZE] + .try_into() + .map_err(|_| StoreError::InvalidSegmentKey)?, + ); + segments.push((segment_number, value.to_vec())); } - segments.sort_by_key(|(idx, _)| *idx); + // Since the iterator is sorted by key (and thus by segment_number), the segments are already in order Ok(segments) } + + fn get_segment_count( + &self, + tape_address: &Pubkey, + ) -> Result { + let cf = self.get_cf_handle(ColumnFamily::Segments)?; + let prefix = tape_address.to_bytes().to_vec(); + let iter = self.db.prefix_iterator_cf(&cf, &prefix); + let count = iter.count(); + Ok(count) + } } #[cfg(test)] @@ -112,65 +137,122 @@ mod tests { } #[test] - fn test_put_segment_count() -> Result<(), StoreError> { + fn test_get_segment_not_found() -> Result<(), StoreError> { let (store, _temp_dir) = setup_store()?; let address = Pubkey::new_unique(); - let data = make_data(42); - - // Write two new segments in the same sector - store.put_segment(&address, 0, data.clone())?; - store.put_segment(&address, 1, data.clone())?; - assert_eq!(store.get_sector_count(&address)?, 1); - - // Overwrite existing segment (should not change sector count) - store.put_segment(&address, 0, data.clone())?; - assert_eq!(store.get_sector_count(&address)?, 1); + let global_seg_idx = 0; + let result = store.get_segment(&address, global_seg_idx); + assert!(matches!(result, Err(StoreError::SegmentNotFound(_, _)))); + Ok(()) + } - // Write new segment in a new sector - store.put_segment(&address, SECTOR_LEAVES as u64, data)?; - assert_eq!(store.get_sector_count(&address)?, 2); + #[test] + fn test_put_segment_too_large() -> Result<(), StoreError> { + let (store, _temp_dir) = setup_store()?; + let address = Pubkey::new_unique(); + let global_seg_idx = 0; + let oversized_data = vec![42u8; PACKED_SEGMENT_SIZE + 1]; + let result = store.put_segment(&address, global_seg_idx, oversized_data); + assert!(matches!(result, Err(StoreError::InvalidSegmentSize(_)))); Ok(()) } #[test] fn test_get_tape_segments() -> Result<(), StoreError> { - let (store, _tmp) = setup_store()?; + let (store, _temp_dir) = setup_store()?; let address = Pubkey::new_unique(); + let data0 = make_data(0); + let data1 = make_data(1); + let data2 = make_data(2); + + store.put_segment(&address, 0, data0.clone())?; + store.put_segment(&address, 1, data1.clone())?; + store.put_segment(&address, 2, data2.clone())?; - // Pick a few scattered indices across 3 sectors - let idx_sector0_a = 0u64; - let idx_sector0_b = 5u64; - let idx_sector1_a = SECTOR_LEAVES as u64; // first in sector 1 - let idx_sector1_b = SECTOR_LEAVES as u64 + 10; - let idx_sector2_a = SECTOR_LEAVES as u64 * 2; // first in sector 2 - - // Write them - store.put_segment(&address, idx_sector0_a, make_data(10))?; - store.put_segment(&address, idx_sector0_b, make_data(20))?; - store.put_segment(&address, idx_sector1_a, make_data(30))?; - store.put_segment(&address, idx_sector1_b, make_data(40))?; - store.put_segment(&address, idx_sector2_a, make_data(50))?; - - // Read back let segments = store.get_tape_segments(&address)?; + assert_eq!(segments.len(), 3); - // We expect exactly 5 entries in ascending global index order - let expected_indices = [ - idx_sector0_a, - idx_sector0_b, - idx_sector1_a, - idx_sector1_b, - idx_sector2_a, - ]; - assert_eq!(segments.len(), expected_indices.len()); - for (i, (idx, data)) in segments.iter().enumerate() { - assert_eq!(*idx, expected_indices[i], "segment index mismatch"); - assert_eq!(data[0], (i as u8 + 1) * 10, "segment data mismatch at index {}", idx); - assert_eq!(data.len(), PACKED_SEGMENT_SIZE); - } + let mut retrieved_segments = segments.clone(); + retrieved_segments.sort_by_key(|(idx, _)| *idx); + + assert_eq!(retrieved_segments[0].0, 0); + assert_eq!(retrieved_segments[0].1, data0); + assert_eq!(retrieved_segments[1].0, 1); + assert_eq!(retrieved_segments[1].1, data1); + assert_eq!(retrieved_segments[2].0, 2); + assert_eq!(retrieved_segments[2].1, data2); + + Ok(()) + } + + #[test] + fn test_get_segment_count() -> Result<(), StoreError> { + let (store, _temp_dir) = setup_store()?; + let address = Pubkey::new_unique(); + + // Initially empty + assert_eq!(store.get_segment_count(&address)?, 0); + + let data0 = make_data(0); + store.put_segment(&address, 0, data0.clone())?; + assert_eq!(store.get_segment_count(&address)?, 1); + + let data1 = make_data(1); + store.put_segment(&address, 1, data1.clone())?; + assert_eq!(store.get_segment_count(&address)?, 2); + + let data2 = make_data(2); + store.put_segment(&address, 2, data2.clone())?; + assert_eq!(store.get_segment_count(&address)?, 3); + + Ok(()) + } + + #[test] + fn test_get_segment_range() -> Result<(), StoreError> { + let (store, _temp_dir) = setup_store()?; + let address = Pubkey::new_unique(); + let data0 = make_data(0); + let data1 = make_data(1); + let data2 = make_data(2); + let data3 = make_data(3); + + store.put_segment(&address, 0, data0.clone())?; + store.put_segment(&address, 1, data1.clone())?; + store.put_segment(&address, 2, data2.clone())?; + store.put_segment(&address, 3, data3.clone())?; + + // Test full range equivalent to get_tape_segments + let full_range = store.get_segment_range(&address, 0, 4)?; + assert_eq!(full_range.len(), 4); + let mut full_retrieved = full_range.clone(); + full_retrieved.sort_by_key(|(idx, _)| *idx); + assert_eq!(full_retrieved[0].0, 0); + assert_eq!(full_retrieved[0].1, data0); + assert_eq!(full_retrieved[1].0, 1); + assert_eq!(full_retrieved[1].1, data1); + assert_eq!(full_retrieved[2].0, 2); + assert_eq!(full_retrieved[2].1, data2); + assert_eq!(full_retrieved[3].0, 3); + assert_eq!(full_retrieved[3].1, data3); + + // Test partial range 1 to 3 + let partial_range = store.get_segment_range(&address, 1, 3)?; + assert_eq!(partial_range.len(), 2); + let mut partial_retrieved = partial_range.clone(); + partial_retrieved.sort_by_key(|(idx, _)| *idx); + assert_eq!(partial_retrieved[0].0, 1); + assert_eq!(partial_retrieved[0].1, data1); + assert_eq!(partial_retrieved[1].0, 2); + assert_eq!(partial_retrieved[1].1, data2); + + // Test empty range start >= end + let empty_range = store.get_segment_range(&address, 2, 2)?; + assert_eq!(empty_range.len(), 0); - // Check that sector count is 3 - assert_eq!(store.get_sector_count(&address)?, 3); + // Test range beyond existing segments + let beyond_range = store.get_segment_range(&address, 4, 5)?; + assert_eq!(beyond_range.len(), 0); Ok(()) } diff --git a/network/src/store/column/stats.rs b/network/src/store/column/stats.rs index cf1d164b..11ea3c0e 100644 --- a/network/src/store/column/stats.rs +++ b/network/src/store/column/stats.rs @@ -29,7 +29,7 @@ impl TapeStore { } fn count_sectors(&self) -> Result { - let cf = self.get_cf_handle(ColumnFamily::Sectors)?; + let cf = self.get_cf_handle(ColumnFamily::Segments)?; let iter = self.db.iterator_cf(&cf, rocksdb::IteratorMode::Start); Ok(iter.count()) } diff --git a/network/src/store/error.rs b/network/src/store/error.rs index 69b5fad5..a0f0cf7f 100644 --- a/network/src/store/error.rs +++ b/network/src/store/error.rs @@ -13,8 +13,8 @@ pub enum StoreError { TapeByNumberCfNotFound, #[error("Tape by address column family not found")] TapeByAddressCfNotFound, - #[error("Sectors column family not found")] - SectorsCfNotFound, + #[error("Segments column family not found")] + SegmentsCfNotFound, #[error("Merkle hashes column family not found")] MerkleHashesCfNotFound, #[error("Tape not found: number {0}")] @@ -22,17 +22,13 @@ pub enum StoreError { #[error("Tape not found for address: {0}")] TapeNotFoundForAddress(String), #[error("Segment not found for tape number {0}, segment {1}")] - SegmentNotFound(u64, u64), + SegmentNotFound(String, u64), #[error("Segment not found for address {0}, segment {1}")] SegmentNotFoundForAddress(String, u64), - #[error("Sector not found for address {0}, sector {1}")] - SectorNotFoundForAddress(String, u64), #[error("Invalid pubkey: {0}")] InvalidPubkey(String), #[error("Invalid hash size: {0}")] InvalidHashSize(usize), - #[error("Invalid sector size, expected {0} bytes")] - InvalidSectorSize(usize), #[error("Invalid segment size, {0} bytes")] InvalidSegmentSize(usize), #[error("Hash not found")] @@ -48,7 +44,7 @@ impl From<&ColumnFamily> for StoreError { match value { ColumnFamily::TapeByNumber => StoreError::TapeByNumberCfNotFound, ColumnFamily::TapeByAddress => StoreError::TapeByAddressCfNotFound, - ColumnFamily::Sectors => StoreError::SectorsCfNotFound, + ColumnFamily::Segments => StoreError::SegmentsCfNotFound, ColumnFamily::MerkleHashes => StoreError::MerkleHashesCfNotFound, ColumnFamily::Health => StoreError::HealthCfNotFound, } diff --git a/network/src/store/layout.rs b/network/src/store/layout.rs index 6985f0b2..1490cb9d 100644 --- a/network/src/store/layout.rs +++ b/network/src/store/layout.rs @@ -7,7 +7,7 @@ use rocksdb::{ pub enum ColumnFamily { TapeByNumber, TapeByAddress, - Sectors, + Segments, MerkleHashes, Health, } @@ -17,7 +17,7 @@ impl ColumnFamily { match self { ColumnFamily::TapeByNumber => "tape_by_number", ColumnFamily::TapeByAddress => "tape_by_address", - ColumnFamily::Sectors => "sectors", + ColumnFamily::Segments => "segments", ColumnFamily::MerkleHashes => "merkle_hashes", ColumnFamily::Health => "health", } @@ -55,7 +55,7 @@ pub fn create_cf_descriptors() -> Vec { let column_families = [ (ColumnFamily::TapeByNumber, 8, Some(plain_table_options(8)), None), (ColumnFamily::TapeByAddress, 32, Some(plain_table_options(32)), None), - (ColumnFamily::Sectors, 32, None, Some(block_based_options())), + (ColumnFamily::Segments, 32, None, Some(block_based_options())), (ColumnFamily::MerkleHashes, 32, None, Some(block_based_options())), (ColumnFamily::Health, 0, None, None), ]; diff --git a/network/src/web.rs b/network/src/web.rs index b3a8052e..a5dca91f 100644 --- a/network/src/web.rs +++ b/network/src/web.rs @@ -228,6 +228,9 @@ pub fn rpc_get_tape_number(store: &TapeStore, params: &Value) -> Result,"segment_number":3}}' /// ``` pub fn rpc_get_segment(store: &TapeStore, params: &Value) -> Result { + // TODO: this will return a packed segment; we probably want the unpacked data (which requires + // the miner pubkey) + let addr = params .get("tape_address") .and_then(Value::as_str) @@ -299,7 +302,6 @@ async fn rpc_handler( - pub async fn web_loop( store: TapeStore, port: u16, diff --git a/program/src/miner/mine.rs b/program/src/miner/mine.rs index 88c7b7e3..7405faa0 100644 --- a/program/src/miner/mine.rs +++ b/program/src/miner/mine.rs @@ -9,6 +9,7 @@ const EPOCHS_PER_YEAR: u64 = 365 * 24 * 60 / EPOCH_BLOCKS; pub fn process_mine(accounts: &[AccountInfo], data: &[u8]) -> ProgramResult { msg!("Starting mine instruction processing"); let current_time = Clock::get()?.unix_timestamp; + msg!("data size: {}", data.len()); let args = Mine::try_from_bytes(data)?; let [ @@ -467,7 +468,7 @@ fn adjust_participation(epoch: &mut Epoch) { #[inline(always)] pub fn get_base_rate(current_epoch: u64) -> u64 { match current_epoch { - n if n < 1 * EPOCHS_PER_YEAR => 10000000000, // Year ~1, about 1.00 TAPE/min + n if n < EPOCHS_PER_YEAR => 10000000000, // Year ~1, about 1.00 TAPE/min n if n < 2 * EPOCHS_PER_YEAR => 7500000000, // Year ~2, about 0.75 TAPE/min n if n < 3 * EPOCHS_PER_YEAR => 5625000000, // Year ~3, about 0.56 TAPE/min n if n < 4 * EPOCHS_PER_YEAR => 4218750000, // Year ~4, about 0.42 TAPE/min diff --git a/program/tests/integration.rs b/program/tests/integration.rs index 47a043d2..edf8e4b7 100644 --- a/program/tests/integration.rs +++ b/program/tests/integration.rs @@ -1,7 +1,6 @@ #![cfg(test)] pub mod utils; use utils::*; -use steel::Discriminator; use steel::Zeroable; use solana_sdk::{ @@ -19,7 +18,6 @@ use tape_api::prelude::*; use tape_api::instruction; use litesvm::LiteSVM; -use packx; use crankx::equix::SolverMemory; use crankx::{ solve_with_memory, @@ -82,9 +80,6 @@ fn run_integration() { verify_metadata_account(&svm); verify_treasury_ata(&svm); - // Override difficulty (packx is too slow for debug mode) - override_epoch_difficulty(&mut svm, 0); - // Mine the genesis tape (to earn some tokens) do_mining_run(&mut svm, &payer, &stored_spool, 5); claim_rewards(&mut svm, &payer, miner_address, ata); @@ -103,7 +98,7 @@ fn run_integration() { let tape_count = 5; for tape_index in 1..tape_count { let stored_tape = create_and_verify_tape(&mut svm, &payer, ata, tape_index); - let _packed_tape = pack_tape(&mut svm, &payer, &stored_tape, &mut stored_spool); + pack_tape(&mut svm, &payer, &stored_tape, &mut stored_spool); } // Verify archive account after tape creation @@ -184,7 +179,7 @@ fn do_mining_run( // submitting the transaction if the commitment is still valid. let mut current_clock = svm.get_sysvar::(); - current_clock.slot = current_clock.slot + 10; + current_clock.slot += 10; svm.set_sysvar::(¤t_clock); svm.expire_blockhash(); @@ -213,8 +208,7 @@ fn do_mining_run( let tape_index = recall_tape - 1; // index in spool (not the tape_number) let packed_tape = &stored_spool.tapes[tape_index as usize]; - let tape_address = packed_tape.address; - let tape_account = svm.get_account(&tape_address).unwrap(); + let tape_account = svm.get_account(&packed_tape.address).unwrap(); let tape = Tape::unpack(&tape_account.data).unwrap(); // Check if we need to provide a PoA solution based on whether the tape has minimum rent. @@ -264,7 +258,7 @@ fn do_mining_run( let pow_solution = solve_challenge(miner_challenge, &unpacked_segment, epoch.mining_difficulty).unwrap(); assert!(pow_solution.is_valid(&miner_challenge, &unpacked_segment).is_ok()); - let merkle_tree = SegmentTree::new(&[tape_address.as_ref()]); + let merkle_tree = SegmentTree::new(&[packed_tape.address.as_ref()]); let proof_nodes: Vec<[u8; 32]> = merkle_tree .get_proof(&leaves, segment_number as usize) .into_iter() @@ -280,8 +274,8 @@ fn do_mining_run( // Tx1: load the packed tape leaf from the spool onto the miner commitment field commit_for_mining( svm, - &payer, - &stored_spool, + payer, + stored_spool, tape_index, segment_number ); @@ -333,14 +327,14 @@ fn get_genesis_tape(svm: &mut LiteSVM, payer: &Keypair) -> StoredTape { let genesis_segment = padded_array::(genesis_data).to_vec(); let segments = vec![genesis_segment]; - let stored_genesis = StoredTape { + + + StoredTape { number: tape.number, address: genesis_pubkey, segments, account: *tape, - }; - - stored_genesis + } } @@ -353,26 +347,6 @@ fn initialize_program(svm: &mut LiteSVM, payer: &Keypair) { assert!(res.is_ok()); } -fn override_epoch_difficulty(svm: &mut LiteSVM, difficulty: u64) { - let (epoch_address, _epoch_bump) = epoch_pda(); - let mut account = svm - .get_account(&epoch_address) - .expect("Epoch account should exist"); - let mut epoch = Epoch::unpack(&account.data) - .expect("Failed to unpack Epoch account") - .clone(); - - epoch.packing_difficulty = difficulty; - - let mut discriminator = [0u8; 8]; - discriminator[0] = Epoch::discriminator(); - let data = [&discriminator, epoch.to_bytes()].concat(); - account.data = data.to_vec(); - - svm.set_account(epoch_address, account) - .expect("failed to override difficulty") -} - fn verify_archive_account(svm: &LiteSVM, expected_tapes_stored: u64) { let (archive_address, _archive_bump) = archive_pda(); let account = svm @@ -464,7 +438,8 @@ fn create_and_verify_tape( writer_address ); - let mut writer_tree = SegmentTree::new(&[tape_address.as_ref()]); + let tape_seed = &[stored_tape.address.as_ref()]; + let mut writer_tree = SegmentTree::new(tape_seed); write_tape( svm, @@ -806,13 +781,15 @@ fn get_packed_segments( stored_tape: &StoredTape, difficulty: u32, ) -> Vec> { + let miner_bytes = miner_address.to_bytes(); + let mem = packx::build_memory(&miner_bytes); let mut packed_segments: Vec> = vec![]; for segment_data in &stored_tape.segments { let canonical_segment = padded_array::(segment_data); - let solution = packx::solve( - &miner_address.to_bytes(), + let solution = packx::solve_with_memory( &canonical_segment, + &mem, difficulty ).expect("Failed to pack segment data"); @@ -835,19 +812,19 @@ fn get_packed_tape( let segment_id = segment_number.to_le_bytes(); let leaf = Leaf::new(&[ segment_id.as_ref(), - &packed_data, + packed_data, ]); merkle_tree.try_add_leaf(leaf) .expect("Failed to add leaf to Merkle tree"); } - return PackedTape { + PackedTape { number: stored_tape.number, address: stored_tape.address, tree: merkle_tree, data: packed_segments, - }; + } } fn commit_for_mining(