refactor: remove sector, use segments directly instead

Cargo.toml

@@ -19,7 +19,7 @@ tape-network = { version = "0.2.1", path = "./network" }
 tape-program = { version = "0.2.1", path = "./program" }
 
 crankx = { version = "0.2.2", features = ["solana"] }
-packx = { version = "0.3.1", features = ["solana"] }
+packx = { version = "0.4.1", features = ["solana"] }
 
 # program dependencies
 bytemuck = "1.14.3"
@@ -87,3 +87,12 @@ http-body-util = "0.1.3"
 
 [profile.release]
 overflow-checks = true
+
+[profile.dev]
+overflow-checks = true
+opt-level = 3
+debug = false
+
+[profile.test]
+opt-level = 3
+debug = false

api/src/consts.rs

@@ -46,12 +46,12 @@ pub const TAPE_PROOF_LEN: usize = TAPE_TREE_HEIGHT;
 /// Segment size in bytes
 pub const SEGMENT_SIZE: usize = 128;
 /// Packed Segment size in bytes
-pub const PACKED_SEGMENT_SIZE: usize = 152; // packx::SOLUTION_SIZE
+pub const PACKED_SEGMENT_SIZE: usize = 145; // packx::SOLUTION_SIZE
 
 /// Maximum number of segments in a tape
-pub const MAX_SEGMENTS_PER_TAPE: usize = 1 << SEGMENT_TREE_HEIGHT - 1;
+pub const MAX_SEGMENTS_PER_TAPE: usize = 1 << (SEGMENT_TREE_HEIGHT - 1);
 /// Maximum number of tapes in a spool
-pub const MAX_TAPES_PER_SPOOL: usize = 1 << TAPE_TREE_HEIGHT - 1;
+pub const MAX_TAPES_PER_SPOOL: usize = 1 << (TAPE_TREE_HEIGHT - 1);
 
 // ====================================================================
 // Token Economics

api/src/types.rs

@@ -31,7 +31,7 @@ impl PoW {
 #[derive(Clone, Copy, Debug, Pod, Zeroable)]
 /// Proof-of-access solution for the tape segment, cryptographically tied to the miner using PackX.
 pub struct PoA {
-    pub bump: [u8; 8],
+    pub bump: u8,
     pub seed: [u8; 16],
     pub nonce: [u8; 128],
     pub path: ProofPath,
@@ -48,7 +48,7 @@ impl PoA {
     }
 
     pub fn as_solution(&self) -> packx::Solution {
-        packx::Solution::new(self.seed, self.nonce, self.bump)
+        packx::Solution::new(self.bump, self.seed, self.nonce)
     }
 }
 

cli/Cargo.toml

@@ -19,6 +19,7 @@ path = "src/main.rs"
 tape-api.workspace = true
 tape-client.workspace = true
 tape-network.workspace = true
+packx.workspace = true
 
 anyhow.workspace = true
 chrono.workspace = true

cli/src/cli.rs

@@ -135,7 +135,10 @@ pub enum SnapshotCommands {
 
     Resync {
         #[arg(help = "Tape account public key to re-sync")]
-        tape: String,
+        tape_address: String,
+
+        #[arg(help = "Miner account public key", short = 'm', long = "miner")]
+        miner_address: Option<String>,
     },
 
     Create {
@@ -150,21 +153,27 @@ pub enum SnapshotCommands {
 
     GetTape {
         #[arg(help = "Tape account public key")]
-        tape: String,
+        tape_address: String,
 
         #[arg(short = 'o', long = "output", help = "Output file")]
         output: Option<String>,
 
         #[arg(short = 'r', long = "raw", help = "Output raw segments instead of decoded tape")]
         raw: bool,
+
+        #[arg(help = "Miner account public key", short = 'm', long = "miner")]
+        miner_address: Option<String>,
     },
 
     GetSegment {
         #[arg(help = "Tape account public key")]
-        tape: String,
+        tape_address: String,
 
         #[arg(help = "Segment index (0 to tape size - 1)")]
         index: u32,
+
+        #[arg(help = "Miner account public key", short = 'm', long = "miner")]
+        miner_address: Option<String>,
     },
 }
 

cli/src/commands/admin.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Result};
-use dialoguer::{theme::ColorfulTheme, Confirm};
 use solana_sdk::signer::Signer;
 
 use crate::cli::{Cli, Context, Commands};
@@ -14,15 +13,6 @@ use tape_client::{
 pub async fn handle_admin_commands(cli:Cli, context: Context) -> Result<()> {
 
     log::print_divider();
-    let proceed = Confirm::with_theme(&ColorfulTheme::default())
-        .with_prompt("→ Are you sure?")
-        .default(false)
-        .interact()
-        .map_err(|e| anyhow::anyhow!("Failed to get user input: {}", e))?;
-    if !proceed {
-        log::print_error("Write operation cancelled");
-        return Ok(());
-    }
 
     match cli.command {
         Commands::Init {} => {

cli/src/commands/network.rs

@@ -1,14 +1,12 @@
 use anyhow::{bail, Result};
 use std::str::FromStr;
 use std::sync::Arc;
-use dialoguer::{theme::ColorfulTheme, Confirm};
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::{signature::Keypair, signer::Signer, pubkey::Pubkey};
 
 use tape_api::prelude::*;
 use tape_client::{register::register_miner, get_miner_account};
 use tape_network::{
-    //archive::archive_loop,
     archive,
     mine::mine_loop,
     web::web_loop,
@@ -114,16 +112,6 @@ pub async fn handle_register(
 
     let (miner_address, _) = miner_pda(context.payer().pubkey(), to_name(&name));
 
-    let proceed = Confirm::with_theme(&ColorfulTheme::default())
-        .with_prompt("→ Are you sure?")
-        .default(false)
-        .interact()
-        .map_err(|e| anyhow::anyhow!("Failed to get user input: {}", e))?;
-    if !proceed {
-        log::print_error("Write operation cancelled");
-        return Ok(());
-    }
-
     register_miner(context.rpc(), context.payer(), &name).await?;
 
     log::print_section_header("Miner Registered");

cli/src/commands/snapshot.rs

@@ -1,6 +1,7 @@
 use std::env;
 use std::io::{self, Write};
 use std::str::FromStr;
+use std::sync::Arc;
 
 use anyhow::Result;
 use chrono::Utc;
@@ -17,28 +18,30 @@ use tapedrive::{decode_tape, MimeType, TapeHeader};
 
 use crate::cli::{Cli, Commands, Context, SnapshotCommands};
 use crate::log;
+use crate::network::get_or_create_miner;
 use crate::utils::write_output;
+use packx;
 
 pub async fn handle_snapshot_commands(cli: Cli, context: Context) -> Result<()> {
     if let Commands::Snapshot(snapshot) = cli.command {
         match snapshot {
             SnapshotCommands::Stats {} => {
                 handle_stats(context)?
             }
-            SnapshotCommands::Resync { tape } => {
-                handle_resync(context, &tape).await?
+            SnapshotCommands::Resync { tape_address, miner_address } => {
+                handle_resync(context, tape_address, miner_address).await?
             }
             SnapshotCommands::Create { output } => {
                 handle_create(context, output)?
             }
             SnapshotCommands::Load { input } => {
                 handle_load(&input)?
             }
-            SnapshotCommands::GetTape { tape, output, raw } => {
-                handle_get_tape(context, &tape, output, raw).await?
+            SnapshotCommands::GetTape { tape_address, miner_address, output, raw } => {
+                handle_get_tape(context, tape_address, miner_address, output, raw).await?
             }
-            SnapshotCommands::GetSegment { tape, index } => {
-                handle_get_segment(context, &tape, index).await?
+            SnapshotCommands::GetSegment { tape_address, miner_address, index } => {
+                handle_get_segment(context, tape_address, miner_address, index).await?
             }
         }
     }
@@ -55,13 +58,36 @@ fn handle_stats(context: Context) -> Result<()> {
     Ok(())
 }
 
-async fn handle_resync(context: Context, tape: &str) -> Result<()> {
-    let tape_pubkey: Pubkey = FromStr::from_str(tape)?;
+async fn handle_resync(
+    context: Context,
+    tape_address: String,
+    miner_address: Option<String>,
+    ) -> Result<()> {
+    let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?;
+
     let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?;
     let starting_slot = tape_account.tail_slot;
-    let store = context.open_primary_store_conn()?;
-    log::print_message(&format!("Re-syncing tape: {tape}, please wait"));
-    sync_from_block(&store, context.rpc(), &tape_pubkey, starting_slot).await?;
+    let store = Arc::new(context.open_primary_store_conn()?);
+
+    let miner_pubkey = get_or_create_miner(
+        context.rpc(), 
+        context.payer(), 
+        miner_address, 
+        None, 
+        false
+    ).await?;
+    log::print_message(&format!("Using miner address: {miner_pubkey}"));
+
+    log::print_message(&format!("Re-syncing tape: {tape_address}, please wait"));
+
+    sync_from_block(
+        &store,
+        context.rpc(),
+        &tape_pubkey, 
+        &miner_pubkey,
+        starting_slot
+    ).await?;
+
     log::print_message("Done");
     Ok(())
 }
@@ -84,20 +110,34 @@ fn handle_load(input: &str) -> Result<()> {
 
 async fn handle_get_tape(
     context: Context,
-    tape: &str,
+    tape_address: String,
+    miner_address: Option<String>,
     output: Option<String>,
     raw: bool,
 ) -> Result<()> {
-    let tape_pubkey: Pubkey = FromStr::from_str(tape)?;
+    let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?;
     let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?;
+
     let total_segments = tape_account.total_segments;
     let store = context.open_read_only_store_conn()?;
+
+    let miner_pubkey = get_or_create_miner(
+        context.rpc(), 
+        context.payer(), 
+        miner_address, 
+        None, 
+        false
+    ).await?;
+    let miner_bytes = miner_pubkey.to_bytes();
+
     let mut data: Vec<u8> = Vec::with_capacity((total_segments as usize) * SEGMENT_SIZE);
     let mut missing: Vec<u64> = Vec::new();
     for seg_idx in 0..total_segments {
         match store.get_segment(&tape_pubkey, seg_idx) {
-            Ok(seg) => {
-                data.extend_from_slice(&seg);
+            Ok(segment_data) => {
+                let solution = packx::Solution::from_bytes(&segment_data.try_into().unwrap());
+                let segment = solution.unpack(&miner_bytes);
+                data.extend_from_slice(&segment);
             }
             Err(StoreError::SegmentNotFoundForAddress(..)) => {
                 data.extend_from_slice(&[0u8; SEGMENT_SIZE]);
@@ -130,8 +170,14 @@ async fn handle_get_tape(
     Ok(())
 }
 
-async fn handle_get_segment(context: Context, tape: &str, index: u32) -> Result<()> {
-    let tape_pubkey: Pubkey = FromStr::from_str(tape)?;
+async fn handle_get_segment(
+    context: Context,
+    tape_address: String,
+    miner_address: Option<String>,
+    index: u32
+) -> Result<()> {
+
+    let tape_pubkey: Pubkey = FromStr::from_str(&tape_address)?;
     let (tape_account, _) = tapedrive::get_tape_account(context.rpc(), &tape_pubkey).await?;
     if (index as u64) >= tape_account.total_segments {
         anyhow::bail!(
@@ -143,10 +189,22 @@ async fn handle_get_segment(context: Context, tape: &str, index: u32) -> Result<
 
     let store = context.open_read_only_store_conn()?;
 
+    let miner_pubkey = get_or_create_miner(
+        context.rpc(), 
+        context.payer(), 
+        miner_address, 
+        None, 
+        false
+    ).await?;
+    let miner_bytes = miner_pubkey.to_bytes();
+
     match store.get_segment(&tape_pubkey, index as u64) {
-        Ok(data) => {
+        Ok(segment_data) => {
+            let solution = packx::Solution::from_bytes(&segment_data.try_into().unwrap());
+            let segment = solution.unpack(&miner_bytes);
+
             let mut stdout = io::stdout();
-            stdout.write_all(&data)?;
+            stdout.write_all(&segment)?;
             stdout.flush()?;
         }
         Err(StoreError::SegmentNotFoundForAddress(..)) => {

cli/src/commands/write.rs

@@ -52,7 +52,7 @@ pub async fn handle_write_command(cli: Cli, context: Context) -> Result<()> {
         let mut header = TapeHeader::new(mime_type, compression_algo, encryption_algo, flags);
 
         let encoded = encode_tape(&data, &mut header)?;
-        let num_segments = (encoded.len() + SEGMENT_SIZE - 1) / SEGMENT_SIZE;
+        let num_segments = encoded.len().div_ceil(SEGMENT_SIZE);
         let chunks: Vec<_> = encoded.chunks(SAFE_SIZE).map(|c| c.to_vec()).collect();
         let chunks_len = chunks.len();
 

client/src/utils/block.rs

@@ -150,6 +150,7 @@ fn merge_events_and_instructions(
 
     // Iterate over events and instructions in parallel
     for (event, instruction) in tape_block.events.iter().zip(&tape_block.instructions) {
+
         match (event, instruction) {
             (EventData::Write(write_event), InstructionData::Write { address, data }) => {
                 merge_write(write_event, address, data, &mut merged)?;
@@ -160,6 +161,7 @@ fn merge_events_and_instructions(
             }
 
             (EventData::Finalize(finalize_event), InstructionData::Finalize { address }) => {
+        log::debug!("Merging event: {:?} with instruction: {:?}", event, instruction);
                 merge_finalize(finalize_event, address, &mut merged)?;
             }
 

network/benches/disk_size_bench.rs

@@ -138,15 +138,15 @@ fn main() {
                     let mut key = Vec::with_capacity(40);
                     key.extend_from_slice(&hash);
                     key.extend_from_slice(&suffix);
-                    store.db.put_cf(&cf_handle, &key, &value).expect("Put failed");
+                    store.db.put_cf(&cf_handle, &key, value).expect("Put failed");
                 }
             }
         } else {
             for _ in 0..(NUM_HASHES * NUM_VALUES_PER_HASH) {
                 let key: [u8; 8] = rng.gen(); 
                 let mut value = [0u8; SEGMENT_SIZE];
                 rng.fill_bytes(&mut value); 
-                store.db.put_cf(&cf_handle, &key, &value).expect("Put failed");
+                store.db.put_cf(&cf_handle, key, value).expect("Put failed");
             }
         }