S3fast patch

.github/workflows/deploy-s3-fast.yml

@@ -0,0 +1,28 @@
+name: Build and Deploy S3 Fast
+
+on:
+  push:
+    branches: [s3fast-patch]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    permissions:
+      contents: read
+    uses: loadnetwork/infra/.github/workflows/reusable-build-redeploy.yml@main
+    with:
+      image_name: load-s3-agent
+      pod_selector: app=load-s3-agent
+      kube_namespace: business
+      runner_labels: '["self-hosted","linux","x64"]'
+      wait_for_pods: true
+    secrets:
+      REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+      REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      MAIN_CLUSTER_USERNAME: ${{ secrets.MAIN_CLUSTER_USERNAME }}
+      MAIN_CLUSTER_PWD: ${{ secrets.MAIN_CLUSTER_PWD }}
+      MAIN_CLUSTER_HOST: ${{ secrets.MAIN_CLUSTER_HOST }}
+      MAIN_CLUSTER_CTX: ${{ secrets.MAIN_CLUSTER_CTX }}
+      INFRA_REPO_TOKEN: ${{ secrets.INFRA_REPO_TOKEN }}

.github/workflows/deploy.yml

@@ -17,7 +17,8 @@ jobs:
       image_name: load-s3-agent
       pod_selector: app=load-s3-agent
       kube_namespace: business
-      runner_labels: '["self-hosted","linux","x64"]'
+      cluster_ref:
+      runner_labels: '["self-hosted","linux","x64","load-s3-fast"]'
       wait_for_pods: true
     secrets:
       REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}

README.md

@@ -30,6 +30,18 @@ echo -n "hello custom tagged world"  | curl -X POST https://load-s3-agent.load.n
     -F 'tags=[{"key":"tag1","value":"tag1"},{"key":"tag2","value":"tag2"}]'
 ```
 
+Optional: have the agent publish an offchain provenance record for the uploaded DataItem (the API returns the provenance transaction id in `offchain_provenance_proof`):
+
+```bash
+echo -n "hello provenance world" | curl -X POST https://load-s3-agent.load.network/upload \
+  -H "Authorization: Bearer $load_acc_api_key" \
+  -H "offchain_provenance: true" \
+  -F "file=@-;type=text/plain"
+```
+
+- onchain provenance example: https://viewblock.io/arweave/tx/b6kTeJISHCmKTqaq_GK5g6hCGPWmMgfR7W4FcJwBwGU
+- offchain dataitem: https://gateway.s3-node-1.load.network/resolve/qvnTWVz4QqVAa7DsiiPER3HMArN88clg_SZc1BIS63s
+
 ### Upload data and return an agent private signed DataItem
 
 *** N.B: any private DataItem does not have the tags indexed nor is queryable ***
@@ -102,11 +114,11 @@ curl -X POST https://load-s3-agent.load.network/tags/query \
 
 ```
 
-Pagination follows Arweave's GQL schema: optional `first` (default 25, max 100) and a cursor `after`.
+* Pagination follows Arweave's GQL schema: optional `first` (default 25, max 100) and a cursor `after`.
 
-`full_metadata` flag (`Optional<bool>`) to return the full tags associated with a query's dataitem
+* `full_metadata` flag (`Optional<bool>`) to return the full tags associated with a query's dataitem
 
-`created_after` / `created_before` (ISO-8601/RFC3339 strings) filter items by their `created_at` timestamp (inclusive bounds).
+* `created_after` / `created_before` (ISO-8601/RFC3339 strings) filter items by their `created_at` timestamp (inclusive bounds).
 
 ```bash
 curl -X POST https://load-s3-agent.load.network/tags/query \

src/core/s3.rs

@@ -91,7 +91,7 @@ pub async fn store_dataitem(
         .await?;
 
     println!("INDEX DATA: {:?} {:?} {:?}", &dataitem_id, &content_type, &tags_for_index);
-    index_dataitem(&dataitem_id, content_type, &tags_for_index).await.unwrap();
+    // index_dataitem(&dataitem_id, content_type, &tags_for_index).await.unwrap();
 
     Ok(dataitem_id)
 }
@@ -128,7 +128,7 @@ pub async fn store_signed_dataitem(data: Vec<u8>) -> Result<String, Error> {
         .send()
         .await?;
 
-    index_dataitem(&dataitem_id, &content_type, &tags_for_index).await?;
+    // index_dataitem(&dataitem_id, &content_type, &tags_for_index).await?;
 
     Ok(dataitem_id)
 }