From 50bc166fd3a27fc38ea50603e0ccc0399fe78539 Mon Sep 17 00:00:00 2001 From: Mmeli Date: Mon, 10 Nov 2025 13:24:29 +0200 Subject: [PATCH 1/2] Load clean data to postgres via supabase, will revert to local server --- Back-End/etl/load.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/Back-End/etl/load.py b/Back-End/etl/load.py index e69de29..39a6534 100644 --- a/Back-End/etl/load.py +++ b/Back-End/etl/load.py @@ -0,0 +1,59 @@ +import os +import pandas as pd +import logging +from sqlalchemy import create_engine, text + +LOG_DIR = 'logs/' +os.makedirs(LOG_DIR, exist_ok=True) + +logging.basicConfig( + filename=os.path.join(LOG_DIR, 'load.log'), + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) + +DB_URL = "postgresql://postgres:12345mmeli@db.inhbqpjnhuqfdybtzcsd.supabase.co:5432/postgres" + +PROCESSED_DIR = "data/processed" +SCHEMA_PATH = "Back-End/sql/schema.sql" + +def load_schema(engine): + """ + Executes schema.sql to create tables if they don't exist. + """ + with open(SCHEMA_PATH, "r") as f: + schema_sql = f.read() + with engine.connect() as conn: + conn.execute(text(schema_sql)) + logging.info("✅ Executed schema.sql") + +def load_csv_to_postgres(engine, filename, table_name): + """ + Loads a cleaned CSV into PostgreSQL. + """ + try: + df = pd.read_csv(os.path.join(PROCESSED_DIR, filename)) + df.to_sql(table_name, engine, if_exists="append", index=False) + logging.info(f"✅ Loaded {filename} into {table_name}") + print(f"✅ Loaded {filename} into {table_name}") + except Exception as e: + logging.error(f"❌ Failed to load {filename}: {e}") + print(f"❌ Failed to load {filename}: {e}") + +if __name__ == "__main__": + engine = create_engine(DB_URL) + + load_schema(engine) + + file_table_map = { + "Amazon Sale Report.csv": "amazon_sale_report", + "Cloud Warehouse Compersion Chart.csv": "cloud_warehouse_comparison", + "Expense IIGF.csv": "expense_iigf", + "International sale Report.csv": "international_sale_report", + "May-2022.csv": "may_2022_pricing", + "P L March 2021.csv": "pl_march_2021", + "Sale Report.csv": "sale_report" + } + + for filename, table_name in file_table_map.items(): + load_csv_to_postgres(engine, filename, table_name) From cbd004b14f07c212bc27420e463807c7ecca6f0e Mon Sep 17 00:00:00 2001 From: Mmeli Date: Tue, 11 Nov 2025 09:50:13 +0200 Subject: [PATCH 2/2] queries and crud --- Back-End/sql/crud.sql | 142 +++++++++++++++++++++++++++++++++++++++ Back-End/sql/queries.sql | 135 +++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) diff --git a/Back-End/sql/crud.sql b/Back-End/sql/crud.sql index e69de29..eef272e 100644 --- a/Back-End/sql/crud.sql +++ b/Back-End/sql/crud.sql @@ -0,0 +1,142 @@ +-- CRUD Operations for RetailPulse Backend +-- This file contains basic Create, Read, Update, Delete operations for each table. + +-- =========================================== +-- AMAZON SALE REPORT +-- =========================================== + +-- CREATE: Insert a new record into amazon_sale_report +INSERT INTO amazon_sale_report (order_id, date, amount, sku) +VALUES (?, ?, ?, ?); + +-- READ: Select all records from amazon_sale_report +SELECT * FROM amazon_sale_report; + +-- READ: Select a specific record by order_id +SELECT * FROM amazon_sale_report WHERE order_id = ?; + +-- UPDATE: Update amount for a specific order_id +UPDATE amazon_sale_report SET amount = ? WHERE order_id = ?; + +-- DELETE: Delete a record by order_id +DELETE FROM amazon_sale_report WHERE order_id = ?; + +-- =========================================== +-- CLOUD WAREHOUSE COMPARISON +-- =========================================== + +-- CREATE: Insert a new comparison record +INSERT INTO cloud_warehouse_comparison (shiprocket, increff) +VALUES (?, ?); + +-- READ: Select all comparison records +SELECT * FROM cloud_warehouse_comparison; + +-- READ: Select a specific record by id (assuming auto-increment id) +SELECT * FROM cloud_warehouse_comparison WHERE id = ?; + +-- UPDATE: Update shiprocket value for a specific id +UPDATE cloud_warehouse_comparison SET shiprocket = ? WHERE id = ?; + +-- DELETE: Delete a record by id +DELETE FROM cloud_warehouse_comparison WHERE id = ?; + +-- =========================================== +-- EXPENSE IIGF +-- =========================================== + +-- CREATE: Insert a new expense record +INSERT INTO expense_iigf (received_amount, expense) +VALUES (?, ?); + +-- READ: Select all expense records +SELECT * FROM expense_iigf; + +-- READ: Select a specific record by id +SELECT * FROM expense_iigf WHERE id = ?; + +-- UPDATE: Update received_amount for a specific id +UPDATE expense_iigf SET received_amount = ? WHERE id = ?; + +-- DELETE: Delete a record by id +DELETE FROM expense_iigf WHERE id = ?; + +-- =========================================== +-- INTERNATIONAL SALE REPORT +-- =========================================== + +-- CREATE: Insert a new international sale record +INSERT INTO international_sale_report (date, customer, sku, pcs, rate, gross_amt) +VALUES (?, ?, ?, ?, ?, ?); + +-- READ: Select all international sale records +SELECT * FROM international_sale_report; + +-- READ: Select records for a specific customer +SELECT * FROM international_sale_report WHERE customer = ?; + +-- UPDATE: Update pcs for a specific record (assuming id) +UPDATE international_sale_report SET pcs = ? WHERE id = ?; + +-- DELETE: Delete a record by id +DELETE FROM international_sale_report WHERE id = ?; + +-- =========================================== +-- MAY 2022 PRICING +-- =========================================== + +-- CREATE: Insert a new pricing record +INSERT INTO may_2022_pricing (sku, style_id, category, tp, ajio_mrp) +VALUES (?, ?, ?, ?, ?); + +-- READ: Select all pricing records +SELECT * FROM may_2022_pricing; + +-- READ: Select records for a specific sku +SELECT * FROM may_2022_pricing WHERE sku = ?; + +-- UPDATE: Update tp for a specific sku +UPDATE may_2022_pricing SET tp = ? WHERE sku = ?; + +-- DELETE: Delete a record by sku +DELETE FROM may_2022_pricing WHERE sku = ?; + +-- =========================================== +-- P&L MARCH 2021 +-- =========================================== + +-- CREATE: Insert a new P&L record +INSERT INTO pl_march_2021 (sku, style_id, category, tp_1, ajio_mrp) +VALUES (?, ?, ?, ?, ?); + +-- READ: Select all P&L records +SELECT * FROM pl_march_2021; + +-- READ: Select records for a specific sku +SELECT * FROM pl_march_2021 WHERE sku = ?; + +-- UPDATE: Update tp_1 for a specific sku +UPDATE pl_march_2021 SET tp_1 = ? WHERE sku = ?; + +-- DELETE: Delete a record by sku +DELETE FROM pl_march_2021 WHERE sku = ?; + +-- =========================================== +-- SALE REPORT +-- =========================================== + +-- CREATE: Insert a new sale report record +INSERT INTO sale_report (sku_code, design_no, stock) +VALUES (?, ?, ?); + +-- READ: Select all sale report records +SELECT * FROM sale_report; + +-- READ: Select records for a specific sku_code +SELECT * FROM sale_report WHERE sku_code = ?; + +-- UPDATE: Update stock for a specific sku_code +UPDATE sale_report SET stock = ? WHERE sku_code = ?; + +-- DELETE: Delete a record by sku_code +DELETE FROM sale_report WHERE sku_code = ?; diff --git a/Back-End/sql/queries.sql b/Back-End/sql/queries.sql index e69de29..1db260c 100644 --- a/Back-End/sql/queries.sql +++ b/Back-End/sql/queries.sql @@ -0,0 +1,135 @@ +-- Analytical Queries for RetailPulse Backend +-- This file contains various analytical queries for each table. + +-- =========================================== +-- AMAZON SALE REPORT +-- =========================================== + +-- Total sales amount +SELECT SUM(amount) AS total_sales FROM amazon_sale_report; + +-- Average sales amount per order +SELECT AVG(amount) AS avg_sales_per_order FROM amazon_sale_report; + +-- Top 10 SKUs by total sales amount +SELECT sku, SUM(amount) AS total_amount +FROM amazon_sale_report +GROUP BY sku +ORDER BY total_amount DESC +LIMIT 10; + +-- Sales by date (daily totals) +SELECT date, SUM(amount) AS daily_sales +FROM amazon_sale_report +GROUP BY date +ORDER BY date; + +-- =========================================== +-- CLOUD WAREHOUSE COMPARISON +-- =========================================== + +-- Average shiprocket and increff values +SELECT AVG(shiprocket) AS avg_shiprocket, AVG(increff) AS avg_increff +FROM cloud_warehouse_comparison; + +-- Comparison summary (count, min, max) +SELECT + COUNT(*) AS total_records, + MIN(shiprocket) AS min_shiprocket, + MAX(shiprocket) AS max_shiprocket, + MIN(increff) AS min_increff, + MAX(increff) AS max_increff +FROM cloud_warehouse_comparison; + +-- =========================================== +-- EXPENSE IIGF +-- =========================================== + +-- Total received amount +SELECT SUM(received_amount) AS total_received FROM expense_iigf; + +-- Expenses grouped by type +SELECT expense, SUM(received_amount) AS total_amount +FROM expense_iigf +GROUP BY expense +ORDER BY total_amount DESC; + +-- =========================================== +-- INTERNATIONAL SALE REPORT +-- =========================================== + +-- Total gross amount +SELECT SUM(gross_amt) AS total_gross_sales FROM international_sale_report; + +-- Sales by customer +SELECT customer, SUM(gross_amt) AS total_sales +FROM international_sale_report +GROUP BY customer +ORDER BY total_sales DESC; + +-- Average rate and pcs per sale +SELECT AVG(rate) AS avg_rate, AVG(pcs) AS avg_pcs FROM international_sale_report; + +-- Sales by date +SELECT date, SUM(gross_amt) AS daily_gross_sales +FROM international_sale_report +GROUP BY date +ORDER BY date; + +-- =========================================== +-- MAY 2022 PRICING +-- =========================================== + +-- Average TP and Ajio MRP +SELECT AVG(tp) AS avg_tp, AVG(ajio_mrp) AS avg_ajio_mrp FROM may_2022_pricing; + +-- Pricing by category +SELECT category, AVG(tp) AS avg_tp, AVG(ajio_mrp) AS avg_ajio_mrp +FROM may_2022_pricing +GROUP BY category; + +-- Top 10 SKUs by TP +SELECT sku, tp +FROM may_2022_pricing +ORDER BY tp DESC +LIMIT 10; + +-- =========================================== +-- P&L MARCH 2021 +-- =========================================== + +-- Average TP 1 and Ajio MRP +SELECT AVG(tp_1) AS avg_tp_1, AVG(ajio_mrp) AS avg_ajio_mrp FROM pl_march_2021; + +-- P&L by category +SELECT category, AVG(tp_1) AS avg_tp_1, AVG(ajio_mrp) AS avg_ajio_mrp +FROM pl_march_2021 +GROUP BY category; + +-- Top 10 SKUs by TP 1 +SELECT sku, tp_1 +FROM pl_march_2021 +ORDER BY tp_1 DESC +LIMIT 10; + +-- =========================================== +-- SALE REPORT +-- =========================================== + +-- Total stock across all SKUs +SELECT SUM(stock) AS total_stock FROM sale_report; + +-- Average stock per SKU +SELECT AVG(stock) AS avg_stock FROM sale_report; + +-- Top 10 SKUs by stock +SELECT sku_code, stock +FROM sale_report +ORDER BY stock DESC +LIMIT 10; + +-- Stock by design number +SELECT design_no, SUM(stock) AS total_stock +FROM sale_report +GROUP BY design_no +ORDER BY total_stock DESC;