diff --git a/gui/__init__.py b/gui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gui/__pycache__/__init__.cpython-313.pyc b/gui/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..41c74d6 Binary files /dev/null and b/gui/__pycache__/__init__.cpython-313.pyc differ diff --git a/gui/__pycache__/main_window.cpython-313.pyc b/gui/__pycache__/main_window.cpython-313.pyc new file mode 100644 index 0000000..84c35e9 Binary files /dev/null and b/gui/__pycache__/main_window.cpython-313.pyc differ diff --git a/gui/__pycache__/worker.cpython-313.pyc b/gui/__pycache__/worker.cpython-313.pyc new file mode 100644 index 0000000..284b108 Binary files /dev/null and b/gui/__pycache__/worker.cpython-313.pyc differ diff --git a/gui/main_window.py b/gui/main_window.py new file mode 100644 index 0000000..083df3c --- /dev/null +++ b/gui/main_window.py @@ -0,0 +1,57 @@ +import sys +from PySide6.QtWidgets import QApplication, QWidget, QVBoxLayout, QTextEdit, QPushButton +from PySide6.QtCore import Signal, QObject, QThread +from scraper.config import index_url +from .worker import Worker + +class EmittingStream(QObject): + text_written = Signal(str) + + def write(self, text): + self.text_written.emit(str(text)) + + def flush(self): + pass + +class ScraperApp(QWidget): + def __init__(self): + super().__init__() + self.setWindowTitle("Scraper GUI") + self.resize(1024, 768) + + self.log = QTextEdit() + self.log.setReadOnly(True) + btn = QPushButton("Start/Stop scraping") + + layout = QVBoxLayout() + layout.addWidget(self.log) + layout.addWidget(btn) + self.setLayout(layout) + + # redirect print() + self.emitter = EmittingStream() + + btn.clicked.connect(self.run_scraping) + + def run_scraping(self): + + # Creation of thread and worker + self.thread = QThread() + self.worker = Worker(index_url) + self.worker.moveToThread(self.thread) + + # Connctions signals + self.thread.started.connect(self.worker.run) + self.worker.log_signal.connect(self.append_log) + self.worker.finished.connect(self.thread.quit) + + self.thread.start() + + def append_log(self, message): + self.log.append(message) + +if __name__ == "__main__": + app = QApplication(sys.argv) + window = ScraperApp() + window.show() + sys.exit(app.exec()) \ No newline at end of file diff --git a/gui/worker.py b/gui/worker.py new file mode 100644 index 0000000..cc8f32f --- /dev/null +++ b/gui/worker.py @@ -0,0 +1,31 @@ +from PySide6.QtCore import QObject, Signal +from scraper.scraper import scraper + +class Worker(QObject): + log_signal = Signal(str) + finished = Signal() + + def __init__(self, url): + super().__init__() + self.url = url + + def run(self): + import builtins + original_print = builtins.print + + def custom_print(*args, **kwargs): + message = " ".join(map(str, args)) + self.log_signal.emit(message) + original_print(*args, **kwargs) + + builtins.print = custom_print + + try: + self.log_signal.emit("🚀 Starting scraping...\n") + scraper(self.url) + self.log_signal.emit("\n✅ Scraping completed.\n") + except Exception as e: + self.log_signal.emit(f"❌ Error: {str(e)}\n") + finally: + builtins.print = original_print + self.finished.emit() \ No newline at end of file diff --git a/main.py b/main.py index 093632f..053dc84 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,9 @@ -from scraper.scraper import scraper -from scraper.config import index_url +import sys +from PySide6.QtWidgets import QApplication +from gui.main_window import ScraperApp -scraper(index_url) \ No newline at end of file +if __name__ == "__main__": + app = QApplication(sys.argv) + window = ScraperApp() + window.show() + sys.exit(app.exec()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d536418..5f5d133 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/scraper/__pycache__/config.cpython-313.pyc b/scraper/__pycache__/config.cpython-313.pyc index a92da0e..b465e14 100644 Binary files a/scraper/__pycache__/config.cpython-313.pyc and b/scraper/__pycache__/config.cpython-313.pyc differ diff --git a/scraper/__pycache__/extract.cpython-313.pyc b/scraper/__pycache__/extract.cpython-313.pyc index 8d9ad06..41b0714 100644 Binary files a/scraper/__pycache__/extract.cpython-313.pyc and b/scraper/__pycache__/extract.cpython-313.pyc differ diff --git a/scraper/__pycache__/save.cpython-313.pyc b/scraper/__pycache__/save.cpython-313.pyc index 5c648de..846f0c3 100644 Binary files a/scraper/__pycache__/save.cpython-313.pyc and b/scraper/__pycache__/save.cpython-313.pyc differ diff --git a/scraper/__pycache__/scraper.cpython-313.pyc b/scraper/__pycache__/scraper.cpython-313.pyc index 3abe28f..975dfdd 100644 Binary files a/scraper/__pycache__/scraper.cpython-313.pyc and b/scraper/__pycache__/scraper.cpython-313.pyc differ diff --git a/scraper/__pycache__/transform.cpython-313.pyc b/scraper/__pycache__/transform.cpython-313.pyc index 0ad8f04..c5b8537 100644 Binary files a/scraper/__pycache__/transform.cpython-313.pyc and b/scraper/__pycache__/transform.cpython-313.pyc differ diff --git a/scraper/config.py b/scraper/config.py index cc618cc..6ebe741 100644 --- a/scraper/config.py +++ b/scraper/config.py @@ -11,19 +11,19 @@ # Errors list and path exec_errors = [] -errors_path = Path("./data/exec_errors.txt") +errors_path = Path("../data/exec_errors.txt") errors_path.parent.mkdir(parents=True, exist_ok=True) # Skipped images list and path skipped_images = [] -skipped_images_path = Path("./data/skipped_images.txt") +skipped_images_path = Path("../data/skipped_images.txt") skipped_images_path.parent.mkdir(parents=True, exist_ok=True) # Remove folders and files if exist -data_dir = Path("./data/images/") +data_dir = Path("../data/images/") if data_dir.exists(): shutil.rmtree(data_dir) -print("🧹 Dossier ./data/images/ supprimé.") +print("🧹 Dossier ../data/images/ supprimé.") if errors_path.exists(): os.remove(errors_path) diff --git a/scraper/extract.py b/scraper/extract.py index 0dddd4e..96e4e79 100644 --- a/scraper/extract.py +++ b/scraper/extract.py @@ -2,6 +2,9 @@ from bs4 import BeautifulSoup from urllib.parse import urljoin +def display_book(book): + print(f"\n▻ Book \"{book}\" saved\n") + def extract_data(url, errors_log): """Function that extracts datas from url. The given url represents a product page i.e. a book page @@ -31,7 +34,9 @@ def extract_data(url, errors_log): # Extraction of product title title = soup.find('li', class_='active').string - print(f" ▷▷ {title} \n") + + display_book(title) + all_datas['title'] = title diff --git a/scraper/save.py b/scraper/save.py index 9cd7da1..519d81a 100644 --- a/scraper/save.py +++ b/scraper/save.py @@ -24,7 +24,7 @@ def download_image(url_img, file_path, errors_log, skipped_images): directory.mkdir(parents=True, exist_ok=True) print(f"➥ Image destination : {file_path}") - print(f"➥ Path length : {len(str(file_path))}") + #print(f"➥ Path length : {len(str(file_path))}") if not file_path.exists(): with open(file_path, "wb") as fp: @@ -34,7 +34,7 @@ def download_image(url_img, file_path, errors_log, skipped_images): print(f"⚠️ Image already exists : {file_path}") skipped_images.append(file_path) - print("═════\n") + print("="*60+"\n") except Exception as e: print(f"❌ Error for {url_img} → {e}\n") diff --git a/scraper/scraper.py b/scraper/scraper.py index 39310fa..3351f48 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -10,21 +10,25 @@ def display_category(cat): c = "┈" * int((35 - n) / 2) l = "┈" * 60 print(l) - print(c + " PROCESSING CATEGORY ⯈⯈ " + cat + " " + c) + print(f"{c} PROCESSING CATEGORY ⯈⯈ \"{cat}\" {c}") print(l) -def display_category_end(cat, count): - print(f"\n▻ Total books scraped for category {cat} : {count}") - print(f"\n▻ Category {cat} saved to {csv_file}\n\n") +def display_category_end(cat, count, total_count): + print(f"\n▻ Total books scraped for category \"{cat}\" : {count} / {total_count}") + print(f"\n▻ Category {cat} saved to : {csv_file}\n\n") print("─" * 60 + "\n") - +def display_books_urls(books_list): + print("▻▻ List of extracted books urls :\n") + for book in books_list: + print(f"{book[0]}\n") + print("\n") def scraper(url): all_categories = get_all_categories(url) print(f"All categories extracted from {url}\n") - + total_books_count = 0 for category in all_categories: display_category(category) @@ -39,14 +43,14 @@ def scraper(url): # Extraction of all pages URL of a category book_urls_with_category = extract_books_urls_from_category(url, category) - print(f"▻ Extracted urls : {book_urls_with_category}\n") + display_books_urls(book_urls_with_category) book_count = 0 # Loop for scraping all products url of the category example for book_url, cat in book_urls_with_category: # Extraction of datas from the product page url - print("├ Extraction...🚧\n") + print("├ Extraction...🚧") extracted_datas = extract_data(book_url,exec_errors) # Transformation of datas @@ -73,8 +77,8 @@ def scraper(url): download_image(image_url, image_path, exec_errors, skipped_images) book_count += 1 - - display_category_end(category, book_count) + total_books_count += book_count + display_category_end(category, book_count, total_books_count) # Logging files if exec_errors: @@ -93,4 +97,5 @@ def scraper(url): else: message = "\n\n🙁" - print(f"{message} Download completed with {len(exec_errors)} error(s).") \ No newline at end of file + print(f"{message} Download completed with {len(exec_errors)} error(s).\n") + print(f"Total books scraped : {total_books_count}\n") \ No newline at end of file