diff --git a/.github/workflows/develop-build.yml b/.github/workflows/develop-build.yml index 1d524e7..b6835d5 100644 --- a/.github/workflows/develop-build.yml +++ b/.github/workflows/develop-build.yml @@ -31,7 +31,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GH_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} # ======================= # booth-checker @@ -92,4 +92,4 @@ jobs: with: subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_BOOTH_DISCORD }} subject-digest: ${{ steps.push_booth_discord.outputs.digest }} - push-to-registry: true \ No newline at end of file + push-to-registry: true diff --git a/.github/workflows/latest-build.yml b/.github/workflows/latest-build.yml index a005986..00e86c3 100644 --- a/.github/workflows/latest-build.yml +++ b/.github/workflows/latest-build.yml @@ -31,7 +31,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GH_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} # ======================= # booth-checker @@ -104,4 +104,4 @@ jobs: - name: Publish release uses: ncipollo/release-action@v1 with: - generateReleaseNotes: true \ No newline at end of file + generateReleaseNotes: true diff --git a/booth_checker/__main__.py b/booth_checker/__main__.py index ca83332..b3b8070 100644 --- a/booth_checker/__main__.py +++ b/booth_checker/__main__.py @@ -16,6 +16,11 @@ from operator import length_hint from unitypackage_extractor.extractor import extractPackage +try: + from .fbx_diff import build_fbx_path_list, calculate_fbx_diff +except ImportError: + from fbx_diff import build_fbx_path_list, calculate_fbx_diff + from shared import * import booth import booth_sql @@ -254,7 +259,6 @@ def generate_changelog_and_summary(item_data, download_url_list, version_json): return changelog_html_path, s3_object_url, summary_result, diff_found, None - def generate_fbx_changelog_and_summary(item_data, download_url_list, version_json): """Generates changelog information for FBX-only tracking.""" previous_fbx = version_json.get('fbx-files', {}) or {} @@ -269,46 +273,13 @@ def generate_fbx_changelog_and_summary(item_data, download_url_list, version_jso logger.error(f'An error occurred while parsing {filename}: {e}') logger.debug(traceback.format_exc()) - previous_hashes = {file_hash for file_hash in previous_fbx.values()} - current_hashes = {file_hash for file_hash in current_fbx.values()} - - added = [] - changed = [] - deleted = [] - - previous_remaining = dict(previous_fbx) - current_remaining = dict(current_fbx) - - for name in set(previous_fbx.keys()) & set(current_fbx.keys()): - old_hash = previous_fbx[name] - new_hash = current_fbx[name] - if old_hash != new_hash: - changed.append(name) - previous_remaining.pop(name, None) - current_remaining.pop(name, None) - - for name, new_hash in current_remaining.items(): - if new_hash in previous_hashes: - continue - added.append(name) - - for name, old_hash in previous_remaining.items(): - if old_hash in current_hashes: - continue - deleted.append(name) + added_entries, changed_entries, deleted_entries = calculate_fbx_diff(previous_fbx, current_fbx) - if not added and not changed and not deleted: + if not added_entries and not changed_entries and not deleted_entries: logger.info('No FBX hash differences detected; skipping changelog generation.') return None, None, None, False, current_fbx - path_list = [] - for name in sorted(added): - path_list.append({'line_str': name, 'status': 1}) - for name in sorted(changed): - path_list.append({'line_str': name, 'status': 3}) - for name in sorted(deleted): - path_list.append({'line_str': name, 'status': 2}) - + path_list = build_fbx_path_list(added_entries, changed_entries, deleted_entries) tree = build_tree(path_list) html_list_items = tree_to_html(tree) if item_data["changelog_show"] else '' summary_data = files_list(tree) @@ -874,15 +845,6 @@ def strftime_now(): while True: logger.info("BoothChecker cycle started") - # BOOTH Heartbeat check once per cycle - try: - logger.info('Checking BOOTH heartbeat') - requests.get("https://booth.pm", timeout=10) - except requests.RequestException as e: - logger.error(f'BOOTH heartbeat failed: {e}. Skipping this cycle.') - sleep(refresh_interval) - continue - # Recreate temporary folders recreate_folder("./download") recreate_folder("./process") diff --git a/booth_checker/booth.py b/booth_checker/booth.py index 3ee1d2e..8d2cb96 100644 --- a/booth_checker/booth.py +++ b/booth_checker/booth.py @@ -9,14 +9,17 @@ def _extract_download_info(div, link_selector, filename_selector): if not download_link or not filename_div: return None - href = download_link.get("data-href") - filename = filename_div.get_text() + href = download_link.get("data-href") or download_link.get("href") + filename = filename_div.get_text(strip=True) - if not href: + if not href or not filename: return None - href = re.sub(r'[^0-9]', '', href) - return [href, filename] + match = re.search(r'/downloadables/(\d+)', href) + if not match: + return None + + return [match.group(1), filename] def _crawling_base(url, cookie, selectors, shortlist, thumblist, product_only_filter=None): response = requests.get(url=url, cookies=cookie) @@ -72,9 +75,19 @@ def crawling(order_num, product_only, cookie, shortlist=None, thumblist=None): 'product_info_selector': 'a', 'product_info_index': 1, 'thumb_selector': 'img', - 'download_item_selector': 'div.legacy-list-item__center, div[data-test="downloadable"]', - 'download_link_selector': 'a.nav-reverse, div.js-download-button', - 'filename_selector': 'div.flex-\\[1\\] b' + 'download_item_selector': ( + 'div.legacy-list-item__center, ' + 'div.mt-16.desktop\\:flex.desktop\\:justify-between.desktop\\:items-center' + ), + 'download_link_selector': ( + 'div.js-download-button[data-test="downloadable"][data-href*="/downloadables/"], ' + 'a.nav-reverse[href*="/downloadables/"]' + ), + 'filename_selector': ( + 'div.min-w-0.u-text-wrap b, ' + 'div.min-w-0.break-words.whitespace-pre-line, ' + 'div.flex-\\[1\\] b' + ) } return _crawling_base(url, cookie, selectors, shortlist, thumblist, product_only_filter=product_only) @@ -85,9 +98,9 @@ def crawling_gift(order_num, cookie, shortlist=None, thumblist=None): 'product_div_class': 'rounded-16 bg-white p-40 mobile:px-16 mobile:pt-24 mobile:pb-40 mobile:rounded-none', 'product_info_selector': 'div.mt-24.text-left a', 'thumb_selector': 'img', - 'download_item_selector': 'div.w-full.text-left, div[data-test="downloadable"]', - 'download_link_selector': 'a.no-underline.flex.items-center.flex.gap-4, div.js-download-button', - 'filename_selector': "div[class='min-w-0 break-words whitespace-pre-line']" + 'download_item_selector': 'div.mt-16.desktop\\:flex.desktop\\:justify-between.desktop\\:items-center', + 'download_link_selector': 'div.js-download-button[data-test="downloadable"][data-href*="/downloadables/"]', + 'filename_selector': 'div.min-w-0.break-words.whitespace-pre-line, div.min-w-0.u-text-wrap b' } return _crawling_base(url, cookie, selectors, shortlist, thumblist) diff --git a/booth_checker/fbx_diff.py b/booth_checker/fbx_diff.py new file mode 100644 index 0000000..4ffda3d --- /dev/null +++ b/booth_checker/fbx_diff.py @@ -0,0 +1,167 @@ +import os +from collections import Counter, defaultdict + + +def _normalize_fbx_entries(fbx_records): + entries = [] + for path_str, file_hash in sorted(fbx_records.items()): + entries.append( + { + "path": path_str, + "basename": os.path.basename(path_str), + "hash": file_hash, + } + ) + return entries + + +def _remove_exact_name_hash_matches(previous_entries, current_entries): + previous_by_key = defaultdict(list) + current_by_key = defaultdict(list) + + for entry in previous_entries: + previous_by_key[(entry["basename"], entry["hash"])].append(entry) + for entry in current_entries: + current_by_key[(entry["basename"], entry["hash"])].append(entry) + + remaining_previous = [] + remaining_current = [] + + for key in sorted(set(previous_by_key) | set(current_by_key)): + previous_list = previous_by_key.get(key, []) + current_list = current_by_key.get(key, []) + shared_count = min(len(previous_list), len(current_list)) + + remaining_previous.extend(previous_list[shared_count:]) + remaining_current.extend(current_list[shared_count:]) + + return remaining_previous, remaining_current + + +def calculate_fbx_diff(previous_fbx, current_fbx): + previous_entries = _normalize_fbx_entries(previous_fbx) + current_entries = _normalize_fbx_entries(current_fbx) + + remaining_previous, remaining_current = _remove_exact_name_hash_matches( + previous_entries, current_entries + ) + + previous_by_name = defaultdict(list) + current_by_name = defaultdict(list) + + for entry in remaining_previous: + previous_by_name[entry["basename"]].append(entry) + for entry in remaining_current: + current_by_name[entry["basename"]].append(entry) + + added = [] + changed = [] + deleted = [] + + for name in sorted(set(previous_by_name) | set(current_by_name)): + previous_list = sorted( + previous_by_name.get(name, []), + key=lambda entry: (entry["hash"], entry["path"]), + ) + current_list = sorted( + current_by_name.get(name, []), + key=lambda entry: (entry["hash"], entry["path"]), + ) + + shared_count = min(len(previous_list), len(current_list)) + for index in range(shared_count): + changed.append( + { + "basename": name, + "previous_hash": previous_list[index]["hash"], + "current_hash": current_list[index]["hash"], + "previous_path": previous_list[index]["path"], + "current_path": current_list[index]["path"], + } + ) + + deleted.extend(previous_list[shared_count:]) + added.extend(current_list[shared_count:]) + + return added, changed, deleted + + +def _short_hash(file_hash): + return file_hash[:8] + + +def _append_path_context(label, basename, basename_counts, previous_path=None, current_path=None): + is_ambiguous_name = basename_counts[basename] > 1 + path_changed = previous_path and current_path and previous_path != current_path + + if not is_ambiguous_name and not path_changed: + return label + + if previous_path and current_path: + if previous_path == current_path: + return f"{label} {{{current_path}}}" + return f"{label} {{from {previous_path} -> {current_path}}}" + + path_value = current_path or previous_path + return f"{label} {{{path_value}}}" + + +def build_fbx_path_list(added_entries, changed_entries, deleted_entries): + basename_counts = Counter() + + for entry in added_entries: + basename_counts[entry["basename"]] += 1 + for entry in changed_entries: + basename_counts[entry["basename"]] += 1 + for entry in deleted_entries: + basename_counts[entry["basename"]] += 1 + + path_list = [] + + for entry in added_entries: + label = _append_path_context( + entry["basename"], + entry["basename"], + basename_counts, + current_path=entry["path"], + ) + path_list.append( + { + "line_str": f"{label} [new {_short_hash(entry['hash'])}]", + "status": 1, + } + ) + + for entry in changed_entries: + label = _append_path_context( + entry["basename"], + entry["basename"], + basename_counts, + previous_path=entry["previous_path"], + current_path=entry["current_path"], + ) + path_list.append( + { + "line_str": ( + f"{label} " + f"[{_short_hash(entry['previous_hash'])} -> {_short_hash(entry['current_hash'])}]" + ), + "status": 3, + } + ) + + for entry in deleted_entries: + label = _append_path_context( + entry["basename"], + entry["basename"], + basename_counts, + previous_path=entry["path"], + ) + path_list.append( + { + "line_str": f"{label} [old {_short_hash(entry['hash'])}]", + "status": 2, + } + ) + + return path_list diff --git a/booth_discord/booth.py b/booth_discord/booth.py index fded0a5..c56a97a 100644 --- a/booth_discord/booth.py +++ b/booth_discord/booth.py @@ -4,6 +4,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import TimeoutException from bs4 import BeautifulSoup class BoothCrawler(): @@ -11,6 +12,8 @@ def __init__(self, selenium_url): self.selenium_url = selenium_url def get_booth_order_info(self, item_number, cookie): + wait_timeout_seconds = 30 + chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--disable-gpu") @@ -26,32 +29,41 @@ def get_booth_order_info(self, item_number, cookie): driver.refresh() try: - WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.CLASS_NAME, "flex.desktop\\:flex-row.mobile\\:flex-col")) + WebDriverWait(driver, wait_timeout_seconds).until( + EC.presence_of_element_located( + ( + By.CSS_SELECTOR, + "#js-item-order a[href*='/orders/'], #js-item-gift a[href*='/gifts/']" + ) + ) ) html = driver.page_source soup = BeautifulSoup(html, "html.parser") - - product_div = soup.find("div", class_="flex desktop:flex-row mobile:flex-col") - if not product_div: - raise Exception("상품이 존재하지 않거나, 구매하지 않은 상품입니다.") - - order_page = product_div.find("a").get("href") - order_parse = self.parse_url(order_page) + + # Prefer direct purchase order when both order/gift sections are present. + order_link = soup.select_one("#js-item-order a[href*='/orders/']") + if order_link is None: + order_link = soup.select_one("#js-item-gift a[href*='/gifts/']") + if order_link is None: + raise Exception("주문/기프트 링크를 찾지 못했습니다. 쿠키 만료 또는 미구매 상품일 수 있습니다.") + + order_parse = self.parse_url(order_link.get("href", "")) return order_parse - + except TimeoutException as exc: + raise Exception( + f"페이지 로딩이 지연되어 주문 정보를 찾지 못했습니다. ({wait_timeout_seconds}초 대기)" + ) from exc finally: driver.quit() def parse_url(self, url): - # 정규식 정의 - pattern = r"https://(?:accounts\.)?booth\.pm/(orders|gifts)/([\w-]+)" - match = re.match(pattern, url) + pattern = r"(?:https://(?:accounts\.)?booth\.pm)?/(orders|gifts)/([\w-]+)" + match = re.search(pattern, url) if match: gift_flag = match.group(1) == "gifts" # gifts이면 True, orders이면 False order_number = match.group(2) return gift_flag, order_number else: - raise ValueError("URL 형식이 잘못되었습니다.") \ No newline at end of file + raise ValueError("URL 형식이 잘못되었습니다.") diff --git a/docker-compose.yml b/docker-compose.yml index 780972d..ff76307 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: - ./changelog:/root/boothchecker/changelog - ./config.json:/root/boothchecker/config.json depends_on: + - porstgres - chrome restart: unless-stopped logging: diff --git a/tests/test_fbx_diff.py b/tests/test_fbx_diff.py new file mode 100644 index 0000000..3949f27 --- /dev/null +++ b/tests/test_fbx_diff.py @@ -0,0 +1,66 @@ +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "booth_checker")) + +from fbx_diff import build_fbx_path_list, calculate_fbx_diff + + +class FbxDiffTests(unittest.TestCase): + def test_same_basename_with_new_hash_is_reported_as_changed(self): + previous_fbx = {"old_package/model.fbx": "aaaaaaaa11111111"} + current_fbx = {"new_package/model.fbx": "bbbbbbbb22222222"} + + added, changed, deleted = calculate_fbx_diff(previous_fbx, current_fbx) + + self.assertEqual(added, []) + self.assertEqual(deleted, []) + self.assertEqual(len(changed), 1) + self.assertEqual(changed[0]["basename"], "model.fbx") + self.assertEqual(changed[0]["previous_hash"], "aaaaaaaa11111111") + self.assertEqual(changed[0]["current_hash"], "bbbbbbbb22222222") + + path_list = build_fbx_path_list(added, changed, deleted) + self.assertEqual(path_list, [ + { + "line_str": ( + "model.fbx {from old_package/model.fbx -> new_package/model.fbx} " + "[aaaaaaaa -> bbbbbbbb]" + ), + "status": 3, + } + ]) + + def test_duplicate_basenames_keep_unchanged_entries_out_of_diff(self): + previous_fbx = { + "pack_a/model.fbx": "samehash00", + "pack_b/model.fbx": "oldhash11", + } + current_fbx = { + "pack_a/model.fbx": "samehash00", + "pack_c/model.fbx": "newhash22", + } + + added, changed, deleted = calculate_fbx_diff(previous_fbx, current_fbx) + + self.assertEqual(added, []) + self.assertEqual(deleted, []) + self.assertEqual( + changed, + [ + { + "basename": "model.fbx", + "previous_hash": "oldhash11", + "current_hash": "newhash22", + "previous_path": "pack_b/model.fbx", + "current_path": "pack_c/model.fbx", + } + ], + ) + + +if __name__ == "__main__": + unittest.main()