scraping.py

import re
import requests
import fitz
import time
import datetime
import jm_database
import pandas as pd
import numpy as np

from jelly_ai import JellyAIModel
from jelly_ai_future import JellyAIFutureModel
from io import BytesIO
from bs4 import BeautifulSoup
from haversine import haversine
from datetime import timedelta


def getRequest(url, params={}):
    try:
        response = requests.get(url, params=params)

        if response.status_code != 200:
            print(f"!!! STATUS_CODE : {response.status_code}")
            print(f"url : {url}")
            return None
    except requests.exceptions.RequestException as e:
        print(f"!!! REQUEST_EXCEPTION : {e}")
        print(f"url : {url}")
        return None

    return response


# params = dict()
def getJson(url, params={}):
    res = getRequest(url, params)
    return res.json()


def getJellyPage(start_page = 1, end_page = 1, is_pdf_crawl = False, is_html_crawl = False):
    title_pattern = re.compile(r'해파리 모니터링 주간보고')
    document_no_pattern = re.compile(r'\d+')

    full_jelly_data = []

    page = start_page
    while True:
        print(f"!!!! {page} page !!!!")
        params = {
            'selectPage': page
        }
        url = 'https://www.nifs.go.kr/board/actionBoard0022List.do'
        res = getRequest(url, params)

        main_parser = BeautifulSoup(res.text, 'lxml')
        board_parser = main_parser.find('div', class_='board-list')
        board_list = board_parser.find('table').find('tbody').find_all('tr')

        for board in board_list:
            board_title = board.select('td.subject > a')[0].text
            if title_pattern.search(board_title) is None:
                # 해파리 모니터링 주간보고 게시글이 아닌거야
                continue

            create_date = int(board.select('td[data-label="작성일"]')[0].text[:4])
            if create_date < 2024:
                if is_html_crawl is False:
                    continue
                # 2023년 부터는 File 형태가 아니라 HTML 형태
                link_data = board.select('td.subject > a')[0]['href']
                document_no = document_no_pattern.findall(link_data)
                url = f"https://www.nifs.go.kr/jelly/jemoNewsView.jely?news_seq={document_no[0]}"
                print(board_title, url)
                freeze_cnt = 0
                while freeze_cnt < 5:
                    res = getRequest(url)
                    if res is None:
                        freeze_cnt += 1
                        time.sleep(5)
                        continue
                    break
                full_jelly_data += jelly_html_parse(res.text)

            else:
                if is_pdf_crawl is False:
                    continue
                # 2024년 이후 PDF 파싱
                link_data = board.select('td[data-label="원본"] > a')
                file_name = link_data[0].find('img').get('title')
                url = f"https://www.nifs.go.kr/{link_data[0]['href']}"
                print(board_title, file_name, url)

                freeze_cnt = 0
                while freeze_cnt < 5:
                    pdf_data = getRequest(url, {})
                    if pdf_data is None:
                        freeze_cnt += 1
                        time.sleep(5)
                        continue
                    break

                doc = fitz.open(stream=BytesIO(pdf_data.content), filetype='pdf')
                full_text = ''
                for idx, page_data in enumerate(doc):
                    full_text += page_data.get_text()
                full_jelly_data += jelly_parse(full_text)

        page += 1
        if page > end_page:
            break

    return full_jelly_data

def jelly_html_parse(full_text):
    result_list = []

    percent_pattern = re.compile(r'([0-9.]+)%')
    density_pattern = re.compile(r'(\S)밀도 출현 해역')
    location_pattern = re.compile(r'- ([\s\S]*)')
    date_pattern = re.compile(r'(\d{4}.\d{2}.\d{2})')

    main_parser = BeautifulSoup(full_text, 'lxml')

    # 문서 날짜 가져오기
    date_parser = main_parser.select('#warp > table:nth-child(1) > tr')[1].select('td:nth-child(2)')[0]
    document_date = date_pattern.findall(date_parser.text)[0]

    # 문서 하단의 지역별 출현율 가져오기
    loc_dict = {}
    detail_parser = main_parser.select('div.tabletitle')
    for idx, detail_table in enumerate(detail_parser):
        if '각 지역별 해파리 출현률' in detail_table.text:
            jelly_row_list = detail_table.find_next_sibling().find_all('tr')
            for i, row in enumerate(jelly_row_list):
                if i < 1:
                    # Header pass
                    continue
                location = row.find('th').text[:2]
                loc_per_list = row.find_all('td')
                loc_dict[location] = {
                    '노무라입깃해파리': loc_per_list[0].text if loc_per_list[0].text != '-' else '0',
                    '보름달물해파리': loc_per_list[1].text if loc_per_list[1].text != '-' else '0',
                    '기타': loc_per_list[2].text if loc_per_list[2].text != '-' else '0'
                }
            break

    jelly_tr_list = main_parser.select('#warp > table:nth-of-type(2) > tr')
    for idx, jelly_data in enumerate(jelly_tr_list):
        jelly_info = jelly_data.find_all('td')
        if len(jelly_info) < 1:
            continue

        jelly_name = jelly_info[0].text.strip()
        if jelly_name == '종류':
            continue

        jelly_percent = percent_pattern.findall(jelly_info[2].text.strip())[0]
        jelly_poison = jelly_info[3].text.strip()
        location_list = jelly_info[1].text.replace('\r', '\n').split('\n')
        density = ''
        for loc_data in location_list:
            density_match = density_pattern.findall(loc_data)
            if len(density_match) > 0:
                density = 'high' if density_match[0] == '고' else 'low'
                continue
            loc_match = location_pattern.findall(loc_data)
            if len(loc_match) > 0 :
                percent_loc = '0'
                for t_loc, t_dict in loc_dict.items():
                    if t_loc in loc_match[0]:
                        percent_loc = t_dict[jelly_name if jelly_name in ('노무라입깃해파리', '보름달물해파리') else '기타']
                        break

                result_list.append({
                    'date': f"{document_date} 00:00:00",
                    'jelly_name': jelly_name,
                    'density': density,
                    'location': loc_match[0],
                    'percent': jelly_percent,
                    'percent_loc': percent_loc,
                    'poison': jelly_poison[0]
                })

    return result_list


def jelly_parse(full_text):
    jelly_dict = {}

    date_pattern = re.compile(r'문서번호\n(\d{4}.\d{2}.\d{2})')
    date_match = date_pattern.findall(full_text)
    doc_date = date_match[0]

    loc_pattern = re.compile(r'각 지역별 해파리 출현율[\s\S]*?기타 해파리\n((?:.*?\n){44})')
    location_match = loc_pattern.findall(full_text)
    location_data = location_match[0].split('\n')
    loc_dict = {}
    for i in range(0, len(location_data) - 1, 4):
        loc_dict[location_data[i]] = {
            '노무라입깃해파리': location_data[i + 1] if location_data[i + 1] != '-' else '0',
            '보름달물해파리': location_data[i + 2] if location_data[i + 2] != '-' else '0',
            '기타': location_data[i + 3] if location_data[i + 3] != '-' else '0'
        }

    jelly_list = r'노무라입깃해파리|보름달물해파리|두빛보름달해파리|유령해파리|야광원양해파리|커튼원양해파리|기수식용해파리|작은상자해파리|오이빗해파리|평면해파리|관해파리|상자해파리|유령해파리1'
    high_low_pattern = {
        'low': re.compile(rf'({jelly_list})\n(?:(?!{jelly_list})[\s\S])*?저밀도 출현 해역\n([\s\S]*?)(?={jelly_list}|살파류\(척삭동물\)|저밀도 출현 해역|□)'),
        'high': re.compile(rf'({jelly_list})\n(?:(?!{jelly_list})[\s\S])*?고밀도 출현 해역\n([\s\S]*?)(?={jelly_list}|살파류\(척삭동물\)|저밀도 출현 해역|□)')
    }
    is_location_pattern = re.compile(r' {3}- [\s\S]*?\n')
    data_pattern = re.compile(rf'({jelly_list})\n[\s\S]*?([0-9.]+)%')
    poison_pattern = re.compile(rf'({jelly_list})\n[\s\S]*?(\S)독성')

    matches = data_pattern.findall(full_text)
    for data in matches:
        jelly_dict.setdefault(data[0], {'low': [], 'high': [], 'per': data[1], 'date': doc_date})

    poison_matches = poison_pattern.findall(full_text)
    for data in poison_matches:
        jelly_dict[data[0]]['poison'] = data[1]

    for type, pattern in high_low_pattern.items():
        high_low_matches = pattern.findall(full_text)
        for loc_data in high_low_matches:
            # loc_list = loc_data[1].split('\n')
            loc_list = is_location_pattern.findall(loc_data[1])
            for location in loc_list:
                if location == '':
                    continue
                for key in loc_dict:
                    if key in location:
                        jelly_dict[loc_data[0]][type].append([
                            location.strip().replace('- ', ''),
                            loc_dict[key][loc_data[0] if loc_data[0] in ('노무라입깃해파리', '보름달물해파리') else '기타']
                        ])
                        break

    result_list = []
    for jelly_name, jelly_data in jelly_dict.items():
        for density in ('high', 'low'):
            for loc_info in jelly_data[density]:
                result_list.append({
                    'date': jelly_data['date'] + ' 00:00:00',
                    'jelly_name': jelly_name,
                    'density': density,
                    'location': loc_info[0],
                    'percent': jelly_data['per'],
                    'percent_loc': loc_info[1],
                    'poison': jelly_data['poison'] if 'poison' in jelly_data else '-'
                })

    return result_list


def getRealtimeKHOA(DataType, ObsCode):
    url = f"http://www.khoa.go.kr/api/oceangrid/{DataType}/search.do"
    params = {
        'ServiceKey': 'G8IaYtKIchyy53cuHH1QSQ==',
        # 'ObsCode': 'DT_0005',
        'ObsCode': ObsCode,
        'ResultType': 'json'
    }

    res = getJson(url, params)

    return res

def scrapingTodayOceanData():
    location_data = jm_database.selectAllOceanInfo()
    now_time = datetime.datetime.now()
    now_time = now_time.replace(minute=0, second=0, microsecond=0)
    print(now_time)
    
    jelly_types = ['노무라입깃해파리', '보름달물해파리']
    
    for location_info in location_data:
        print(f"===== {location_info['ocean_title']} =====")
        res = getRealtimeKHOA('buObsRecent', location_info['obs_code'])

        if 'error' in res['result']:
            continue

        # 수온, 염분, 기온, 기압, 풍향, 풍속, 유향, 유속, 유의파고
        realtime_data = {
            'realtime_data': now_time,
            'ocean_id': location_info['id'],
            'water_temp': float(res['result']['data']['water_temp']) if res['result']['data']['water_temp'] != '' else None,
            'psu': float(res['result']['data']['Salinity']) if res['result']['data']['Salinity'] != '' else None,
            'swh': float(res['result']['data']['wave_height']) if res['result']['data']['wave_height'] != '' else None,
            'temp': float(res['result']['data']['air_temp']) if res['result']['data']['air_temp'] != '' else None,
            'hpa': float(res['result']['data']['air_pres']) if res['result']['data']['air_pres'] != '' else None,
            'wind_dir': float(res['result']['data']['wind_dir']) if res['result']['data']['wind_dir'] != '' else None,
            'wind_spd': float(res['result']['data']['wind_speed']) if res['result']['data']['wind_speed'] != '' else None,
            'current_dir': float(res['result']['data']['current_dir']) if res['result']['data']['current_dir'] != '' else None,
            'current_spd': float(res['result']['data']['current_speed']) if res['result']['data']['current_speed'] != '' else None,
        }

        # haversine_list = [haversine(lat_lon, (float(x['lat']), float(x['lon']))) for x in res['result']['data']]
        # min_index = haversine_list.index(min(haversine_list))
        # realtime_data['current_dir'] = res['result']['data'][min_index]['current_direct']
        # realtime_data['current_spd'] = res['result']['data'][min_index]['current_speed']

        # 여기서 없는 값을 조사해서, DB의 마지막 값을 가져오긴 해야된다.
        print(realtime_data)

        db_last_data = None
        for key, val in realtime_data.items():
            if val is not None:
                continue
            if db_last_data is None:
                db_last_data = jm_database.selectLastOceanData(realtime_data['ocean_id'])
            realtime_data[key] = db_last_data[key] if db_last_data is not None else 0
            
        for jelly_type in jelly_types:
            insertAlertData(realtime_data, jelly_type)
        
        jm_database.insertOceanData(realtime_data)
        
    ##############################
    ###여기에서 미래예측부분 실행###
    ##############################
    read62OceanData()


def insertAlertData(realtime_data, jelly_type):
    now_time = datetime.datetime.now()

    model = JellyAIModel()
    model.loadModels()

    # realtime_data를 numpy array로 변환
    df = pd.DataFrame([realtime_data])
    df.drop(columns=['realtime_data', 'ocean_id'], inplace=True)
    # input_data = np.array([[realtime_data.get('water_temp', 0),
    #                         realtime_data.get('psu', 0),
    #                         realtime_data.get('swh', 0),
    #                         realtime_data.get('temp', 0),
    #                         realtime_data.get('hpa', 0),
    #                         realtime_data.get('wind_dir', 0),
    #                         realtime_data.get('wind_spd', 0),
    #                         realtime_data.get('current_dir', 0),
    #                         realtime_data.get('current_spd', 0)]])
    
    appear_pred, density_pred, percent_loc = model.predictJelly(jelly_type, df)

    alert_data = {
        'jelly': jelly_type,
        'beach_id': realtime_data['ocean_id'],  # jm_ocean_info 테이블의 ID
        'time': now_time.strftime('%Y-%m-%d %H'),  # 경보 시간
        'level': '위험',  # 기본값
        'image_url': 'image',  # 기본값
        'appear_pred': int(appear_pred[0][0]),  # 예측된 출현 값
        'density_pred': int(density_pred[0]),  # 예측된 밀도 값
        'percent_loc': float(percent_loc[0][0]),  # 예측된 위치 퍼센트 값
        'create_date': now_time.strftime('%Y-%m-%d %H:%M:%S'),  # 생성 시간
        'update_date': now_time.strftime('%Y-%m-%d %H:%M:%S')  # 수정 시간 (기본값으로 생성 시간)
    }

    jm_database.insertAlertData(alert_data)
    

def read62OceanData():#여기서 62일 전부터 지금까지의 데이터를 가져와 정제해야한다
    location_data = jm_database.selectAllOceanInfo()
    now_time = datetime.datetime.now()
    today = now_time.replace(hour=0, minute=0, second=0, microsecond=0)
    
    jelly_types = ['노무라입깃해파리', '보름달물해파리']
    
    for location_info in location_data:
        # 해운대 외에는 현재 데이터가 없어서, 임시로 예외처리
        if location_info['id'] != 38:
            continue

        datas = jm_database.selectLast62OceanData(location_info['id'])#여기에 해수욕장 id가 들어가야하지만 일단은 해운대만 시연하므로 해운대의 id를 넣었다다

        #datas=pd.DataFrame(datas)
        
        ###여기서 데이터를 일단위로 정제(일단 db쿼리에서 12:00의 데이터만 추출 추후에 고도화)
        
        datas=datas[-61:] #인덱스가 클수록 최근데이터라고 가정정
        datas = pd.DataFrame(datas)

        for jelly_type in jelly_types:
            insertFutureAlertData(datas.copy(), jelly_type)


def insertFutureAlertData(datas, jelly_type):#데이터를 받아서 모델을 실행하고 결과값을 db에 입력
    now_time = datetime.datetime.now()

    model = JellyAIFutureModel()
    model.loadModels()

    ocean_id=datas['ocean_id'][0]
    datas.drop(columns=['ocean_id'], inplace=True)
    

    
    # input_data = np.array([[realtime_data.get('water_temp', 0),
    #                         realtime_data.get('psu', 0),
    #                         realtime_data.get('swh', 0),
    #                         realtime_data.get('temp', 0),
    #                         realtime_data.get('hpa', 0),
    #                         realtime_data.get('wind_dir', 0),
    #                         realtime_data.get('wind_spd', 0),
    #                         realtime_data.get('current_dir', 0),
    #                         realtime_data.get('current_spd', 0)]])
    
    appear_preds = model.predictJelly(jelly_type, datas) #데이터를 내보낼때 1일,...,7일 리스트 형태로 내보냄

    for i, appear_pred in enumerate(appear_preds):
        
        ntime = now_time+timedelta(days=i+1)
        
        alert_data = {
            'jelly': jelly_type,
            'beach_id': ocean_id,  # jm_ocean_info 테이블의 ID
            'time': ntime.strftime('%Y-%m-%d'),  # 경보 시간(미래래)
            'appear_pred': int(appear_pred),  # 예측된 출현 값
            'density_pred': 1,  # 예측된 밀도 값
            'percent_loc': -1,  # 예측된 위치 퍼센트 값
            'create_date': now_time.strftime('%Y-%m-%d %H:%M:%S'),  # 생성 시간
            'update_date': now_time.strftime('%Y-%m-%d %H:%M:%S')  # 수정 시간 (기본값으로 생성 시간)
        }
        print(alert_data)
        jm_database.insertFutureAlertData(alert_data)
    
def insertPastOceanData(location_info):#과거 해양데이터 db에 입력
    
    past_datas = pd.read_excel('해운대 2025.xlsx')
    past_datas = past_datas.to_dict('records')
    #print(past_datas)
    for past_data in past_datas:
        
        # 수온, 염분, 기온, 기압, 풍향, 풍속, 유향, 유속, 유의파고
        past_data = {
            'realtime_data': datetime.datetime.strptime(past_data['관측시간'],'%Y/%m/%d %H:%M:%S'),
            'ocean_id': location_info,
            'water_temp': float(past_data['수온']),
            'psu': float(past_data['염도']),#이걸 어디서 찾지..
            'swh': float(past_data['유의파고(m)']),
            'temp': float(past_data['기온']),
            'hpa': float(past_data['기압(hPa)']),
            'wind_dir': float(past_data['풍향(deg)']),
            'wind_spd': float(past_data['풍속(m/s)']),
            'current_dir': float(past_data['유향(deg)']),
            'current_spd': float(past_data['유속(cm/s)'])
        }
        #print(past_data['realtime_data'])
        jm_database.insertOceanData(past_data)
    
if __name__ == '__main__':
    jm_database.loadConfig()
    # insertPastOceanData(38)
    # read62OceanData()
    #scrapingTodayOceanData()
    scrapingTodayOceanData()