diff --git a/distibuteModule/.idea/.gitignore b/distibuteModule/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/distibuteModule/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/distibuteModule/.idea/distibuteModule.iml b/distibuteModule/.idea/distibuteModule.iml new file mode 100644 index 0000000..d0876a7 --- /dev/null +++ b/distibuteModule/.idea/distibuteModule.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/distibuteModule/.idea/inspectionProfiles/profiles_settings.xml b/distibuteModule/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/distibuteModule/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/distibuteModule/.idea/misc.xml b/distibuteModule/.idea/misc.xml new file mode 100644 index 0000000..f6104af --- /dev/null +++ b/distibuteModule/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/distibuteModule/.idea/modules.xml b/distibuteModule/.idea/modules.xml new file mode 100644 index 0000000..7fdc090 --- /dev/null +++ b/distibuteModule/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/distibuteModule/.idea/vcs.xml b/distibuteModule/.idea/vcs.xml new file mode 100644 index 0000000..6c0b863 --- /dev/null +++ b/distibuteModule/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/distibuteModule/Dockerfile b/distibuteModule/Dockerfile new file mode 100644 index 0000000..d022409 --- /dev/null +++ b/distibuteModule/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.9 +RUN apt-get update && apt-get install -y libgl1-mesa-glx +COPY ./requirements.txt /app/requirements.txt +WORKDIR /app +EXPOSE 5001 +RUN pip install --trusted-host pypi.python.org -r requirements.txt +CMD ["python", "distributed_flask.py"] \ No newline at end of file diff --git a/distibuteModule/FrameDrop.py b/distibuteModule/FrameDrop.py new file mode 100644 index 0000000..e42b3aa --- /dev/null +++ b/distibuteModule/FrameDrop.py @@ -0,0 +1,227 @@ +import cv2 +import numpy as np +import time +import os +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +import threading + +import copy + +Num=0 + +class framedrop(): + def __init__(self, filePath, savePath, flag, threshold, NodeList): + self.filePath = filePath + self.threshold = threshold + self.cap = cv2.VideoCapture(filePath, apiPreference=None) + self.w = round(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.h = round(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') + self.fps = self.cap.get(cv2.CAP_PROP_FPS) + self.frame = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) + self.videoWriter = cv2.VideoWriter(savePath, self.fourcc, self.fps, (self.w, self.h)) + # Method + self.method = {'CORREL': cv2.HISTCMP_CORREL, # cv2.HISTCMP_CORREL: 상관관계 (1: 완전 일치, -1: 완전 불일치, 0: 무관계) + 'CHISQR': cv2.HISTCMP_CHISQR, # cv2.HISTCMP_CHISQR: 카이제곱 (0: 완전 일치, 무한대: 완전 불일치) + 'INTERSECT': cv2.HISTCMP_INTERSECT, + # cv2.HISTCMP_INTERSECT: 교차 (1: 완전 일치, 0: 완전 불일치 - 1로 정규화한 경우) + 'BHATTACHARYYA': cv2.HISTCMP_BHATTACHARYYA, # cv2.HISTCMP_BHATTACHARYYA 값이 작을수록 유사한 것으로 판단 + 'HELLINGER': cv2.HISTCMP_HELLINGER, + 'CHISQR_ALT': cv2.HISTCMP_CHISQR_ALT, + 'KL_DIV': cv2.HISTCMP_KL_DIV} + self.flag = flag + self.NodeList = NodeList + self.start_time = 0 + self.current_time = 0 + self.droplist = [] + self.startList=[True for _ in range(len(NodeList))] + + + def drop(self): + self.ready() + self.startList = [True for _ in range(len(self.NodeList))] + FirstList=[False for _ in range(len(self.NodeList))] + if not self.cap.isOpened(): # check File exists + print("Video is not opened!") + return + drop_frame = 0 + beforeRet, beforeFrame = self.cap.read() + count = 1 + select_node = self.NodeSelector() + self.send_to_node(beforeFrame, select_node, count) + + self.start_time = time.time() + beforeFrameHist = self.preprocess(beforeFrame) + + before_num = count + print('Frame Count : ' + str(self.frame)) + while (True): + print("totla_frame="+str(self.frame)+" now_frame="+str(count),flush=True) + nowRet, nowFrame = self.cap.read() + if not nowRet: break + nowFrameHist = self.preprocess(nowFrame) + + result = self.calculateSimilarity(beforeFrameHist, nowFrameHist, self.flag) + print("result value : "+str(result)+" threshold : "+str(self.threshold),flush=True) + select_node = self.NodeSelector() + if result >= self.threshold: + drop_frame += 1 + self.droplist.append(before_num) + else: + # nowFrame을 전송 + temp_name=copy.deepcopy(select_node.name) + temp_name=temp_name.replace("node","") + selectedNodeNum = int(temp_name) + print(self.startList,flush=True) + print("selectedNodeNum : "+ str(selectedNodeNum),flush=True) + global Num + if FirstList[selectedNodeNum-1] == False: + port_num=30100+selectedNodeNum + threading.Thread(target=self.send_start_message, args=[port_num,select_node.ip]).start() + FirstList[selectedNodeNum-1] = True + self.startList[selectedNodeNum-1] = False + Num+=1 + self.send_to_node(nowFrame, select_node, count) + before_num = count + beforeFrameHist = nowFrameHist + beforeFrame = nowFrame + count += 1 + + + # similarity = "#"+str(count) + " -> " + str(round(result,6)) + "\n" + self.cap.release() + # self.merge_im(drop_frame) + print("send_end_message",flush=True) + self.send_end_message() + + return self.startList + def ready(self): + for n in self.NodeList: + temp_name=copy.deepcopy(n.name) + temp_name=temp_name.replace("node","") + selectedNodeNum = int(temp_name) + print(self.startList,flush=True) + print("selectedNodeNum : "+ str(selectedNodeNum),flush=True) + port_num=30100+selectedNodeNum + retries = Retry(total=10000, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) + retries.initial = 1 + adapter = HTTPAdapter(max_retries=retries) + + # Session 생성 및 Retry 설정 적용 + session = requests.Session() + session.mount('http://', adapter) + session.mount('https://', adapter) + + url ='http://'+n.ip + ':' + str(port_num) + '/ready' + print(url,flush=True) + response = session.get(url) + e=time.time() + print(response,flush=True) + return + + def send_start_message(self,port_num,ip): + # Retry 설정 + s=time.time() + print("port num : "+str(port_num)+"ip is : "+str(ip),flush=True) + retries = Retry(total=10000, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) + retries.initial = 1 + adapter = HTTPAdapter(max_retries=retries) + + # Session 생성 및 Retry 설정 적용 + session = requests.Session() + session.mount('http://', adapter) + session.mount('https://', adapter) + + url ='http://'+ip + ':' + str(port_num) + '/start' + print(url,flush=True) + response = session.get(url) + e=time.time() + print(response,flush=True) + print(url+"time : "+str(e-s),flush=True) + global Num + Num-=1 + return + + + def send_end_message(self): + retries = Retry(total=10000, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]) + adapter = HTTPAdapter(max_retries=retries) + + # Session 생성 및 Retry 설정 적용 + session = requests.Session() + session.mount('http://', adapter) + session.mount('https://', adapter) + while(True): + if Num==0: + time.sleep(3) + break + for now in self.NodeList: + port_num = 30100+int(now.name.replace("node","")) + url = 'http://' + now.ip + ':' + str(port_num) + '/end' + print(url,flush=True) + response = session.get(url) + print(response,flush=True) + return response + + def merge_im(self, drop_frame): + while True: + if len(os.listdir("/home/share/nfs/result/")) >= self.frame - drop_frame: # 디렉터리 내 파일 갯수가 num_files 이상이 면 + break # 반복문 종료 + else: + time.sleep(1) # 1초 대기 후 다시 실행 + for dirpath, dirnames, filenames in os.walk("/home/share/nfs/result/"): + for f in filenames: + fp = os.path.join(dirpath, f) + while os.path.getsize(fp) == 0: ## 크기가 0인 파일이 있다면 대기 + time.sleep(1) + + def NodeSelector(self): + self.current_time = time.time() + elapsed_time = self.current_time - self.start_time + self.start_time=time.time() + min_endtime = 100000000 + for node in self.NodeList: + print(node.name+"PCT : "+ str(node.PCT)+ " PT : "+str(node.PT),flush=True) + node.PCT = node.PCT - elapsed_time if node.PCT - elapsed_time > 0 else 0 + if min_endtime > abs(node.PCT - node.TT) + node.PT: + min_endtime = abs(node.PCT - node.TT) + node.PT + select_node = node + return select_node + + def send_to_node(self, Frame, node, count): + node.PCT += node.PT + print(count) + directory_path = "/home/share/nfs/" + node.name + if not os.path.isdir(directory_path): + try: + os.mkdir(directory_path) + except OSError as error: + print(error) + + file_path = directory_path + "/" + str(count) + ".png" + cv2.imwrite(file_path, Frame) + print(node.name + "선택") + print() + + + + def preprocess(self, frame): + # ---① 각 이미지를 HSV로 변환 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # ---② H,S 채널에 대한 히스토그램 계산 + hist = cv2.calcHist([hsv], [0, 1], None, [180, 256], [0, 180, 0, 256]) + # ---③ 0~1로 정규화 + result = cv2.normalize(hist, hist, 0, 1, cv2.NORM_MINMAX) + return result + + def calculateSimilarity(self, beforeFrameHist, nowFrameHist, flag): + + ret = cv2.compareHist(beforeFrameHist, nowFrameHist, self.method[flag]) + + if flag == cv2.HISTCMP_INTERSECT: + ret = ret / np.sum(beforeFrameHist) + + return ret + diff --git a/distibuteModule/cpu.yaml b/distibuteModule/cpu.yaml new file mode 100644 index 0000000..6563edd --- /dev/null +++ b/distibuteModule/cpu.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: yolonode1 +spec: + selector: + matchLabels: + run: yolonode1 + replicas: 1 + template: + metadata: + labels: + run: yolonode1 + spec: + volumes: + - name: my-pv-storage + persistentVolumeClaim: + claimName: my-pv-claim + containers: + - name: yolonode1 + image: dlskawo0409/capston:22_11_17 + imagePullPolicy: Never + ports: + - containerPort: 80 + command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"] + volumeMounts: + - mountPath: "/home/share/nfs" + name: my-pv-storage + + nodeSelector: + key: worker1 diff --git a/distibuteModule/distribute_flask.py b/distibuteModule/distribute_flask.py new file mode 100644 index 0000000..f5cfe56 --- /dev/null +++ b/distibuteModule/distribute_flask.py @@ -0,0 +1,246 @@ +from flask import Flask, jsonify, request +import threading +import json +import cv2 +import node2 +import requests +import urllib.parse +import os +import time +import subprocess +from FrameDrop import framedrop + +app = Flask(__name__) +nodeList = [] +droplist = [] +total_frames = 0 +dropframe=0 +fps = 0 +start = 0 +mergeTime = 0 +numcount=0 +class VideoInformation: + def __init__(self, frameWidth, frameHeight, frameCount, fps, videoLength, nodeCount): + self.frameWidth = frameWidth + self.frameHeight = frameHeight + self.frameCount = frameCount + self.fps = fps + self.videoLength = videoLength + self.videoPath = "" + self.nodeCount = nodeCount + +class DownloadInformation: + def __init__(self, downloadPath, waitTime,dropCount): + self.downloadPath = downloadPath; + self.waitTime = waitTime; + self.dropCount=dropCount + + +def do_something(AvailNode_List, videoPath,threshold): + print("do something") + # resultPath = videoPath.split("'\'")[:-1] + resultPath = os.path.dirname(videoPath) + resultPath = os.path.join(resultPath, "result") + resultPath = os.path.join(resultPath, "result_" + os.path.basename(videoPath)) + print(resultPath) + global start + start = time.time() + FrameDrop = framedrop(videoPath, resultPath, 'CORREL', threshold, + AvailNode_List) + global dropframe + dropframe = FrameDrop.drop() + + directory_path = '/home/share/nfs/result' # 체크하려는 디렉토리 경로로 변경해야 합니다 + while True: + print(str(dropframe)+" "+str(total_frames),flush=True) + file_count = count_files(directory_path) + result=total_frames-dropframe + print("result : "+str(result)+" file_count : "+str(file_count),flush=True) + if result== file_count or result == file_count+1 or result==file_count-1 : + break + time.sleep(1) + merge() + response = requests.get('http://192.168.0.11:30600/distribution/download') + delete(nodeList) + + +def count_files(directory): + count = 0 + for _, _, files in os.walk(directory): + count += len(files) + print(count,flush=True) + return count + +@app.route("/") +def hello(): + return "Hello main_falsk" + + +@app.route("/videoinformation") +def home(): + print(":hi") + try: + + uri = request.url + except Exception: + print(1) + try: + # URI 디코딩 + decoded_uri = urllib.parse.unquote(uri) + except Exception: + print(2) + # 쿼리스트링 파라미터 가져오기 + try: + + parsed_uri = urllib.parse.urlparse(decoded_uri) + except Exception: + print(3) + try: + query_params = urllib.parse.parse_qs(parsed_uri.query) + except: + print(4) + print() + + # 필요한 파라미터 값 가져오기 + mode = query_params.get('mode')[0] + videoPath = query_params.get('filepath')[0] + threshold = float(query_params.get('threshold')[0]) + + print("mode : " + mode) + print() + + if videoPath == "": + print("get fileName is fail") + return jsonify({'error': 'get fileName is fail'}) + + print("videoPath " + videoPath) + print() + + cap = cv2.VideoCapture(videoPath, apiPreference=None) + if not cap.isOpened(): # check File exists + print("Video is not opened!") + return jsonify({'error': 'Video is not opened!'}) + global fps, total_frames + + fps = cap.get(cv2.CAP_PROP_FPS) # FPS (초당 프레임 수) + width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # 영상 가로 크기 + height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # 영상 세로 크기 + total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) # 총 프레임 수 + + model_flops = 1 + try: + with open("/home/share/nfs/flops/model_flops.txt", "r") as file: + model_flops = file.read() + file.close() + print("model flops is " + str(model_flops)) + except FileNotFoundError: + print("model_flops file does not exist.") + + model_flops = (float)(model_flops) * (round(width)/512) * (round(height)/512) + + print('http://192.168.0.11:5002/start/' + mode) + response = requests.get('http://192.168.0.11:5002/start/' + mode) + str_data = response.content.decode('utf-8') + # JSON 문자열을 파싱하여 딕셔너리로 변환/ + json_data = json.loads(str_data) + json_data2 = json.loads(json_data) + AvailNode_List = [ + node2.Node(ob['name'], ob['cpu_usage'], ob['total_cpu'], ob['memory_usage'], ob['total_memory'], ob['isgpu'], + ob['FLOPS'], ob['ip']) for ob in json_data2] + print(AvailNode_List) + + # for i in range(len(AvailNode_List)): + # nodeList.append(False) + + for n in AvailNode_List: + n.print_node() + deleteAndMakeFile(n.name) + #nodeList.append(False) + n.FLOPS = float(n.FLOPS) + if model_flops != 0: + n.PT = model_flops / n.FLOPS + else: + n.PT = 9999999/ n.FLOPS + deleteAndMakeFile("result") + + threading.Thread(target=do_something, args=[AvailNode_List, videoPath,threshold]).start() + video_information = VideoInformation(width, height, total_frames, fps, total_frames / fps, len(AvailNode_List)) + return jsonify(vars(video_information)) + +def deleteAndMakeFile(nodeName): + basePath = "/home/share/nfs" + filePath = os.path.join(basePath, nodeName) + for file in os.scandir(filePath): + os.remove(file.path) + + + +@app.route("/yolo/end/") +def check_end(nodeNum): + print(nodeNum, flush=True) + nodeList[nodeNum - 1] = True + if all(nodeList): + merge() + response = requests.get('http://192.168.0.11:30600/distribution/download') + # requests.get('http://192.168.0.3:8080/distribution/download') + delete(nodeList) + return 'OK' +def delete(node): + response = requests.get('http://192.168.0.11:5002/end/'+str(len(node))) + + +def merge(): + num = 0 + image_dir = "/home/share/nfs/result/" + images_files = sorted([f for f in os.listdir(image_dir) if f.endswith('.png')]) + first_png_path = os.path.join(image_dir, images_files[0]) + first_image = cv2.imread(first_png_path) + height, width, _ = first_image.shape + + # VideoWriter 객체 생성 + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + print(total_frames, fps, width, height, flush=True) + video_writer = cv2.VideoWriter("/home/share/nfs/result/result.mp4", fourcc, fps, (width, height)) + print("im herer1111", flush=True) + for i in range(1, int(total_frames)): + file_name = str(i) + ".png" + image_path = os.path.join(image_dir, file_name) + print(image_path, flush=True) + if os.path.exists(image_path): + print(str(i) + 'is true', flush=True) + frame = cv2.imread(image_path) + # 영상에 프레임 추가 + video_writer.write(frame) + + # 사용한 리소스 해제 + video_writer.release() + global mergeTime + global start + end = time.time() + mergeTime = end - start; + +@app.route("/yolo/progress/", methods=['POST']) +def send_Progress2Spring(nodeNum): + temp = request.get_data().decode( 'utf-8' ) + temp = temp.split("=")[1] + persent = int(float(temp) *100) + print(persent, flush=True) + url = 'http://192.168.0.11:30600/progress/' +str(nodeNum) +"/" +str(persent) + resource = requests.post(url) + return 'OK' + +@app.route("/download") +def returnDownloadInformation(): + global mergeTime + filePath = "/home/share/nfs/result/result.mp4" + downloadInformation = DownloadInformation(filePath, mergeTime,dropframe) + print(jsonify(vars(downloadInformation)), flush=True) + return jsonify(vars(downloadInformation)) + +@app.route("//ready") +def ready(nodeNum): + global numcount + numcount+=1 + return "OK" +if __name__ == "__main__": + app.run(host='0.0.0.0', port=5001, debug=True) diff --git a/distibuteModule/distributed_server.yaml b/distibuteModule/distributed_server.yaml new file mode 100644 index 0000000..35681cb --- /dev/null +++ b/distibuteModule/distributed_server.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 # for versions before 1.8.0 use apps/v1beta1 +kind: Deployment +metadata: + name: distributed-server-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: distributed-server + template: + metadata: + labels: + app: distributed-server + spec: + volumes: + - name: my-pv-storage + persistentVolumeClaim: + claimName: my-pv-claim + containers: + - name: distributed-web-server + image: jaehyuk00/distributed_flask:v6 + ports: + - containerPort: 5001 + volumeMounts: + - mountPath: "/home/share/nfs" + name: my-pv-storage + imagePullPolicy: Never + + nodeSelector: + key: master +--- +apiVersion: v1 +kind: Service +metadata: + name: distributed-web-server-service +spec: + ports: + - name: "5001" + port: 5003 + targetPort: 5001 + nodePort: 30500 + selector: + app: distributed-server + type: NodePort + diff --git a/distibuteModule/flops4.py b/distibuteModule/flops4.py new file mode 100644 index 0000000..7dbc8aa --- /dev/null +++ b/distibuteModule/flops4.py @@ -0,0 +1,147 @@ +import time +from ast import parse +#python3 flops.py --weights yolov5s.pt --device 0 --path /home/share/nfs/flops --nodeNum 2 +#python3 flops.py --weights yolov5s.pt --path /home/share/nfs/flops --nodeNum 2 --width 512 --height 512 + +#python3 flops.py --weights yolov5s.pt --path /home/share/nfs/flops --nodeNum 2 --width 1024 --height 1024 + +import torch +import argparse +import os +import sys + + +from utils.general import check_requirements +from pathlib import Path +from ptflops import get_model_complexity_info +from models.common import DetectMultiBackend +from utils.torch_utils import select_device + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model path(s)') + parser.add_argument('--width', nargs='+', type=int, default=512, help='video width') + parser.add_argument('--height', nargs='+', type=int, default=512, help='video height') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--path', default="/home/share/nfs/flops/", type=str) + parser.add_argument('--nodeNum', default=1 ,type=int) + opt = parser.parse_args() + return opt + +def main(opt): + check_requirements(exclude=('tensorboard', 'thop')) + opt_save = opt + run(**vars(opt_save)) + +def saveNodeFlops(path, nodeNum, board_flops): + change = False + if os.path.exists(path): + newLines = [] + with open(path, 'r') as file: + lines = file.readlines() + for line in lines: + line = line.strip() + nowNum = int(line.split()[0].replace("node" ,"")) + if(nowNum == nodeNum): + continue + newLines.append([nowNum, line+"\n"]) + file.close() + newLines.append([nodeNum,"node"+str(nodeNum)+" "+str(round(board_flops,4))+"\n" ]) + newLines.sort(key=lambda x:x[0]) + with open(path, 'w') as file: + for line in newLines: + file.write(line[1]) + file.close() + + else: # 파일이 존재하지 않을 때 + + with open(path, 'w+') as file: + file.write("node"+str(nodeNum)+" "+str(round(board_flops,4))+"\n") + file.close() + +def saveModelFlops(path, model_flops): + if not os.path.exists(path): + print("there is no file!") + return + + path = os.path.join( path, "model_flops.txt") + if os.path.exists(path): + return + else: + with open(path, 'w+') as file: + file.write(str(model_flops)) + file.close + + + +def run( + + weights, + width, + height, + device, + path , + nodeNum + +): + dnn = False + data = None + half = False + width = width[0] + height = height[0] + + + device = select_device(device) + print(f"Using device: {device}") + + input_data = torch.randn(1, 3, width, height) + print(input_data.shape) + + + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + if torch.cuda.is_available(): + # 모델과 입력 데이터를 GPU(CUDA) 메모리로 이동 + model.cuda() + input_data = input_data.cuda() + + model_flops, params = get_model_complexity_info(model, (3,width,height), as_strings=False, print_per_layer_stat=False) + + print(f"Model FLOPs: {model_flops/ 10e9}G") + + # save model Flops + saveModelFlops(path, model_flops / 10e9) + + + _ = model.forward(input_data) + # Measure inference time + start_time = time.time() + #print(start_time) + for i in range(3): + _ = model.forward(input_data) + end_time = time.time() + #print(end_time) + inference_time = (end_time - start_time) /3 + #print(inference_time) + + # Compute FLOPs of board_flops + board_flops = model_flops / inference_time / 10e9 + + print(f"Inference time per frame: {inference_time:.4f} seconds") + print(f"boardFlops: {board_flops:.4f}G") + + flopsFileName = "node_flops.txt" + path = os.path.join(path, flopsFileName) + saveNodeFlops(path,nodeNum,board_flops) + + + + +if __name__ == "__main__": + opt = parse_opt() + main(opt) \ No newline at end of file diff --git a/distibuteModule/gpuyolo.yaml b/distibuteModule/gpuyolo.yaml new file mode 100644 index 0000000..1e47b9a --- /dev/null +++ b/distibuteModule/gpuyolo.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: yolonode2 +spec: + selector: + matchLabels: + run: yolonode2 + replicas: 1 + template: + metadata: + labels: + run: yolonode2 + spec: + volumes: + - name: my-pv-storage + persistentVolumeClaim: + claimName: my-pv-claim + containers: + - name: yolonode2 + image: dlskawo0409/gpu:v3 + imagePullPolicy: Never + ports: + - containerPort: 5001 + command: ["python3", ".yolo/yoloFlask.py", "--filePath", "/home/share/nfs/node1", "--save_dir", "/home/share/nfs/result", "--address", "192.168.0.11:30700"] + volumeMounts: + - mountPath: "/home/share/nfs" + name: my-pv-storage + resources: + limits: + nvidia.com/gpu: 1 # requesting 1 GPUs + + nodeSelector: + key: worker2 +--- +apiVersion: v1 +kind: Service +metadata: + name: node1yolo +spec: + ports: + - name: "80" + port: 5100 + targetPort: 5001 + nodePort: 30800 + selector: + app: node1yolo + type: NodePort diff --git a/distibuteModule/home/share/nfs/traffic-mini.mp4 b/distibuteModule/home/share/nfs/traffic-mini.mp4 new file mode 100644 index 0000000..05285da Binary files /dev/null and b/distibuteModule/home/share/nfs/traffic-mini.mp4 differ diff --git a/distibuteModule/main.py b/distibuteModule/main.py new file mode 100644 index 0000000..18ac0af --- /dev/null +++ b/distibuteModule/main.py @@ -0,0 +1,34 @@ + +import manager +import node +from flask import Flask, request, jsonify +import json +import os + +app = Flask(__name__) + +m = manager.kubemanager() +m.spring_web_server() +m.distributed_server() +m.node_flops_check() +@app.route('/start/') +def distribute_start(mode): + new_directory = "/home/share/nfs/result" + if not os.path.exists(new_directory): + os.makedirs(new_directory) + + Avail_NodeList = m.Node_Information(mode) + print("main print") + for n in Avail_NodeList: + n.print_node() + json_data = json.dumps([obj.__dict__ for obj in Avail_NodeList]) + print(json_data) + return jsonify(json_data) + +@app.route('/distribution/download') +def happy(): + print('happy') + return 'Ok' + +if __name__ == '__main__': + app.run(debug=True, host='0.0.0.0', port=5002) diff --git a/distibuteModule/manager.py b/distibuteModule/manager.py new file mode 100644 index 0000000..5ed484d --- /dev/null +++ b/distibuteModule/manager.py @@ -0,0 +1,293 @@ +import subprocess +import requests +import time +import node +import yaml +import os + +class kubemanager(): + def __init__(self) -> None: + self.spring_web_server_yaml='./spring_web_server.yaml' + self.distributed_server_yaml='./distributed_server.yaml' + self.Upload_File_Check='http://192.168.0.5:30620/FileIsUpload' + self.NodeList=[] + self.Available_Node=[] + self.threshold=0 + + # spring 웹 서버 POD 쿠버네티스에 뛰움 + def spring_web_server(self): + output = subprocess.check_output('kubectl apply -f '+self.spring_web_server_yaml, shell=True).decode() + print(output) + + + # 분배모듈 서버 POD 쿠버네티스에 뛰움 + def distributed_server(self): + output = subprocess.check_output('kubectl apply -f '+self.distributed_server_yaml, shell=True).decode() + print(output) + + + # Yaml 파일이 존재하는지 검사 + def Yaml_File_Check(output): + return "does not exist" not in output + + # spring 웹 서버에서 File Upload될 때 까지 기다림 + def response(self): + response="" + while(response!=True): + response = requests.get(self.Upload_File_Check) + time.sleep(1) + return + + # 현재 쿠버네티스에 접속 중인 노드 확인 후 노드 정보를 가진 객체 생성 + def Node_Information(self, mode): + self.NodeList = [] + self.Available_Node = [] + output = subprocess.check_output('kubectl top nodes', shell=True).decode() + result = output.split() + if len(result) <= 5: + print("There are no nodes in Kubernetes") + else: + self.getNodeInfo(result, mode) + + return self.Available_Node_Check() + + def getNodeInfo(self, result, mode): + for i in range(5, len(result), 5): + if '' in result[i:i + 5]: continue + if result[i] == 'master': continue + nodeName = result[i] + nodeIp = self.getInternalIp(nodeName) + print(str(nodeName)+" ip is "+str(nodeIp)) + if nodeIp != "": + nodeInfo = result[i:i + 5] + nodeInfo.append(nodeIp) + new_node = node.Node(nodeInfo) + self.NodeList.append(new_node) + if mode == "single" and len(self.NodeList) == 1: + break + else: + continue + + + + def getInternalIp(self, nodeName): + + output = subprocess.check_output('kubectl get no -o wide | grep '+nodeName, shell=True).decode() + result = output.split() + nodeIp = "" + if len(result) <= 10: + print('cant get result of "kubectl get no -o wide" ') + else: + if result[0] == nodeName: + nodeIp = result[5] + if nodeIp is None: + print('error get nodeIp ' + str(nodeName)) + else: + return nodeIp + + + + # 노드 중 Yolo실행 가능한 노드 검색 + def Available_Node_Check(self): + flops={} + file_path="/home/share/nfs/flops/node_flops.txt" + if os.path.exists(file_path): + file = open(file_path, 'r') + else: + print("파일이 존재하지 않습니다.") + + + while True: + line = file.readline() + if not line: break + node_name,flops_info=line.split(" ") + flops[node_name]=flops_info + print("nodeList "+str(len(self.NodeList))) + for n in self.NodeList: + if n.total_memory-n.memory_usage > self.threshold and n.name in flops: + n.set_flops(flops[n.name]) + self.Available_Node.append(n) + return self.Yolo_Pod_exec() + + + # 사용가능 한 노드에는 Yolo Pod 생성 + def Yolo_Pod_exec(self): + for n in self.Available_Node: + if n.isgpu: + with open("gpuyolo.yaml", "r") as f: + data = yaml.safe_load_all(f) + for item in data: + print(item) + if item.get("kind") == "Deployment": + yaml_data = item + break + + # Service 파트 찾기 + for item in data: + if item.get("kind") == "Service": + service_data = item + break + else: + with open("cpu.yaml", "r") as f: + data = yaml.safe_load_all(f) + for item in data: + print(item) + if item.get("kind") == "Deployment": + yaml_data = item + break + + # Service 파트 찾기 + for item in data: + if item.get("kind") == "Service": + service_data = item + break + POD_Name=n.name+"yolo" + print("!2312312") + + + print(yaml_data["metadata"]["name"]) + yaml_data["metadata"]["name"]=POD_Name + yaml_data["spec"]["selector"]["matchLabels"]["app"], \ + yaml_data["spec"]["template"]["metadata"]["labels"]["app"] = ["updated-label"] * 2 + yaml_data['spec']['template']['spec']['containers'][0]['name'] = POD_Name + if n.isgpu: + yaml_data['spec']['template']['spec']['containers'][0]['command']=\ + ["python3", "./yolo/yoloFlask.py", "--filePath", "/home/share/nfs/"+n.name, "--save_dir", "/home/share/nfs/result", "--address", "192.168.0.11:30500"] + else: + yaml_data['spec']['template']['spec']['containers'][0]['command']=\ + ["python3", "./yoloFlask.py", "--filePath", "/home/share/nfs/"+n.name, "--save_dir", "/home/share/nfs/result", "--address", "192.168.0.11:30500"] + yaml_data['spec']['template']['spec']['nodeSelector']['key'] = n.name + + with open(POD_Name+'.yaml', "w") as f: + yaml.dump(yaml_data, f) + f.write("---\n") + print(service_data["metadata"]["name"]) + service_data["metadata"]["name"]=POD_Name + service_data["spec"]["selector"]["app"]="updated-label" + service_data["spec"]["ports"][0]["nodePort"]=30100+int(n.name[4:]) + with open(POD_Name+'.yaml', "a") as f: + yaml.dump(service_data, f) + + output = subprocess.check_output('kubectl apply -f '+POD_Name+'.yaml', shell=True).decode() + print(output) + #output = subprocess.check_output('kubectl apply -f '+'yolo_service_plz.yaml', shell=True).decode() + #print(output) + #time.sleep(5) + #self.Yolo_Ready_Check() + return self.Available_Node + + def node_flops_check(self): + directory_path = "/home/share/nfs/flops" + + # 디렉토리가 존재하지 않는 경우에만 생성 + if not os.path.exists(directory_path): + os.mkdir(directory_path) + print("디렉토리가 생성되었습니다.") + nodeName=[] + output = subprocess.check_output('kubectl top nodes', shell=True).decode() + result = output.split() + if len(result) <= 5: + print("There are no nodes in Kubernetes") + else: + for i in range(5, len(result), 5): + if '' in result[i:i + 5]: continue + if result[i] == 'master': continue + nodeName.append(result[i]) + flops=[] + file_path="/home/share/nfs/flops/node_flops.txt" + if os.path.exists(file_path): + file = open(file_path, 'r') + while True: + line = file.readline() + if not line: break + node_name,_=line.split(" ") + flops.append(node_name) + no_flops_node = [] + for name in nodeName: + if name not in flops: + no_flops_node.append(name) + print("All" +str(len(nodeName))+"nodes, only" +str(len(flops))+ " nodes have flops information") + check_list=self.flops_measurement(no_flops_node) + else: + file = open(file_path, 'w') + file.close() + print("There are no nodes available. Run after a while.") + check_list=self.flops_measurement(nodeName) + self.check_end(check_list) + return + + def flops_measurement(self,node): + for n in node: + gpu=False + output = subprocess.check_output("kubectl get nodes -l 'nvidia.com/gpu'", shell=True).decode() + info=output.split() + for i in range(5,len(info)): + if info[i]==n: + gpu=True + if gpu: + with open("flops_gpu.yaml", "r") as f: + data = yaml.safe_load(f) + else: + with open("flops.yaml", "r") as f: + data = yaml.safe_load(f) + POD_Name=n + nodenum=n[-1] + print(nodenum) + if gpu: + data['spec']['template']['spec']['containers'][0]['command'] = ["python3", "./yolo/flops.py", "--weights","./yolo/yolov5s.pt", "--path","/home/share/nfs/flops", "--nodeNum",nodenum, "--width", "512", "--height","512"] + else: + data['spec']['template']['spec']['containers'][0]['command'] = ["python3", "./flops.py", "--weights", "yolov5s.pt", "--path", "/home/share/nfs/flops", "--nodeNum", nodenum, "--width", "512", "--height", "512"] + data["metadata"]["name"]=POD_Name + data["spec"]["selector"]["matchLabels"]["app"], \ + data["spec"]["template"]["metadata"]["labels"]["app"] = ["updated-label"] * 2 + data['spec']['template']['spec']['containers'][0]['name'] = POD_Name + data['spec']['template']['spec']['nodeSelector']['key'] = n + with open(POD_Name+'flops.yaml', "w") as f: + yaml.dump(data, f) + output = subprocess.check_output('kubectl apply -f '+POD_Name+'flops.yaml', shell=True).decode() + print(output) + return node + + def check_end(self,check_list): + file_path="/home/share/nfs/flops/node_flops.txt" + + while True: + skip=True + node=[] + if os.path.exists(file_path): + file = open(file_path, 'r') + while True: + line = file.readline() + if not line: break + node_name,_=line.split(" ") + node.append(node_name) + for n in check_list: + if n not in node : + print(n+" is not exsist wait plz!") + time.sleep(3) + skip=False + if skip: + break + for n in check_list: + output = subprocess.check_output('kubectl delete -f '+n+'flops.yaml', shell=True).decode() + print(output) + + # 만든 POD이 Running 상태인지 점검 + """sumary_line + def Yolo_Ready_Check(self): + output = subprocess.check_output('kubectl get pods', shell=True).decode() + POD_List=output.split() + # 5개 단위로 자름 + POD_List = [POD_List[i:i+5] for i in range(0, len(POD_List), 5)] + POD_Situation={} + for POD in POD_List: + name=POD[0].split("-") + POD_Situation[name[0]]=POD[2] + print(POD_Situation) + for n in self.Available_Node: + if n.name+"yolo" not in POD_Situation: + print("POD is not establish") + elif POD_Situation[n.name+"yolo"]!="Running": + print(n.name+" Yolo POD have problem") + return self.Available_Node + """ diff --git a/distibuteModule/node.py b/distibuteModule/node.py new file mode 100644 index 0000000..5e08cad --- /dev/null +++ b/distibuteModule/node.py @@ -0,0 +1,35 @@ +import subprocess + + +cal_total = lambda a,b: a//b*100 +mb_to_gb = lambda mb: mb / 1024 + +class Node: + def __init__(self,node_info): + self.name=node_info[0] + self.cpu_usage=int(node_info[1][:-1])/1000 # m단위 제거 + self.total_cpu=round(cal_total(self.cpu_usage*1000,int(node_info[2][:-1]))/1000) + self.memory_usage=int(node_info[3][:-2])/1000 # Mi단위 제거 + self.total_memory=round(mb_to_gb(cal_total(self.memory_usage*1000,int(node_info[4][:-1])))) + self.isgpu=self.check_gpu() + self.FLOPS=0 + self.ip =node_info[-1] + + def check_gpu(self): + output = subprocess.check_output("kubectl get nodes -l 'nvidia.com/gpu'", shell=True).decode() + info=output.split() + for i in range(5,len(info)): + if info[i]==self.name: + return True + return False + + #For Debug + def print_node(self): + print("name : "+str(self.name)) + print("cpu info "+str(self.cpu_usage)+"/"+str(self.total_cpu) + " Core") + print("memory info "+str(self.memory_usage)+"/"+str(self.total_memory)+" GB") + print("CPU : "+str(self.isgpu)) + print("FLOPS : "+str(self.FLOPS)) + print("ip : "+str(self.ip)) + print("-----------------------") + \ No newline at end of file diff --git a/distibuteModule/node2.py b/distibuteModule/node2.py new file mode 100644 index 0000000..4c61d4a --- /dev/null +++ b/distibuteModule/node2.py @@ -0,0 +1,30 @@ +import subprocess + + +cal_total = lambda a,b: a//b*100 +mb_to_gb = lambda mb: mb / 1024 + +class Node: + def __init__(self, name, cpu_usage,total_cpu,memory_usage,total_memory,isgpu,FLOPS,ip): + self.name=name + self.cpu_usage=cpu_usage + self.total_cpu=total_cpu + self.memory_usage=memory_usage + self.total_memory=total_memory + self.isgpu=isgpu + self.FLOPS=FLOPS + self.ip = ip; + self.TT=0 + self.PT=0 + self.PCT=0 + + #For Debug + def print_node(self): + print("name : "+str(self.name)) + print("cpu info "+str(self.cpu_usage)+"/"+str(self.total_cpu) + " Core") + print("memory info "+str(self.memory_usage)+"/"+str(self.total_memory)+" GB") + print("CPU : "+str(self.isgpu)) + print("FLOPS : "+str(self.FLOPS)) + print("IP : " + str(self.ip)) + print("-----------------------") + \ No newline at end of file diff --git a/distibuteModule/real_test.py b/distibuteModule/real_test.py new file mode 100644 index 0000000..af670eb --- /dev/null +++ b/distibuteModule/real_test.py @@ -0,0 +1,84 @@ +from flask import Flask, jsonify +import threading +import json +import cv2 +import node2 +import requests +import subprocess +from FrameDrop import framedrop + +app = Flask(__name__) + +class VideoInformation: + def __init__(self, frameWidth, frameHeight, frameCount, fps, videoLength): + self.frameWidth = frameWidth + self.frameHeight = frameHeight + self.frameCount = frameCount + self.fps = fps + self.videoLength = videoLength + +def do_something(AvailNode_List): + FrameDrop = framedrop("C:/Users/Public/black.mp4", "C:/Users/Public/bk.mp4", 'CORREL',85,AvailNode_List) + drop_frame = FrameDrop.drop() + +@app.route("/") +def hello(): + return "Hello main_falsk" + +@app.route("/videoinformation") +def home(): + + cap = cv2.VideoCapture("C:/Users/Public/black.mp4", apiPreference=None) + if not cap.isOpened(): # check File exists + print("Video is not opened!") + fps = cap.get(cv2.CAP_PROP_FPS) # FPS (초당 프레임 수) + width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # 영상 가로 크기 + height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # 영상 세로 크기 + total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) # 총 프레임 수 + + # response = requests.get('http://192.168.0.11:5002/start') + + #str_data = response.content.decode('utf-8') + + # JSON 문자열을 파싱하여 딕셔너리로 변환 + #json_data = json.loads(str_data) + json_data2=[ + { + "name": "master", + "cpu_usage": 0.669, + "total_cpu": 4, + "memory_usage": 1.734, + "total_memory": 8, + "isgpu": False, + "FLOPS": 0 + }, + { + "name": "node1", + "cpu_usage": 0.185, + "total_cpu": 5, + "memory_usage": 0.718, + "total_memory": 2, + "isgpu": False, + "FLOPS": 0 + }, + { + "name": "node2", + "cpu_usage": 0.294, + "total_cpu": 4, + "memory_usage": 0.879, + "total_memory": 4, + "isgpu": True, + "FLOPS": 0 + } +] + AvailNode_List = [node2.Node(ob['name'], ob['cpu_usage'],ob['total_cpu'],ob['memory_usage'],ob['total_memory'],ob['isgpu'],ob['FLOPS']) for ob in json_data2] + print(AvailNode_List) + for n in AvailNode_List: + n.print_node() + threading.Thread(target=do_something,args=(AvailNode_List,)).start() + video_information = VideoInformation(width, height, total_frames, fps, total_frames/fps) + return jsonify(vars(video_information)) + + +if __name__ == "__main__": + app.run(host = '0.0.0.0',port = 5001, debug = True) \ No newline at end of file diff --git a/distibuteModule/spring_web_server.yaml b/distibuteModule/spring_web_server.yaml new file mode 100644 index 0000000..2ae16e5 --- /dev/null +++ b/distibuteModule/spring_web_server.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 # for versions before 1.8.0 use apps/v1beta1 +kind: Deployment +metadata: + name: spring-server-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: spring-server + template: + metadata: + labels: + app: spring-server + spec: + volumes: + - name: my-pv-storage + persistentVolumeClaim: + claimName: my-pv-claim + containers: + - name: spring-web-server + image: dlskawo0409/spring_test4 + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: "/home/share/nfs" + name: my-pv-storage + imagePullPolicy: Never + + nodeSelector: + key: master +--- +apiVersion: v1 +kind: Service +metadata: + name: spring-web-server-service +spec: + ports: + - name: "8080" + port: 8081 + targetPort: 8080 + nodePort: 30600 + selector: + app: spring-server + type: NodePort + diff --git a/distibuteModule/test.py b/distibuteModule/test.py new file mode 100644 index 0000000..5a99684 --- /dev/null +++ b/distibuteModule/test.py @@ -0,0 +1,3 @@ +import json +params = json.loads("persent=0.4838709677419355") +print(params) \ No newline at end of file diff --git a/distibuteModule/write_yolo_yaml.py b/distibuteModule/write_yolo_yaml.py new file mode 100644 index 0000000..e69de29 diff --git a/distibuteModule/yoloFlask_2.py b/distibuteModule/yoloFlask_2.py new file mode 100644 index 0000000..5fdd360 --- /dev/null +++ b/distibuteModule/yoloFlask_2.py @@ -0,0 +1,297 @@ +from flask import Flask, jsonify +import json +import requests + +import argparse +import os +import sys +from pathlib import Path +import time +import torch +import threading + +from models.common import DetectMultiBackend # ▒~U~D▒~Z~T +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages #▒~U~D▒~Z~T +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_requirements, cv2, non_max_suppression, print_args, scale_coords,increment_path) +from utils.plots import Annotator, colors +from utils.torch_utils import select_device, smart_inference_mode + +""" + +Python .\detect_cpu_preLoad.py --filePath D:\Capstone\yolo5_distribute\node1 --save_dir D:\Capstone\yolo5_distribute\result +python3 ./yolo_flask.py --filePath /home/share/nfs/node1 --save_dir /home/share/nfs/result + +""" + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative +app = Flask(__name__) +global OPT +OPT="" +global IsLast +IsLast = False + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s.pt', help='model path(s)') + parser.add_argument('--filePath', type=str, default=ROOT / 'node1/input', help='file/dir/URL/glob') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--save_dir', type=str, default=ROOT / 'node1/result', help='file/dir/URL/glob, 0 for webcam') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--address', default="", help='response address') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def main(opt): + print("start main", flush=True); + check_requirements(exclude=('tensorboard', 'thop')) + print("pass requierements", flush=True); + opt_save = opt + run(**vars(opt_save)) + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model.pt path(s) + filePath=ROOT / 'node1/', # file/dir/URL/glob, 0 for webcam + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + save_dir = "./node1/result", + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show resultss + save_crop=False, # save cropped prediction boxes + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidences + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + address = "" +): + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + save_img = True + model.warmup(imgsz=(1, 3, *imgsz)) # warmup + mode = 'image' + nodepath = os.path.normpath(filePath) # ▒~G▒~H째 ▒~E▒▒~S~\ ▒~]▒▒~@ + nodeNumStr=nodepath[-1] + + check_dir(save_dir) + + frameCount = 0; + leftFrame = 0; + + print("detection start " ,flush=True) + + while(1): + for (root, dirs, files) in os.walk(filePath): + leftFrame = len(files) + for file in files: + source = os.path.join(filePath, file) #/home/share/nfs/node1/1.png + print(source,flush=True) + save_path = os.path.join(save_dir, file) #/home/share/nfs/result\1.png + # Dataloader + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt) + bs = 1 # batch_size + vid_path, vid_writer = [None] * bs, [None] * bs + + # Run inference + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + p = Path(p) # to Path + # save_path = "/home/share/nfs/node1/cpuResult.mp4" # im.jpg + s += '%gx%g ' % im.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + imc = im0.copy() if save_crop else im0 # for save_crop + annotator = Annotator(im0, line_width=line_thickness, example=str(names)) + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(im.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Write results + for *xyxy, conf, cls in reversed(det): + if save_img or save_crop or view_img: # Add bbox to image + c = int(cls) # integer class + label = None if hide_labels else ( + names[c] if hide_conf else f'{names[c]} {conf:.2f}') + annotator.box_label(xyxy, label, color=colors(c, True)) + + # Save results (image with detections) + if save_img: + if dataset.mode == 'image': + print(save_path,flush=True) + cv2.imwrite(save_path, im0) + else: # 'video' or 'stream' + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str( + Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, + (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info( + f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + #resultTime = time.time() - start + removeFile(source) + frameCount = frameCount + 1 + + if frameCount % 10 == 0: + thread = threading.Thread(target=send_Progress, args=(str(nodeNumStr), frameCount, leftFrame)) + thread.start() + + + if IsLast and not newFileCheck(filePath): # end all + thread = threading.Thread(target=send_Progress, args=(str(nodeNumStr), frameCount, 0)) + thread.start() + url = 'http://192.168.0.11:30500/yolo/end/' + str(nodeNumStr) + response = requests.get(url, timeout=10) + print("finish 2",flush=True) + print(response,flush=True) + exit() + else: + waitNewFile(filePath) + + + + +@app.route("/end") +def last(): + global IsLast + IsLast = True + data = {'message': 'yolo going end'} + print(IsLast,flush=True) + return jsonify(data) + +@app.route("/start") +def start(): + global OPT + data = {'message': 'yolo Start'} + thread = threading.Thread(target=main, args=(OPT,)) + thread.start() + return jsonify(data) + +def removeFile(file): + if os.path.isfile(file): + os.remove(file) + +def newFileCheck(filePath): + temp = os.listdir(filePath) + count = len(temp) + if count == 0: + return False + else: + return True + +def waitNewFile(filePath): + temp = os.listdir(filePath) + count = len(temp) + while count == 0: + time.sleep(4) + temp = os.listdir(filePath) + count = len(temp) + +def check_dir(save_dir): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + print(f"Created 'result' directory at {save_dir}") + else: + print(f"'result' directory already exists at {save_dir}") + +def send_Progress(nodeNumStr, frameCount, leftFrame): + persent = frameCount/(frameCount+leftFrame) + data = {'persent' : persent} + url = 'http://192.168.0.11:30500/yolo/progress/' + str(nodeNumStr) + print(url, flush=True) + response = requests.post(url, data=data) + print(response, flush=True) + return response + +def getNodeNum(opt): + filePath = opt.filePath + nodepath = os.path.normpath(filePath) # ▒~G▒~H째 ▒~E▒▒~S~\ ▒~]▒▒~@ + nodeNumStr = nodepath[-1] + return str(nodeNumStr) + +@app.route("/read") +def ready(): + data = {'message': 'I am ready'} + print("I am ready", flush=True) + return jsonify(data) + + + +if __name__ == "__main__": + OPT = parse_opt() + nodeNumStr = getNodeNum(OPT) + app.run(host = '0.0.0.0',port = 5001, debug = True) + url = 'http://192.168.0.11:30500/' + str(nodeNumStr) +"/ready" + response = requests.get(url) + print(response, flush=True) diff --git a/testCode/node.py b/testCode/node.py new file mode 100644 index 0000000..a1ba642 --- /dev/null +++ b/testCode/node.py @@ -0,0 +1,31 @@ +import subprocess + + +cal_total = lambda a,b: a//b*100 +mb_to_gb = lambda mb: mb / 1024 + +class Node: + def __init__(self,name,cpu_usage,total_cpu,memory_usage,total_memory,isgpu,FLOPS): + self.name=name + self.cpu_usage=cpu_usage + self.total_cpu=total_cpu + self.memory_usage=memory_usage + self.total_memory=total_memory + self.isgpu=isgpu + self.FLOPS=FLOPS + self.TT=0 + self.PT=0 + self.PCT=0 + + #For Debug + def print_node(self): + print("name : "+str(self.name)) + print("cpu info "+str(self.cpu_usage)+"/"+str(self.total_cpu) + " Core") + print("memory info "+str(self.memory_usage)+"/"+str(self.total_memory)+" GB") + print("CPU : "+str(self.isgpu)) + print("FLOPS : "+str(self.FLOPS)) + print("-----------------------") + + def set_value(self,PT,TT): + self.TT=TT + self.PT=PT \ No newline at end of file diff --git a/testCode/test_drop.py b/testCode/test_drop.py new file mode 100644 index 0000000..92a5eda --- /dev/null +++ b/testCode/test_drop.py @@ -0,0 +1,175 @@ +import cv2 +import numpy as np +import time +import node +import random + +class framedrop(): + def __init__(self, filePath, savePath, flag, threshold,NodeList): + self.filePath = filePath + self.threshold=threshold + self.cap = cv2.VideoCapture(filePath, apiPreference=None) + self.w = round(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.h = round(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.fourcc = cv2.VideoWriter_fourcc('M','P','4','V') + self.fps = self.cap.get(cv2.CAP_PROP_FPS) + self.frame = round(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) + self.videoWriter = cv2.VideoWriter(savePath, self.fourcc, self.fps,(self.w,self.h)) + #Method + self.method = {'CORREL' :cv2.HISTCMP_CORREL, # cv2.HISTCMP_CORREL: 상관관계 (1: 완전 일치, -1: 완전 불일치, 0: 무관계) + 'CHISQR':cv2.HISTCMP_CHISQR, # cv2.HISTCMP_CHISQR: 카이제곱 (0: 완전 일치, 무한대: 완전 불일치) + 'INTERSECT':cv2.HISTCMP_INTERSECT, # cv2.HISTCMP_INTERSECT: 교차 (1: 완전 일치, 0: 완전 불일치 - 1로 정규화한 경우) + 'BHATTACHARYYA':cv2.HISTCMP_BHATTACHARYYA, # cv2.HISTCMP_BHATTACHARYYA 값이 작을수록 유사한 것으로 판단 + 'HELLINGER':cv2.HISTCMP_HELLINGER, + 'CHISQR_ALT':cv2.HISTCMP_CHISQR_ALT, + 'KL_DIV':cv2.HISTCMP_KL_DIV} + self.flag = flag + self.NodeList=NodeList + self.start_time=0 + self.current_time=0 + + def drop(self): + if not self.cap.isOpened(): # check File exists + print("Video is not opened!") + return + drop_frame=0 + beforeRet, beforeFrame = self.cap.read() + print('Frame Count : '+str(self.frame)) + print() + print("Frame 1") + selsect_node=self.NodeSelector() + + self.send_to_node(beforeFrame,selsect_node) + + self.start_time=time.time() + beforeFrameHist = self.preprocess(beforeFrame) + count = 1 + + + while(count != self.frame): + nowRet, nowFrame = self.cap.read() + nowFrameHist = self.preprocess(nowFrame) + + result = self.calculateSimilarity(beforeFrameHist, nowFrameHist, self.flag) + print('Frame '+str(count+1)) + selsect_node=self.NodeSelector() + + if result >= self.threshold: + drop_frame+=1 + self.send_to_node() + else: + # nowFrame을 전송 + self.send_to_node(nowFrame,selsect_node) + beforeFrameHist = nowFrameHist + beforeFrame=nowFrame + count +=1 + self.start_time=time.time() + #similarity = "#"+str(count) + " -> " + str(round(result,6)) + "\n" + self.cap.release() + print() + for node in self.NodeList: + print(node.name+"의 남은 작업시간 : "+'{:.4f}'.format(node.PCT)+"sec") + return drop_frame + + def NodeSelector(self): + self.current_time=time.time() + elapsed_time=self.current_time-self.start_time + min_endtime=100000000 + for node in self.NodeList: + + node.PCT=node.PCT-elapsed_time if node.PCT-elapsed_time > 0 else 0 + #print(node.name+"의 남은 작업시간 : "+'{:.4f}'.format(node.PCT)+"sec"+ ", 예상 작업 완료시간 : "+'{:.4f}'.format(abs(node.PCT-node.TT)+node.PT)+"sec") + if min_endtime > abs(node.PCT-node.TT)+node.PT : + min_endtime=abs(node.PCT-node.TT)+node.PT + select_node=node + return select_node + + def send_to_node(self,Frame,node): + node.PCT+=node.PT + # 파일 저장 + print(node.name +"선택" ) + print() + + + + + + + + + def preprocess(self, frame): + # ---① 각 이미지를 HSV로 변환 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # ---② H,S 채널에 대한 히스토그램 계산 + hist = cv2.calcHist([hsv], [0, 1], None, [180, 256], [0, 180, 0, 256]) + # ---③ 0~1로 정규화 + result =cv2.normalize(hist, hist, 0, 1, cv2.NORM_MINMAX) + return result + + + def calculateSimilarity(self, beforeFrameHist, nowFrameHist, flag ): + + ret = cv2.compareHist(beforeFrameHist, nowFrameHist, self.method[flag]) + + if flag == cv2.HISTCMP_INTERSECT: + ret = ret/np.sum(beforeFrameHist) + + return ret + + +json_data2=[ + { + "name": "master", + "cpu_usage": 0.669, + "total_cpu": 4, + "memory_usage": 1.734, + "total_memory": 8, + "isgpu": False, + "FLOPS": 0 + }, + { + "name": "node1", + "cpu_usage": 0.185, + "total_cpu": 5, + "memory_usage": 0.718, + "total_memory": 2, + "isgpu": False, + "FLOPS": 0 + }, + { + "name": "node2", + "cpu_usage": 0.294, + "total_cpu": 4, + "memory_usage": 0.879, + "total_memory": 4, + "isgpu": True, + "FLOPS": 0 + }, + { + "name": "node3", + "cpu_usage": 0.294, + "total_cpu": 4, + "memory_usage": 0.879, + "total_memory": 4, + "isgpu": True, + "FLOPS": 0 + }, + { + "name": "node4", + "cpu_usage": 0.294, + "total_cpu": 4, + "memory_usage": 0.879, + "total_memory": 4, + "isgpu": True, + "FLOPS": 0 + } +] + + +AvailNode_List = [node.Node(ob['name'], ob['cpu_usage'],ob['total_cpu'],ob['memory_usage'],ob['total_memory'],ob['isgpu'],ob['FLOPS']) for ob in json_data2] +FrameDrop = framedrop("C:/Users/Public/test1.mp4", "C:/Users/Public/blacaaaaaaak.mp4", 'CORREL',85,AvailNode_List) +print("1 Frame당 처리 시간") +for node in AvailNode_List: + node.set_value(random.randint(1, 10),0) + print(node.name+ " "+ str(node.PT)) +drop_frame = FrameDrop.drop() \ No newline at end of file diff --git a/webpage/distribute/build.gradle b/webpage/distribute/build.gradle index 028c550..6370731 100644 --- a/webpage/distribute/build.gradle +++ b/webpage/distribute/build.gradle @@ -18,6 +18,8 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-test' annotationProcessor('org.projectlombok:lombok') testAnnotationProcessor('org.projectlombok:lombok') + + } tasks.named('test') { diff --git a/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/Mode.class b/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/Mode.class index 13ba8f5..77d3bb3 100644 Binary files a/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/Mode.class and b/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/Mode.class differ diff --git a/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/videoInformation.class b/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/videoInformation.class index 8a3847f..aaff91f 100644 Binary files a/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/videoInformation.class and b/webpage/distribute/build/classes/java/main/com/example/distribute/Configuration/videoInformation.class differ diff --git a/webpage/distribute/build/classes/java/main/com/example/distribute/DistributeApplication.class b/webpage/distribute/build/classes/java/main/com/example/distribute/DistributeApplication.class index 1bd67ae..70ed74e 100644 Binary files a/webpage/distribute/build/classes/java/main/com/example/distribute/DistributeApplication.class and b/webpage/distribute/build/classes/java/main/com/example/distribute/DistributeApplication.class differ diff --git a/webpage/distribute/build/classes/java/main/com/example/distribute/uploadController/UploadController.class b/webpage/distribute/build/classes/java/main/com/example/distribute/uploadController/UploadController.class index 2bd0d3b..14f6f6f 100644 Binary files a/webpage/distribute/build/classes/java/main/com/example/distribute/uploadController/UploadController.class and b/webpage/distribute/build/classes/java/main/com/example/distribute/uploadController/UploadController.class differ diff --git a/webpage/distribute/build/libs/distribute-webPage (2).zip b/webpage/distribute/build/libs/distribute-webPage (2).zip index a998df8..979023f 100644 Binary files a/webpage/distribute/build/libs/distribute-webPage (2).zip and b/webpage/distribute/build/libs/distribute-webPage (2).zip differ diff --git a/webpage/distribute/build/libs/distribute-webPage (3).zip b/webpage/distribute/build/libs/distribute-webPage (3).zip index ea35457..ea4e671 100644 Binary files a/webpage/distribute/build/libs/distribute-webPage (3).zip and b/webpage/distribute/build/libs/distribute-webPage (3).zip differ diff --git a/webpage/distribute/build/libs/distribute-webPage (4).zip b/webpage/distribute/build/libs/distribute-webPage (4).zip deleted file mode 100644 index e69c5c6..0000000 Binary files a/webpage/distribute/build/libs/distribute-webPage (4).zip and /dev/null differ diff --git a/webpage/distribute/build/libs/distribute-webPage.jar b/webpage/distribute/build/libs/distribute-webPage.jar index e73ddef..6926750 100644 Binary files a/webpage/distribute/build/libs/distribute-webPage.jar and b/webpage/distribute/build/libs/distribute-webPage.jar differ diff --git a/webpage/distribute/build/resources/main/templates/file.html b/webpage/distribute/build/resources/main/templates/file.html index 3e52018..ab85bfb 100644 --- a/webpage/distribute/build/resources/main/templates/file.html +++ b/webpage/distribute/build/resources/main/templates/file.html @@ -2,10 +2,12 @@ + + Yolo File Upload + + - - Yolo File Upload - +

YOLOv5 on K3S

@@ -16,9 +18,16 @@

Maximum file size is 500MB

- + +

Threshold 를 정해주세요

+ +
+
+ +
+

diff --git a/webpage/distribute/build/resources/main/templates/mode.html b/webpage/distribute/build/resources/main/templates/mode.html index 8eb4074..5a3e44a 100644 --- a/webpage/distribute/build/resources/main/templates/mode.html +++ b/webpage/distribute/build/resources/main/templates/mode.html @@ -1,32 +1,47 @@ - + + + + + + Yolo File Upload + - - Yolo File Upload - +

YOLOv5 on K3S

-

Upload File!

+

Choose Node Mode!

- -

Choose Single or Multi node

+
+ +
+
+

- Single - Multi +
+
+ +
+ Single +
+
+
+ +
+ Multi +
+
+ +
+

- - diff --git a/webpage/distribute/build/resources/main/templates/videoinformation.html b/webpage/distribute/build/resources/main/templates/videoinformation.html index 4199fa1..79363ad 100644 --- a/webpage/distribute/build/resources/main/templates/videoinformation.html +++ b/webpage/distribute/build/resources/main/templates/videoinformation.html @@ -1,27 +1,47 @@ + + + + Yolo File Upload

YOLOv5 on K3S

-

Video Information

-

-

-

-

-

-
-

-

YOLOv5 is Ready

-

Click the Start button

-
- -
-

Press the button and wait for it to finish

+ +
+
+
+ +

Video Information

+

+

+

+

+

+
+

+ +
+
+
+ +
+
+
+
+

nodePrgress

+ +
+
+ +
+
+ \ No newline at end of file diff --git a/webpage/distribute/build/tmp/compileJava/compileTransaction/stash-dir/UploadController.class.uniqueId0 b/webpage/distribute/build/tmp/compileJava/compileTransaction/stash-dir/UploadController.class.uniqueId0 index ad487a2..050a418 100644 Binary files a/webpage/distribute/build/tmp/compileJava/compileTransaction/stash-dir/UploadController.class.uniqueId0 and b/webpage/distribute/build/tmp/compileJava/compileTransaction/stash-dir/UploadController.class.uniqueId0 differ diff --git a/webpage/distribute/build/tmp/compileJava/previous-compilation-data.bin b/webpage/distribute/build/tmp/compileJava/previous-compilation-data.bin index 3da1a08..79f5899 100644 Binary files a/webpage/distribute/build/tmp/compileJava/previous-compilation-data.bin and b/webpage/distribute/build/tmp/compileJava/previous-compilation-data.bin differ diff --git a/webpage/distribute/src/main/java/com/example/distribute/Configuration/ConversionService.java b/webpage/distribute/src/main/java/com/example/distribute/Configuration/ConversionService.java new file mode 100644 index 0000000..5abdda2 --- /dev/null +++ b/webpage/distribute/src/main/java/com/example/distribute/Configuration/ConversionService.java @@ -0,0 +1,19 @@ +package com.example.distribute.Configuration; +import org.springframework.stereotype.Service; + +@Service +public class ConversionService { + private String conversionStatus; + + + public String getConversionStatus() { + return conversionStatus; + } + + public void setConversionStatus(String conversionStatus) { + this.conversionStatus = conversionStatus; + } + + // 변환 상태를 확인하는 로직 등의 비즈니스 로직을 구현할 수 있습니다. +} + diff --git a/webpage/distribute/src/main/java/com/example/distribute/Configuration/Mode.java b/webpage/distribute/src/main/java/com/example/distribute/Configuration/Mode.java index 476718e..670f197 100644 --- a/webpage/distribute/src/main/java/com/example/distribute/Configuration/Mode.java +++ b/webpage/distribute/src/main/java/com/example/distribute/Configuration/Mode.java @@ -1,8 +1,5 @@ package com.example.distribute.Configuration; - - - public class Mode { private String mode=""; diff --git a/webpage/distribute/src/main/java/com/example/distribute/Configuration/Progress.java b/webpage/distribute/src/main/java/com/example/distribute/Configuration/Progress.java new file mode 100644 index 0000000..1d18faf --- /dev/null +++ b/webpage/distribute/src/main/java/com/example/distribute/Configuration/Progress.java @@ -0,0 +1,15 @@ +package com.example.distribute.Configuration; + +public class Progress { + + private int persent; + + public Progress(){} + public int getPersent(){ + return this.persent; + } + + public void setPersent(int persent){ + this.persent = persent; + } +} diff --git a/webpage/distribute/src/main/java/com/example/distribute/Configuration/downloadInformation.java b/webpage/distribute/src/main/java/com/example/distribute/Configuration/downloadInformation.java new file mode 100644 index 0000000..8dfbca4 --- /dev/null +++ b/webpage/distribute/src/main/java/com/example/distribute/Configuration/downloadInformation.java @@ -0,0 +1,6 @@ +package com.example.distribute.Configuration; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record downloadInformation(int waitTime, String downloadPath, int dropCount) { } diff --git a/webpage/distribute/src/main/java/com/example/distribute/Configuration/videoInformation.java b/webpage/distribute/src/main/java/com/example/distribute/Configuration/videoInformation.java index 01d1047..1717ceb 100644 --- a/webpage/distribute/src/main/java/com/example/distribute/Configuration/videoInformation.java +++ b/webpage/distribute/src/main/java/com/example/distribute/Configuration/videoInformation.java @@ -3,4 +3,4 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) -public record videoInformation(int frameWeight, int frameHeight, int frameCount, double fps, double videoLength) { } \ No newline at end of file +public record videoInformation(int frameWidth, int frameHeight, int frameCount, double fps, double videoLength, int nodeCount, int threshold) { } //nodeCount \ No newline at end of file diff --git a/webpage/distribute/src/main/java/com/example/distribute/DistributeApplication.java b/webpage/distribute/src/main/java/com/example/distribute/DistributeApplication.java index a5e9fbf..5074981 100644 --- a/webpage/distribute/src/main/java/com/example/distribute/DistributeApplication.java +++ b/webpage/distribute/src/main/java/com/example/distribute/DistributeApplication.java @@ -14,19 +14,17 @@ public class DistributeApplication { public static void main(String[] args) { SpringApplication.run(DistributeApplication.class, args); } - @Bean - CommandLineRunner init(StorageService storageService) { - return (args) -> { - storageService.deleteAll(); - storageService.init(); - }; - } +// @Bean +// CommandLineRunner init(StorageService storageService) { +// return (args) -> { +// storageService.deleteAll(); +// storageService.init(); +// }; +// } @Bean public RestTemplate restTemplate(RestTemplateBuilder builder) { return builder.build(); } - - } diff --git a/webpage/distribute/src/main/java/com/example/distribute/restAPI/RESTConversionController.java b/webpage/distribute/src/main/java/com/example/distribute/restAPI/RESTConversionController.java new file mode 100644 index 0000000..4317c3b --- /dev/null +++ b/webpage/distribute/src/main/java/com/example/distribute/restAPI/RESTConversionController.java @@ -0,0 +1,47 @@ +package com.example.distribute.restAPI; + +import com.example.distribute.Configuration.ConversionService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class RESTConversionController { + + private final ConversionService conversionService; + + @Autowired + public RESTConversionController(ConversionService conversionService) { + this.conversionService = conversionService; + } + + @GetMapping("/distribution/download") + public ResponseEntity checkConversionStatus() { + conversionService.setConversionStatus("completed"); + String conversionStatus = conversionService.getConversionStatus(); + System.out.println("/distribution/download"); + // conversionStatus 사용 + + if (conversionStatus!=null) { + return ResponseEntity.ok(conversionStatus); + } else { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Conversion status not found"); + } + } + + @GetMapping("/status/conversion") + public ResponseEntity getConversionStatus(){ + String conversionStatus = conversionService.getConversionStatus(); +// System.out.println(conversionStatus); + if (conversionStatus != null) { + return ResponseEntity.ok(conversionStatus); + } else { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Conversion status not found"); + } + } + + +} diff --git a/webpage/distribute/src/main/java/com/example/distribute/uploadController/UploadController.java b/webpage/distribute/src/main/java/com/example/distribute/uploadController/UploadController.java index 54a2f2b..3919e33 100644 --- a/webpage/distribute/src/main/java/com/example/distribute/uploadController/UploadController.java +++ b/webpage/distribute/src/main/java/com/example/distribute/uploadController/UploadController.java @@ -1,9 +1,14 @@ package com.example.distribute.uploadController; -import com.example.distribute.Configuration.Mode; -import com.example.distribute.Configuration.videoInformation; -import com.example.distribute.storage.*; +import com.example.distribute.Configuration.*; +import org.springframework.core.io.Resource; +import com.example.distribute.storage.StorageException; +import com.example.distribute.storage.StorageFileNotFoundException; +import com.example.distribute.storage.StorageService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; @@ -12,22 +17,41 @@ import org.springframework.web.multipart.MultipartFile; import org.springframework.web.servlet.mvc.support.RedirectAttributes; import org.springframework.web.servlet.view.RedirectView; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.MalformedURLException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.StringTokenizer; + +import org.springframework.core.io.UrlResource; + + @Controller public class UploadController { private final Mode mode; private final StorageService storageService; - private String destinationFile; - private String videoDistributeUrl = "http://localhost:5001"; + private String destinationFile = ""; + private String videoDistributeUrl = "http://192.168.0.11:30500";// "http://192.168.0.11:30500" ;//"http://localhost:5001"; // 5001로 변환해야함 + private String downloadPath; + private final ConversionService conversionService; + private Progress[] progressList; + private float threshold; + @Autowired - public UploadController(Mode mode, StorageService storageService) { + public UploadController(Mode mode, StorageService storageService, ConversionService conversionService, Progress[] PropresetList) { this.mode = mode; this.storageService = storageService; + this.conversionService = conversionService; + this.progressList = PropresetList; } @RequestMapping(value = "/", method = RequestMethod.GET) public String upload(Model model){ + conversionService.setConversionStatus("ready"); model.addAttribute("mode", new Mode()); return "mode"; @@ -46,7 +70,9 @@ public String uploadFile(Model model){ } @PostMapping("/mode/file") - public String handleFileUpload(@RequestParam("file") MultipartFile file) { + public String handleFileUpload(@RequestParam("file") MultipartFile file ,@RequestParam("threshold") String threshold ) { + this.threshold = Float.parseFloat(threshold); + storageService.store(file); destinationFile = storageService.store(file); return "redirect:/mode/file/videoinformation"; @@ -56,24 +82,100 @@ public String handleFileUpload(@RequestParam("file") MultipartFile file) { public String showVideoInformation(Model model, RestTemplate restTemplate)throws Exception{ model.addAttribute("mode",mode.getMode()); model.addAttribute("filePath", destinationFile); + System.out.println(destinationFile); + System.out.println(this.threshold); + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(videoDistributeUrl+"/videoinformation").queryParam("mode", mode.getMode()).queryParam("filepath", destinationFile).queryParam("threshold",this.threshold); + String url = builder.toUriString(); + System.out.print(url); + videoInformation videoinformation = restTemplate.getForObject(url, com.example.distribute.Configuration.videoInformation.class); + + model.addAttribute("frameWidth", videoinformation.frameWidth()); + model.addAttribute("frameHeight", videoinformation.frameHeight()); + model.addAttribute("frameCount", videoinformation.frameCount()); + model.addAttribute("fps", videoinformation.fps()); + model.addAttribute("videoLength", videoinformation.videoLength()); + model.addAttribute("nodeCount",videoinformation.nodeCount()); + + initProgressList(videoinformation.nodeCount()); +// initProgressList(3); +// model.addAttribute("nodeCount",3); - videoInformation videoinformation = restTemplate.getForObject(videoDistributeUrl+"/videoinformation", com.example.distribute.Configuration.videoInformation.class); - model.addAttribute("frameWeight",videoinformation.frameWeight()); - model.addAttribute("frameHeight",videoinformation.frameHeight()); - model.addAttribute("frameCount",videoinformation.frameCount()); - model.addAttribute("fps",videoinformation.fps()); - model.addAttribute("videoLength",videoinformation.videoLength()); + return "videoinformation"; + } + @GetMapping("mode/file/download") + public String showDownLoadPage(Model model , RestTemplate restTemplate){ + System.out.println("show DownLoadPage"); + downloadInformation downloadinformation = restTemplate.getForObject(videoDistributeUrl+"/download", com.example.distribute.Configuration.downloadInformation.class); + System.out.println("1"); + model.addAttribute("mode", mode.getMode()); + System.out.println("2"); + model.addAttribute("waitTime", downloadinformation.waitTime()); + model.addAttribute("dropCount",downloadinformation.dropCount()); + System.out.println(downloadinformation.waitTime()); + System.out.println(downloadinformation.dropCount()); + downloadPath = downloadinformation.downloadPath(); + System.out.println(downloadPath); - return "videoinformation"; +// downloadPath = "D:\\Capstone\\yolo5_light_file_size\\data\\video\\traffic-mini.mp4"; + + return "download"; + } + + @PostMapping("/api/videoUrl") + public ResponseEntity videoUrl() throws Exception{ + HashMap videoUrl = new HashMap<>(); + videoUrl.put("url", "D:\\Capstone\\yolo5_light_file_size\\data\\video\\traffic-mini.mp4"); + + return new ResponseEntity>(videoUrl, HttpStatus.OK); } + @GetMapping("mode/file/download/final") + public ResponseEntity downloadFile() { + // downloadPath = "D:\\Capstone\\yolo5_light_file_size\\data\\video\\traffic-mini.mp4"; // 다운로드할 파일의 경로 + + try { + Path file = Paths.get(downloadPath); + Resource resource = new UrlResource(file.toUri()); + + if (resource.exists()) { + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } + } catch (MalformedURLException e) { + e.printStackTrace(); + } + + // 다운로드할 파일이 존재하지 않을 경우에 대한 처리 + return ResponseEntity.notFound().build(); + } + @GetMapping("/mode/file/progress/{progressBarId}") + @ResponseBody + public String updateProgress(@PathVariable("progressBarId") String progressBarId) { + int id = Integer.parseInt(progressBarId); + String strPersent = Integer.toString(progressList[id].getPersent()); + return "{\"progressBarId\": \"" + progressBarId + "\", \"progress\": " + strPersent + "}"; + } + + @PostMapping("/progress/{progressBarId}/{persent}") + public ResponseEntity setProgresss(@PathVariable("progressBarId") int progressBarId, @PathVariable("persent") int persent){ + progressList[progressBarId].setPersent(persent); + return ResponseEntity.ok(200); + } + + @GetMapping("/nodecount") + public ResponseEntity returnNodeCount(){ + return ResponseEntity.ok(progressList.length -1); + } @ExceptionHandler(StorageFileNotFoundException.class) public ResponseEntity handleStorageFileNotFound(StorageFileNotFoundException exc) { return ResponseEntity.notFound().build(); } + @ExceptionHandler(StorageException.class) public RedirectView handleStorageException(StorageException e ,RedirectAttributes rttr) { String redirect = "/mode/file"; @@ -91,5 +193,15 @@ public String isModeNUll(String nowPage){ return nowPage; } + protected void initProgressList(int nodeCount){ + this.progressList = new Progress[nodeCount+1]; + for(int i = 1; i<=nodeCount; i++){ + progressList[i] = new Progress(); + progressList[i].setPersent(0); + } + for(int i =1; i{ + return response.json(); + }) + .then((response) =>{ + videoUrl = response; + let videoObject = document.getElementById("videoPlayer"); + let video = '