From 03e7e069d86c74d552da549e7d03bc11662c7462 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Tue, 16 Jun 2015 14:49:12 -0500 Subject: [PATCH 01/10] Added file upload viw with url pattern added to url.py --- queue/urls.py | 4 +++- queue/views.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/queue/urls.py b/queue/urls.py index 4f0a02d..a9516ea 100644 --- a/queue/urls.py +++ b/queue/urls.py @@ -1,7 +1,7 @@ __author__ = 'mstacy' from django.conf.urls import patterns, url from django.contrib import admin -from queue.views import Run, Queue, UserTasks, UserResult +from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView from rest_framework.urlpatterns import format_suffix_patterns # q = QueueTask() @@ -17,6 +17,8 @@ #url(r'run/', include(tasks_url)), #url(r'run/$',Run.as_view(),name='run-main'), url(r'run/(?P[-\w .]+)/$', Run.as_view(), name='run-main'), + #url(r'file-upload/$', FileUploadView.as_view(), name='file-upload'), + url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), url(r'task/(?P[-\w]+)/$', UserResult.as_view(), name='queue-task-result'), url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'), url(r'^$', Queue.as_view(), name="queue-main"), diff --git a/queue/views.py b/queue/views.py index d0431eb..0a6e80c 100644 --- a/queue/views.py +++ b/queue/views.py @@ -2,16 +2,19 @@ from rest_framework.settings import api_settings from rest_framework.response import Response from rest_framework.reverse import reverse -from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly +from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly,AllowAny from rest_framework.views import APIView from ccelery.q import QueueTask, list_tasks, task_docstring from models import Run_model from rest_framework.renderers import JSONRenderer, JSONPRenderer from renderer import QueueRunBrowsableAPIRenderer -from rest_framework.parsers import JSONParser +from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser from util import trim from rest_framework.authtoken.models import Token #task = list_tasks()['available_tasks'] +#from rest_framework.viewsets import ModelViewSet +#from serializer import FileUploadSerializer +import os q = QueueTask() @@ -40,7 +43,7 @@ def get(self, request,format=None): class Run(APIView): permission_classes = (DjangoModelPermissionsOrAnonReadOnly,) model = Run_model - parser_classes = (JSONParser,) + parser_classes = (JSONParser,MultiPartParser,FormParser) renderer_classes = (QueueRunBrowsableAPIRenderer, JSONRenderer, JSONPRenderer,) def __init__(self,q=q, *args, **kwargs): @@ -85,6 +88,41 @@ def post(self, request,task_name=None,format=None): result['result_url']=reverse('queue-task-result', kwargs={'task_id':result['task_id']}, request=request) return Response(result) +class FileUploadView(APIView): + + permission_classes =(AllowAny,) + #parser_classes = (MultiPartParser, FormParser,) + parser_classes = (FileUploadParser,) + renderer_classes = (JSONRenderer,) + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request,filename, format=None): + resultDir = os.path.join("/data/tmp", self.get_username(request)) + try: + os.makedirs(resultDir) + except: + pass + for key,value in request.FILES.items(): + print key,value + my_file = request.FILES['file'] #.get('filename',None) + + with open("%s/%s" % (resultDir,filename), 'wb+') as temp_file: + for chunk in my_file.chunks(): + temp_file.write(chunk) + return Response({"file":"%s/%s" % (resultDir,filename)}) + +#class FileUploadViewSet(ModelViewSet): +# permission_classes =(IsAuthenticatedOrReadOnly,) +# queryset = FileUpload.objects.all() +# serializer_class = FileUploadSerializer +# parser_classes = (MultiPartParser, FormParser,) + +# def perform_create(self, serializer): +# serializer.save(owner=self.request.user, datafile=self.request.data.get('datafile')) class UserResult(APIView): permission_classes = (DjangoModelPermissionsOrAnonReadOnly,) From 33f26a407b8bad9cd85b354b9eb3a82149878389 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Thu, 18 Jun 2015 12:58:53 -0500 Subject: [PATCH 02/10] Upload hell --- queue/urls.py | 4 ++-- queue/views.py | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/queue/urls.py b/queue/urls.py index a9516ea..d2f56f1 100644 --- a/queue/urls.py +++ b/queue/urls.py @@ -17,8 +17,8 @@ #url(r'run/', include(tasks_url)), #url(r'run/$',Run.as_view(),name='run-main'), url(r'run/(?P[-\w .]+)/$', Run.as_view(), name='run-main'), - #url(r'file-upload/$', FileUploadView.as_view(), name='file-upload'), - url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), + url(r'file-upload/', FileUploadView.as_view(), name='file-upload'), + #url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), url(r'task/(?P[-\w]+)/$', UserResult.as_view(), name='queue-task-result'), url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'), url(r'^$', Queue.as_view(), name="queue-main"), diff --git a/queue/views.py b/queue/views.py index 0a6e80c..3898a9b 100644 --- a/queue/views.py +++ b/queue/views.py @@ -91,8 +91,8 @@ def post(self, request,task_name=None,format=None): class FileUploadView(APIView): permission_classes =(AllowAny,) - #parser_classes = (MultiPartParser, FormParser,) - parser_classes = (FileUploadParser,) + parser_classes = (MultiPartParser, FormParser,) + #parser_classes = (FileUploadParser,) renderer_classes = (JSONRenderer,) def get_username(self, request): username = "guest" @@ -100,20 +100,27 @@ def get_username(self, request): username = request.user.username return username - def post(self, request,filename, format=None): + def post(self, request, format=None): resultDir = os.path.join("/data/tmp", self.get_username(request)) try: os.makedirs(resultDir) except: pass - for key,value in request.FILES.items(): - print key,value - my_file = request.FILES['file'] #.get('filename',None) + result={} + for key,value in request.FILES.iteritems(): + filename= value.name + local_file = "%s/%s" % (resultDir,filename) + with open(local_file, 'wb+') as temp_file: + for chunk in value.chunks(): + temp_file.write(value) + result[key]=local_file + return Response(result) + #my_file = request.FILES['file'] #.get('filename',None) - with open("%s/%s" % (resultDir,filename), 'wb+') as temp_file: - for chunk in my_file.chunks(): - temp_file.write(chunk) - return Response({"file":"%s/%s" % (resultDir,filename)}) + #with open("%s/%s" % (resultDir,filename), 'wb+') as temp_file: + # for chunk in my_file.chunks(): + # temp_file.write(chunk) + #return Response({"file":"%s/%s" % (resultDir,filename)}) #class FileUploadViewSet(ModelViewSet): # permission_classes =(IsAuthenticatedOrReadOnly,) From 7c10576e780dc8f0d61d4c716fe354acb2f3937b Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Wed, 24 Jun 2015 16:28:44 -0500 Subject: [PATCH 03/10] Updated file upload handle function --- queue/views.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/queue/views.py b/queue/views.py index 3898a9b..cc37f64 100644 --- a/queue/views.py +++ b/queue/views.py @@ -11,6 +11,8 @@ from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser from util import trim from rest_framework.authtoken.models import Token +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator #task = list_tasks()['available_tasks'] #from rest_framework.viewsets import ModelViewSet #from serializer import FileUploadSerializer @@ -94,6 +96,11 @@ class FileUploadView(APIView): parser_classes = (MultiPartParser, FormParser,) #parser_classes = (FileUploadParser,) renderer_classes = (JSONRenderer,) + + @method_decorator(csrf_exempt) + def dispatch(self, request, *args, **kwargs): + return super(FileUploadView, self).dispatch(request, *args, **kwargs) + def get_username(self, request): username = "guest" if request.user.is_authenticated(): @@ -110,11 +117,20 @@ def post(self, request, format=None): for key,value in request.FILES.iteritems(): filename= value.name local_file = "%s/%s" % (resultDir,filename) - with open(local_file, 'wb+') as temp_file: - for chunk in value.chunks(): - temp_file.write(value) + self.handle_file_upload(request.FILES[key],local_file) + #with open(local_file, 'wb+') as temp_file: + # for chunk in value.chunks(): + # temp_file.write(value) result[key]=local_file return Response(result) + def handle_file_upload(self,f,filename): + if f.multiple_chunks(): + with open(filename, 'wb+') as temp_file: + for chunk in f.chunks(): + temp_file.write(chunk) + else: + with open(filename, 'wb+') as temp_file: + temp_file.write(f.read()) #my_file = request.FILES['file'] #.get('filename',None) #with open("%s/%s" % (resultDir,filename), 'wb+') as temp_file: From b2d4327cbc16d64921394e3a1f506818b8e20a48 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Wed, 24 Jun 2015 16:32:53 -0500 Subject: [PATCH 04/10] Cleaned up code fragments --- queue/views.py | 67 +------------------------------------------------- 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/queue/views.py b/queue/views.py index cc37f64..4a53956 100644 --- a/queue/views.py +++ b/queue/views.py @@ -118,9 +118,6 @@ def post(self, request, format=None): filename= value.name local_file = "%s/%s" % (resultDir,filename) self.handle_file_upload(request.FILES[key],local_file) - #with open(local_file, 'wb+') as temp_file: - # for chunk in value.chunks(): - # temp_file.write(value) result[key]=local_file return Response(result) def handle_file_upload(self,f,filename): @@ -131,21 +128,6 @@ def handle_file_upload(self,f,filename): else: with open(filename, 'wb+') as temp_file: temp_file.write(f.read()) - #my_file = request.FILES['file'] #.get('filename',None) - - #with open("%s/%s" % (resultDir,filename), 'wb+') as temp_file: - # for chunk in my_file.chunks(): - # temp_file.write(chunk) - #return Response({"file":"%s/%s" % (resultDir,filename)}) - -#class FileUploadViewSet(ModelViewSet): -# permission_classes =(IsAuthenticatedOrReadOnly,) -# queryset = FileUpload.objects.all() -# serializer_class = FileUploadSerializer -# parser_classes = (MultiPartParser, FormParser,) - -# def perform_create(self, serializer): -# serializer.save(owner=self.request.user, datafile=self.request.data.get('datafile')) class UserResult(APIView): permission_classes = (DjangoModelPermissionsOrAnonReadOnly,) @@ -198,51 +180,4 @@ def get(self, request,format=None, **kwargs): task_name = request.GET.get('taskname', None) username = self.get_username(request) data = self.q.history(username, task_name=task_name, page=page, limit=limit,request=request) - #for item in data: - - - # data['next']= reverse('queue-user-tasks',kwargs={'page':page+1,},request=request) - # if isinstance(data,ObjectId): - # return str(obj) - #print data - return Response(data) #,indent=4,sort_keys=True)) - - -""" - # print task_docstring(task_name) - html ="
Task Name: %s
Task Docstring:%sExample Curl:
%s
" - baseHelp =""curl --data-ascii '{ "function": "%s", - "queue": "%s", - "args": [], - "kwargs": {} - }' %s -H Content-Type:application/json"" % (task_name,"celery",reverse("%s-run" % (task_name), request=request)) - html = html % (task_name,task_docstring(task_name),baseHelp) - #html = html % (task_name,markdown.markdown(trim(task_docstring(task_name))),baseHelp) - return Response(html) #, template_name='rest_framework/queue_run_api.html') - - - -from json import JSONEncoder -from bson.objectid import ObjectId - -class MongoEncoder(JSONEncoder): - def default(self, obj, **kwargs): - if isinstance(obj, ObjectId): - return str(obj) - else: - return JSONEncoder.default(obj, **kwargs) - - - from rest_framework.renderers import JSONRenderer, JSONPRenderer -from rest_framework import renderers -import json - -class PlainTextRenderer(renderers.BaseRenderer): - media_type = 'text/plain' - format = 'txt' - - def render(self, data, media_type=None, renderer_context=None): - return data.encode(self.charset) - - -""" + return Response(data) From 3297af7904b3d30db721156cffea7f4f035f5a3e Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Sun, 28 Jun 2015 17:20:05 -0500 Subject: [PATCH 05/10] Saved user with password set --- api/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/views.py b/api/views.py index c0bbeaf..35e8532 100644 --- a/api/views.py +++ b/api/views.py @@ -61,6 +61,7 @@ def post(self,request,format=None): if password: user.set_password(password) data = {"password":"Successfully Updated"} + user.save() return Response(data) auth_tok = request.DATA.get('auth-token', None) if str(auth_tok).lower()=="update": From cb15fe2b7bf1163803ae0721c4f9baf22559c0ff Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Wed, 8 Jul 2015 13:52:21 -0500 Subject: [PATCH 06/10] update subtask return status --- queue/ccelery/q.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/queue/ccelery/q.py b/queue/ccelery/q.py index 85af568..fe8a046 100644 --- a/queue/ccelery/q.py +++ b/queue/ccelery/q.py @@ -218,24 +218,43 @@ def result(self, task_id=None, redirect=True): def task(self, task_id=None): """Return task log and task results""" doc = self.db[self.database][self.collection].find_one({'task_id': task_id}, {'_id': False}) + col = self.db[self.database][self.tomb_collection] if doc: - col = self.db[self.database][self.tomb_collection] result = col.find_one({'_id': task_id}, {'_id': False}) if result: - if 'traceback' in result: - result['traceback'] = pickle.loads(result['traceback']) - if 'children' in result: - result['children'] = pickle.loads(result['children']) - if 'result' in result: - result['result'] = pickle.loads(result['result']) - if isinstance(result['result'], Exception): - result['result'] = "ERROR: %s" % (result['result'].message) + result=self.unpickle_result(result) + #if 'traceback' in result: + # result['traceback'] = pickle.loads(result['traceback']) + #if 'children' in result: + # result['children'] = pickle.loads(result['children']) + #if 'result' in result: + # result['result'] = pickle.loads(result['result']) + # if isinstance(result['result'], Exception): + # result['result'] = "ERROR: %s" % (result['result'].message) else: result = self.status(task_id=task_id) doc['result'] = result return doc else: - raise Exception("Task was not found. Not a valid task_id") + result = col.find_one({'_id': task_id}, {'_id': False}) + if result: + result=self.unpickle_result(result) + return result + else: + result = self.status(task_id=task_id) + return result + #raise Exception("Task was not found. Not a valid task_id") + + def unpickle_result(self,result): + if 'traceback' in result: + result['traceback'] = pickle.loads(result['traceback']) + if 'children' in result: + result['children'] = pickle.loads(result['children']) + if 'result' in result: + result['result'] = pickle.loads(result['result']) + if isinstance(result['result'], Exception): + result['result'] = "ERROR: %s" % (result['result'].message) + return result def reset(self, user=None): From 8cadf6ccd358dea3b226f5d4dbbf7aa9585c9d90 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Sun, 20 Sep 2015 12:14:49 -0500 Subject: [PATCH 07/10] Added mgmic app to restful api - added to api/url.py --- api/urls.py | 1 + mgmic/__init__.py | 0 mgmic/admin.py | 3 ++ mgmic/models.py | 3 ++ mgmic/tests.py | 3 ++ mgmic/urls.py | 27 +++++++++++++++ mgmic/views.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 125 insertions(+) create mode 100644 mgmic/__init__.py create mode 100644 mgmic/admin.py create mode 100644 mgmic/models.py create mode 100644 mgmic/tests.py create mode 100644 mgmic/urls.py create mode 100644 mgmic/views.py diff --git a/api/urls.py b/api/urls.py index fad112b..daabaa3 100644 --- a/api/urls.py +++ b/api/urls.py @@ -22,6 +22,7 @@ url(r'^queue/', include('queue.urls')), url(r'^data_store/',include('data_store.urls')), url(r'^catalog/', include('catalog.urls')), + url(r'^mgmic/', include('mgmic.urls')), # Admin Urls url(r'^admin/', include(admin.site.urls)), # Main Project View - Customize depending on what Apps are enabled diff --git a/mgmic/__init__.py b/mgmic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mgmic/admin.py b/mgmic/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/mgmic/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/mgmic/models.py b/mgmic/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/mgmic/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/mgmic/tests.py b/mgmic/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/mgmic/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/mgmic/urls.py b/mgmic/urls.py new file mode 100644 index 0000000..8d04f7d --- /dev/null +++ b/mgmic/urls.py @@ -0,0 +1,27 @@ +__author__ = 'mstacy' +from django.conf.urls import patterns, url +from django.contrib import admin +#from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView +from rest_framework.urlpatterns import format_suffix_patterns +from mgmic.views import CheckMapFile +# q = QueueTask() +#tasks_url = [] + +#for task in q.list()['available_tasks']: +# tasks_url.append(url(r'%s/$' % task, Run.as_view(), name="%s-run" % (task))) +# #tasks_url.append(url(r'%s/.(api|json|jsonp|xml|yaml)$' % task, Run.as_view(), name="%s-run-format" % (task))) + +admin.autodiscover() + +urlpatterns = patterns('', + #url(r'run/', include(tasks_url)), + #url(r'run/$',Run.as_view(),name='run-main'), + url(r'check-mapfile/', CheckMapFile.as_view(), name='cmap-main'), + #url(r'file-upload/', FileUploadView.as_view(), name='file-upload'), + #url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), + #url(r'task/(?P[-\w]+)/$', UserResult.as_view(), name='queue-task-result'), + #url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'), + #url(r'^$', Queue.as_view(), name="queue-main"), +) + +urlpatterns = format_suffix_patterns(urlpatterns, allowed=['api', 'json', 'jsonp', 'xml', 'yaml']) diff --git a/mgmic/views.py b/mgmic/views.py new file mode 100644 index 0000000..1d3018d --- /dev/null +++ b/mgmic/views.py @@ -0,0 +1,88 @@ +from django.shortcuts import render +#from rest_framework.settings import api_settings +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly,AllowAny +from rest_framework.views import APIView +#from ccelery.q import QueueTask, list_tasks, task_docstring +#from models import Run_model +from rest_framework.renderers import JSONRenderer, JSONPRenderer +#from renderer import QueueRunBrowsableAPIRenderer +from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser +#from util import trim +#from rest_framework.authtoken.models import Token +#from django.views.decorators.csrf import csrf_exempt +#from django.utils.decorators import method_decorator +#task = list_tasks()['available_tasks'] +#from rest_framework.viewsets import ModelViewSet +#from serializer import FileUploadSerializer +import os +from dockertask import docker_task +rom subprocess import Popen, PIPE +class CheckMapFile(APIView): + permission_classes = ( IsAuthenticatedOrReadOnly,) + + def __init__(self, *args, **kwargs): + self.docker_cmd = "validate_mapping_file.py -m %s -o %s " + self.docker_opts = "-v /data:/data" + self.docker_container = "qiime_env" + + super(CheckMapFile, self).__init__(*args, **kwargs) + + def post(self, request,filename=None,format=None): + if filename: + p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + resultDir = os.path.join("/data/tmp", output.split()[0]) + os.makedirs(resultDir) + else: + raise('Please provide map file filename') + docker_cmd = self.docker_cmd % (filename,resultDir) + result = docker_task(docker_name=self.docker_container,docker_opts=self.docker_opts,docker_command=docker_cmd,id=output.split()[0]) + return Response({ + 'MapFile': filename, + 'resultDir': resultDir + }) + +# Create your views here. +""" +class FileUploadView(APIView): + + permission_classes =(AllowAny,) + parser_classes = (MultiPartParser, FormParser,) + #parser_classes = (FileUploadParser,) + renderer_classes = (JSONRenderer,) + + @method_decorator(csrf_exempt) + def dispatch(self, request, *args, **kwargs): + return super(FileUploadView, self).dispatch(request, *args, **kwargs) + + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request, format=None): + resultDir = os.path.join("/data/tmp", self.get_username(request)) + try: + os.makedirs(resultDir) + except: + pass + result={} + for key,value in request.FILES.iteritems(): + filename= value.name + local_file = "%s/%s" % (resultDir,filename) + self.handle_file_upload(request.FILES[key],local_file) + result[key]=local_file + return Response(result) + def handle_file_upload(self,f,filename): + if f.multiple_chunks(): + with open(filename, 'wb+') as temp_file: + for chunk in f.chunks(): + temp_file.write(chunk) + else: + with open(filename, 'wb+') as temp_file: + temp_file.write(f.read()) + +""" From 66e010dc57125398e1da1d29e713cfe9b3b539fd Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Sun, 20 Sep 2015 12:19:49 -0500 Subject: [PATCH 08/10] added mgmic to installed apps in settings --- api/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/settings.py b/api/settings.py index db24d82..6021e63 100644 --- a/api/settings.py +++ b/api/settings.py @@ -84,7 +84,8 @@ 'rest_framework.authtoken', 'queue', 'catalog', - 'data_store' + 'data_store', + 'mgmic' ) From 5102e787a16c24f43925808a66c01eda853f21c3 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Wed, 23 Sep 2015 11:33:58 -0500 Subject: [PATCH 09/10] add check map file web service --- api/config.py | 2 ++ api/settings.py | 2 ++ mgmic/urls.py | 2 +- mgmic/views.py | 71 ++++++++++++++++++++++++++++++++++++++---------- requirements.txt | 2 ++ 5 files changed, 64 insertions(+), 15 deletions(-) diff --git a/api/config.py b/api/config.py index 780c567..82a71a8 100644 --- a/api/config.py +++ b/api/config.py @@ -53,3 +53,5 @@ #*********** Data Store ************************************************ DATA_STORE_EXCLUDE = ['admin','local','cybercom_auth','system.users','cybercom_queue','df'] DATA_STORE_MONGO_URI = 'mongodb://localhost:27017/' +#*********** DOCKER_HOST_DATA_DIRECTORY ******************** +DOCKER_HOST_DATA_DIRECTORY = "/data" diff --git a/api/settings.py b/api/settings.py index 6021e63..273ee20 100644 --- a/api/settings.py +++ b/api/settings.py @@ -22,6 +22,8 @@ "api.processor.title", ) +#DOCKER_HOST_DATA_DIRECTORY when host machine runs need to know if different then /data +DOCKER_HOST_DATA_DIRECTORY= config.DOCKER_HOST_DATA_DIRECTORY # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = config.SECRET_KEY diff --git a/mgmic/urls.py b/mgmic/urls.py index 8d04f7d..fb38fde 100644 --- a/mgmic/urls.py +++ b/mgmic/urls.py @@ -3,7 +3,7 @@ from django.contrib import admin #from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView from rest_framework.urlpatterns import format_suffix_patterns -from mgmic.views import CheckMapFile +from views import CheckMapFile # q = QueueTask() #tasks_url = [] diff --git a/mgmic/views.py b/mgmic/views.py index 1d3018d..197dc56 100644 --- a/mgmic/views.py +++ b/mgmic/views.py @@ -16,34 +16,77 @@ #task = list_tasks()['available_tasks'] #from rest_framework.viewsets import ModelViewSet #from serializer import FileUploadSerializer -import os +import os,glob,shutil,httplib from dockertask import docker_task -rom subprocess import Popen, PIPE -class CheckMapFile(APIView): - permission_classes = ( IsAuthenticatedOrReadOnly,) +from subprocess import Popen, PIPE, call +from django.conf import settings +from urlparse import urlparse +import requests +class CheckMapFile(APIView): + permission_classes = ( AllowAny,) + renderer_classes = (JSONRenderer,) def __init__(self, *args, **kwargs): self.docker_cmd = "validate_mapping_file.py -m %s -o %s " - self.docker_opts = "-v /data:/data" + temp = getattr(settings,"DOCKER_HOST_DATA_DIRECTORY", "/data") + self.docker_opts = "-v %s:/data" % (temp) self.docker_container = "qiime_env" - super(CheckMapFile, self).__init__(*args, **kwargs) - def post(self, request,filename=None,format=None): + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request,format=None): + filename =request.POST.get('filename',None) if filename: - p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, err = p.communicate() - resultDir = os.path.join("/data/tmp", output.split()[0]) - os.makedirs(resultDir) + if os.path.isfile(filename): + p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0]) + #pass + else: + if not check_url_exist(filename): + raise Exception("Please Check URL or Local File Path(local files must be in /data directory) %s" % filename) + resultDir = os.path.join("/data/tmp", self.get_username(request)) + map_read = os.path.join(resultDir,filename.split('/')[-1]) + logfile= open(resultDir + "/logfile.txt","w") + call(['wget','-O',map_read,filename],stdout=logfile,stderr=logfile) + logfile.close() + p = Popen(['md5sum', map_read], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0]) + filename=map_read + try: + os.makedirs(resultDir) + except: + shutil.rmtree(resultDir) + os.makedirs(resultDir) else: - raise('Please provide map file filename') + raise Exception('Please provide map file filename') docker_cmd = self.docker_cmd % (filename,resultDir) result = docker_task(docker_name=self.docker_container,docker_opts=self.docker_opts,docker_command=docker_cmd,id=output.split()[0]) + logfile= glob.glob(resultDir + '/*.log') + status = "" + with open(logfile[0]) as f: + data = f.read() + if "No errors" in data: + status="SUCCESS" + else: + status="FAILURE" return Response({ - 'MapFile': filename, - 'resultDir': resultDir + 'status':status, + 'log': data, + 'local-file': filename }) +def check_url_exist(url): + p = urlparse(url) + c = httplib.HTTPConnection(p.netloc) + c.request("HEAD", p.path) + return c.getresponse().status < 400 # Create your views here. """ class FileUploadView(APIView): diff --git a/requirements.txt b/requirements.txt index e47dcfe..f7a95f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,5 @@ six==1.7.2 mongoengine==0.8.7 django-rest-framework-mongoengine==2.0.2 wsgiref==0.1.2 +https://github.com/ouinformatics/dockertask/zipball/master +requests From 84063db126611ca87a6cfd96e7f7dd8c7f7c2a82 Mon Sep 17 00:00:00 2001 From: Mark Stacy Date: Wed, 23 Sep 2015 18:31:51 -0500 Subject: [PATCH 10/10] Set default of docker data directory --- api/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/settings.py b/api/settings.py index 273ee20..7635efa 100644 --- a/api/settings.py +++ b/api/settings.py @@ -23,7 +23,11 @@ ) #DOCKER_HOST_DATA_DIRECTORY when host machine runs need to know if different then /data -DOCKER_HOST_DATA_DIRECTORY= config.DOCKER_HOST_DATA_DIRECTORY +try + DOCKER_HOST_DATA_DIRECTORY= config.DOCKER_HOST_DATA_DIRECTORY +except: + DOCKER_HOST_DATA_DIRECTORY="/data" + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = config.SECRET_KEY