diff --git a/api/config.py b/api/config.py index 780c567..82a71a8 100644 --- a/api/config.py +++ b/api/config.py @@ -53,3 +53,5 @@ #*********** Data Store ************************************************ DATA_STORE_EXCLUDE = ['admin','local','cybercom_auth','system.users','cybercom_queue','df'] DATA_STORE_MONGO_URI = 'mongodb://localhost:27017/' +#*********** DOCKER_HOST_DATA_DIRECTORY ******************** +DOCKER_HOST_DATA_DIRECTORY = "/data" diff --git a/api/settings.py b/api/settings.py index db24d82..7635efa 100644 --- a/api/settings.py +++ b/api/settings.py @@ -22,6 +22,12 @@ "api.processor.title", ) +#DOCKER_HOST_DATA_DIRECTORY when host machine runs need to know if different then /data +try + DOCKER_HOST_DATA_DIRECTORY= config.DOCKER_HOST_DATA_DIRECTORY +except: + DOCKER_HOST_DATA_DIRECTORY="/data" + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = config.SECRET_KEY @@ -84,7 +90,8 @@ 'rest_framework.authtoken', 'queue', 'catalog', - 'data_store' + 'data_store', + 'mgmic' ) diff --git a/api/urls.py b/api/urls.py index fad112b..daabaa3 100644 --- a/api/urls.py +++ b/api/urls.py @@ -22,6 +22,7 @@ url(r'^queue/', include('queue.urls')), url(r'^data_store/',include('data_store.urls')), url(r'^catalog/', include('catalog.urls')), + url(r'^mgmic/', include('mgmic.urls')), # Admin Urls url(r'^admin/', include(admin.site.urls)), # Main Project View - Customize depending on what Apps are enabled diff --git a/api/views.py b/api/views.py index c0bbeaf..35e8532 100644 --- a/api/views.py +++ b/api/views.py @@ -61,6 +61,7 @@ def post(self,request,format=None): if password: user.set_password(password) data = {"password":"Successfully Updated"} + user.save() return Response(data) auth_tok = request.DATA.get('auth-token', None) if str(auth_tok).lower()=="update": diff --git a/mgmic/__init__.py b/mgmic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mgmic/admin.py b/mgmic/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/mgmic/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/mgmic/models.py b/mgmic/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/mgmic/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/mgmic/tests.py b/mgmic/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/mgmic/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/mgmic/urls.py b/mgmic/urls.py new file mode 100644 index 0000000..fb38fde --- /dev/null +++ b/mgmic/urls.py @@ -0,0 +1,27 @@ +__author__ = 'mstacy' +from django.conf.urls import patterns, url +from django.contrib import admin +#from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView +from rest_framework.urlpatterns import format_suffix_patterns +from views import CheckMapFile +# q = QueueTask() +#tasks_url = [] + +#for task in q.list()['available_tasks']: +# tasks_url.append(url(r'%s/$' % task, Run.as_view(), name="%s-run" % (task))) +# #tasks_url.append(url(r'%s/.(api|json|jsonp|xml|yaml)$' % task, Run.as_view(), name="%s-run-format" % (task))) + +admin.autodiscover() + +urlpatterns = patterns('', + #url(r'run/', include(tasks_url)), + #url(r'run/$',Run.as_view(),name='run-main'), + url(r'check-mapfile/', CheckMapFile.as_view(), name='cmap-main'), + #url(r'file-upload/', FileUploadView.as_view(), name='file-upload'), + #url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), + #url(r'task/(?P[-\w]+)/$', UserResult.as_view(), name='queue-task-result'), + #url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'), + #url(r'^$', Queue.as_view(), name="queue-main"), +) + +urlpatterns = format_suffix_patterns(urlpatterns, allowed=['api', 'json', 'jsonp', 'xml', 'yaml']) diff --git a/mgmic/views.py b/mgmic/views.py new file mode 100644 index 0000000..197dc56 --- /dev/null +++ b/mgmic/views.py @@ -0,0 +1,131 @@ +from django.shortcuts import render +#from rest_framework.settings import api_settings +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly,AllowAny +from rest_framework.views import APIView +#from ccelery.q import QueueTask, list_tasks, task_docstring +#from models import Run_model +from rest_framework.renderers import JSONRenderer, JSONPRenderer +#from renderer import QueueRunBrowsableAPIRenderer +from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser +#from util import trim +#from rest_framework.authtoken.models import Token +#from django.views.decorators.csrf import csrf_exempt +#from django.utils.decorators import method_decorator +#task = list_tasks()['available_tasks'] +#from rest_framework.viewsets import ModelViewSet +#from serializer import FileUploadSerializer +import os,glob,shutil,httplib +from dockertask import docker_task +from subprocess import Popen, PIPE, call +from django.conf import settings +from urlparse import urlparse +import requests + +class CheckMapFile(APIView): + permission_classes = ( AllowAny,) + renderer_classes = (JSONRenderer,) + def __init__(self, *args, **kwargs): + self.docker_cmd = "validate_mapping_file.py -m %s -o %s " + temp = getattr(settings,"DOCKER_HOST_DATA_DIRECTORY", "/data") + self.docker_opts = "-v %s:/data" % (temp) + self.docker_container = "qiime_env" + super(CheckMapFile, self).__init__(*args, **kwargs) + + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request,format=None): + filename =request.POST.get('filename',None) + if filename: + if os.path.isfile(filename): + p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0]) + #pass + else: + if not check_url_exist(filename): + raise Exception("Please Check URL or Local File Path(local files must be in /data directory) %s" % filename) + resultDir = os.path.join("/data/tmp", self.get_username(request)) + map_read = os.path.join(resultDir,filename.split('/')[-1]) + logfile= open(resultDir + "/logfile.txt","w") + call(['wget','-O',map_read,filename],stdout=logfile,stderr=logfile) + logfile.close() + p = Popen(['md5sum', map_read], stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0]) + filename=map_read + try: + os.makedirs(resultDir) + except: + shutil.rmtree(resultDir) + os.makedirs(resultDir) + else: + raise Exception('Please provide map file filename') + docker_cmd = self.docker_cmd % (filename,resultDir) + result = docker_task(docker_name=self.docker_container,docker_opts=self.docker_opts,docker_command=docker_cmd,id=output.split()[0]) + logfile= glob.glob(resultDir + '/*.log') + status = "" + with open(logfile[0]) as f: + data = f.read() + if "No errors" in data: + status="SUCCESS" + else: + status="FAILURE" + return Response({ + 'status':status, + 'log': data, + 'local-file': filename + }) + +def check_url_exist(url): + p = urlparse(url) + c = httplib.HTTPConnection(p.netloc) + c.request("HEAD", p.path) + return c.getresponse().status < 400 +# Create your views here. +""" +class FileUploadView(APIView): + + permission_classes =(AllowAny,) + parser_classes = (MultiPartParser, FormParser,) + #parser_classes = (FileUploadParser,) + renderer_classes = (JSONRenderer,) + + @method_decorator(csrf_exempt) + def dispatch(self, request, *args, **kwargs): + return super(FileUploadView, self).dispatch(request, *args, **kwargs) + + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request, format=None): + resultDir = os.path.join("/data/tmp", self.get_username(request)) + try: + os.makedirs(resultDir) + except: + pass + result={} + for key,value in request.FILES.iteritems(): + filename= value.name + local_file = "%s/%s" % (resultDir,filename) + self.handle_file_upload(request.FILES[key],local_file) + result[key]=local_file + return Response(result) + def handle_file_upload(self,f,filename): + if f.multiple_chunks(): + with open(filename, 'wb+') as temp_file: + for chunk in f.chunks(): + temp_file.write(chunk) + else: + with open(filename, 'wb+') as temp_file: + temp_file.write(f.read()) + +""" diff --git a/queue/ccelery/q.py b/queue/ccelery/q.py index 85af568..fe8a046 100644 --- a/queue/ccelery/q.py +++ b/queue/ccelery/q.py @@ -218,24 +218,43 @@ def result(self, task_id=None, redirect=True): def task(self, task_id=None): """Return task log and task results""" doc = self.db[self.database][self.collection].find_one({'task_id': task_id}, {'_id': False}) + col = self.db[self.database][self.tomb_collection] if doc: - col = self.db[self.database][self.tomb_collection] result = col.find_one({'_id': task_id}, {'_id': False}) if result: - if 'traceback' in result: - result['traceback'] = pickle.loads(result['traceback']) - if 'children' in result: - result['children'] = pickle.loads(result['children']) - if 'result' in result: - result['result'] = pickle.loads(result['result']) - if isinstance(result['result'], Exception): - result['result'] = "ERROR: %s" % (result['result'].message) + result=self.unpickle_result(result) + #if 'traceback' in result: + # result['traceback'] = pickle.loads(result['traceback']) + #if 'children' in result: + # result['children'] = pickle.loads(result['children']) + #if 'result' in result: + # result['result'] = pickle.loads(result['result']) + # if isinstance(result['result'], Exception): + # result['result'] = "ERROR: %s" % (result['result'].message) else: result = self.status(task_id=task_id) doc['result'] = result return doc else: - raise Exception("Task was not found. Not a valid task_id") + result = col.find_one({'_id': task_id}, {'_id': False}) + if result: + result=self.unpickle_result(result) + return result + else: + result = self.status(task_id=task_id) + return result + #raise Exception("Task was not found. Not a valid task_id") + + def unpickle_result(self,result): + if 'traceback' in result: + result['traceback'] = pickle.loads(result['traceback']) + if 'children' in result: + result['children'] = pickle.loads(result['children']) + if 'result' in result: + result['result'] = pickle.loads(result['result']) + if isinstance(result['result'], Exception): + result['result'] = "ERROR: %s" % (result['result'].message) + return result def reset(self, user=None): diff --git a/queue/urls.py b/queue/urls.py index 4f0a02d..d2f56f1 100644 --- a/queue/urls.py +++ b/queue/urls.py @@ -1,7 +1,7 @@ __author__ = 'mstacy' from django.conf.urls import patterns, url from django.contrib import admin -from queue.views import Run, Queue, UserTasks, UserResult +from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView from rest_framework.urlpatterns import format_suffix_patterns # q = QueueTask() @@ -17,6 +17,8 @@ #url(r'run/', include(tasks_url)), #url(r'run/$',Run.as_view(),name='run-main'), url(r'run/(?P[-\w .]+)/$', Run.as_view(), name='run-main'), + url(r'file-upload/', FileUploadView.as_view(), name='file-upload'), + #url(r'file-upload/(?P[-\w.]+)', FileUploadView.as_view(), name='file-upload'), url(r'task/(?P[-\w]+)/$', UserResult.as_view(), name='queue-task-result'), url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'), url(r'^$', Queue.as_view(), name="queue-main"), diff --git a/queue/views.py b/queue/views.py index d0431eb..4a53956 100644 --- a/queue/views.py +++ b/queue/views.py @@ -2,16 +2,21 @@ from rest_framework.settings import api_settings from rest_framework.response import Response from rest_framework.reverse import reverse -from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly +from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly,AllowAny from rest_framework.views import APIView from ccelery.q import QueueTask, list_tasks, task_docstring from models import Run_model from rest_framework.renderers import JSONRenderer, JSONPRenderer from renderer import QueueRunBrowsableAPIRenderer -from rest_framework.parsers import JSONParser +from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser from util import trim from rest_framework.authtoken.models import Token +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator #task = list_tasks()['available_tasks'] +#from rest_framework.viewsets import ModelViewSet +#from serializer import FileUploadSerializer +import os q = QueueTask() @@ -40,7 +45,7 @@ def get(self, request,format=None): class Run(APIView): permission_classes = (DjangoModelPermissionsOrAnonReadOnly,) model = Run_model - parser_classes = (JSONParser,) + parser_classes = (JSONParser,MultiPartParser,FormParser) renderer_classes = (QueueRunBrowsableAPIRenderer, JSONRenderer, JSONPRenderer,) def __init__(self,q=q, *args, **kwargs): @@ -85,6 +90,44 @@ def post(self, request,task_name=None,format=None): result['result_url']=reverse('queue-task-result', kwargs={'task_id':result['task_id']}, request=request) return Response(result) +class FileUploadView(APIView): + + permission_classes =(AllowAny,) + parser_classes = (MultiPartParser, FormParser,) + #parser_classes = (FileUploadParser,) + renderer_classes = (JSONRenderer,) + + @method_decorator(csrf_exempt) + def dispatch(self, request, *args, **kwargs): + return super(FileUploadView, self).dispatch(request, *args, **kwargs) + + def get_username(self, request): + username = "guest" + if request.user.is_authenticated(): + username = request.user.username + return username + + def post(self, request, format=None): + resultDir = os.path.join("/data/tmp", self.get_username(request)) + try: + os.makedirs(resultDir) + except: + pass + result={} + for key,value in request.FILES.iteritems(): + filename= value.name + local_file = "%s/%s" % (resultDir,filename) + self.handle_file_upload(request.FILES[key],local_file) + result[key]=local_file + return Response(result) + def handle_file_upload(self,f,filename): + if f.multiple_chunks(): + with open(filename, 'wb+') as temp_file: + for chunk in f.chunks(): + temp_file.write(chunk) + else: + with open(filename, 'wb+') as temp_file: + temp_file.write(f.read()) class UserResult(APIView): permission_classes = (DjangoModelPermissionsOrAnonReadOnly,) @@ -137,51 +180,4 @@ def get(self, request,format=None, **kwargs): task_name = request.GET.get('taskname', None) username = self.get_username(request) data = self.q.history(username, task_name=task_name, page=page, limit=limit,request=request) - #for item in data: - - - # data['next']= reverse('queue-user-tasks',kwargs={'page':page+1,},request=request) - # if isinstance(data,ObjectId): - # return str(obj) - #print data - return Response(data) #,indent=4,sort_keys=True)) - - -""" - # print task_docstring(task_name) - html ="
Task Name: %s
Task Docstring:%sExample Curl:
%s
" - baseHelp =""curl --data-ascii '{ "function": "%s", - "queue": "%s", - "args": [], - "kwargs": {} - }' %s -H Content-Type:application/json"" % (task_name,"celery",reverse("%s-run" % (task_name), request=request)) - html = html % (task_name,task_docstring(task_name),baseHelp) - #html = html % (task_name,markdown.markdown(trim(task_docstring(task_name))),baseHelp) - return Response(html) #, template_name='rest_framework/queue_run_api.html') - - - -from json import JSONEncoder -from bson.objectid import ObjectId - -class MongoEncoder(JSONEncoder): - def default(self, obj, **kwargs): - if isinstance(obj, ObjectId): - return str(obj) - else: - return JSONEncoder.default(obj, **kwargs) - - - from rest_framework.renderers import JSONRenderer, JSONPRenderer -from rest_framework import renderers -import json - -class PlainTextRenderer(renderers.BaseRenderer): - media_type = 'text/plain' - format = 'txt' - - def render(self, data, media_type=None, renderer_context=None): - return data.encode(self.charset) - - -""" + return Response(data) diff --git a/requirements.txt b/requirements.txt index e47dcfe..f7a95f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,5 @@ six==1.7.2 mongoengine==0.8.7 django-rest-framework-mongoengine==2.0.2 wsgiref==0.1.2 +https://github.com/ouinformatics/dockertask/zipball/master +requests