Ads and scraper logging refactoring

.gitignore

@@ -2,3 +2,4 @@ node_modules
 python2.7
 2.7
 *.pyc
+logs/*

ads/migrations/0003_auto_20151029_1429.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ads', '0002_ads_link'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ads',
+            name='link',
+            field=models.TextField(default=b'', null=True),
+            preserve_default=True,
+        ),
+    ]

ads/migrations/0004_ads_placeholders.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import multiselectfield.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ads', '0003_auto_20151029_1429'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ads',
+            name='placeholders',
+            field=multiselectfield.db.fields.MultiSelectField(blank=True, max_length=165, null=True, choices=[(b'actu-top', b'Page Actu, haut de page'), (b'actu-side', b'Page Actu, colonne droite'), (b'actu-bottom', b'Page Actu, base de page'), (b'results-top', b'Pages R\xc3\xa9sultats, haut de page'), (b'results-side', b'Pages R\xc3\xa9sultats, colonne droite'), (b'results-bottom', b'Pages R\xc3\xa9sultats, base de page'), (b'blog-top', b'Pages Blog, haut de page'), (b'blog-side', b'Pages Blog, colonne droite'), (b'blog-bottom', b'Pages Blog, base de page'), (b'shop-top', b'Page Angulation, haut de page'), (b'shop-bottom', b'Page Angulation, base de page'), (b'sponsors-top', b'Page Mentors & Sponsors, haut de page'), (b'sponsors-side', b'Page Mentors & Sponsors, colonne droite'), (b'sponsors-bottom', b'Page Mentors & Sponsors, base de page')]),
+            preserve_default=True,
+        ),
+    ]

ads/models.py

@@ -1,13 +1,50 @@
 from django.db import models
+from django.conf import settings
+from multiselectfield import MultiSelectField
 
-# Create your models here.
+
+class AdsManager(models.Manager):
+    def get_queryset(self):
+        return super(AdsManager, self).get_queryset()
+
+    def by_placeholder(self, placeholder):
+        return super(AdsManager, self).get_queryset().filter(
+            placeholders__contains=placeholder
+        )
+
+    def by_category(self, category):
+
+        if category == 'square':
+            qs = super(AdsManager, self).get_queryset().filter(square=1)
+        elif category == 'horizontal':
+            qs = super(AdsManager, self).get_queryset().filter(horizontal=1)
+        elif category == 'vertical':
+            qs = super(AdsManager, self).get_queryset().filter(vertical=1)
+        else:
+            qs = super(AdsManager, self).get_queryset()
+
+        return qs
+
+    def by_placeholder_and_category(self, placeholder, category):
+        criteria = {
+            'placeholders__contains': placeholder,
+            category: 1,
+        }
+        qs = super(AdsManager, self).get_queryset().filter(**criteria)
+
+        return qs
 
 
 class Ads(models.Model):
     name = models.CharField(max_length=510)
     url = models.TextField(null=True)
     secureUrl = models.TextField(null=True)
     link = models.TextField(null=True, default='')
+    placeholders = MultiSelectField(
+        choices=settings.ADS_PLACEHOLDERS,
+        null=True,
+        blank=True,
+    )
     horizontal = models.PositiveSmallIntegerField(null=True)
     vertical = models.PositiveSmallIntegerField(null=True)
     square = models.PositiveSmallIntegerField(null=True)
@@ -16,5 +53,14 @@ class Ads(models.Model):
         auto_now=False
     )
 
+    objects = AdsManager()
+
     def __unicode__(self):
         return u'%s' % self.name
+
+
+class Placeholder(object):
+
+    def __init__(self, code, label):
+        self.code = code
+        self.label = label

apiv1/serializers.py

@@ -23,10 +23,15 @@ class AdsSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Ads
-        fields = ('id', 'name', 'link', 'url', 'secureUrl',
+        fields = ('id', 'name', 'link', 'url', 'secureUrl', 'placeholders',
                   'horizontal', 'vertical', 'square', 'date', )
 
 
+class AdsPlaceholdersSerializer(serializers.Serializer):
+    code = serializers.CharField(max_length=30)
+    label = serializers.CharField(max_length=100)
+
+
 class BloggersSerializer(serializers.ModelSerializer):
 
     class Meta:

apiv1/urls.py

@@ -46,6 +46,10 @@
         view=apiv1.views.AdsCreateReadView.as_view(),
         name='REST View'),
 
+    url(regex=r'^ads/placeholders/$',
+        view=apiv1.views.AdsPlaceholdersReadView.as_view(),
+        name='REST View'),
+
     url(regex=r'^ads/(?P<pk>[-\w]+)/$',
         view=apiv1.views.AdsReadUpdateDeleteView.as_view(),
         name='REST View'),

apiv1/views.py

@@ -4,13 +4,13 @@
     RetrieveUpdateDestroyAPIView,
 )
 from rest_framework.permissions import (
-    IsAuthenticated,
     IsAuthenticatedOrReadOnly
 )
 
 from apiv1.serializers import (
     NewsSerializer,
     AdsSerializer,
+    AdsPlaceholdersSerializer,
     BloggersSerializer,
     BlogPostsSerializer,
     SkiclubsSerializer,
@@ -22,9 +22,9 @@
 
 from django.utils import timezone
 from django.core.cache import cache
-
+from django.conf import settings
 from news.models import News
-from ads.models import Ads
+from ads.models import Ads, Placeholder
 from skiclubs.models import Skiclubs
 from pages.models import Pages
 from rankings.models import Races
@@ -43,7 +43,7 @@ class NewsCreateReadView(ListCreateAPIView):
 
     def get_queryset(self):
         now = timezone.now()
-        return News.objects.filter(date__lte=now).order_by('date').reverse()
+        return News.objects.filter(date__lte=now).exclude(mag=1).order_by('date').reverse()
 
 
 class MagCreateReadView(NewsCreateReadView):
@@ -87,15 +87,38 @@ class AdsCreateReadView(ListCreateAPIView):
     permission_classes = (IsAuthenticatedOrReadOnly, )
 
     def get_queryset(self):
-        if self.request.GET.get('category'):
-            cat = self.request.GET.get('category')
-            if cat == 'square':
-                return Ads.objects.filter(square=1)
-            if cat == 'horizontal':
-                return Ads.objects.filter(horizontal=1)
-            if cat == 'vertical':
-                return Ads.objects.filter(vertical=1)
-        return Ads.objects.all()
+        ads = []
+        placeholder = self.request.GET.get('placeholder', '')
+        ads_placeholders = getattr(settings, 'ADS_PLACEHOLDERS', None)
+
+        if placeholder == '' or placeholder in dict(ads_placeholders).keys():
+            category = self.request.GET.get('category', None)
+
+            if category is not None:
+                ads = Ads.objects.by_placeholder_and_category(
+                    placeholder,
+                    category
+                )
+            else:
+                ads = Ads.objects.all()
+
+        return ads
+
+
+class AdsPlaceholdersReadView(ListAPIView):
+    serializer_class = AdsPlaceholdersSerializer
+    permission_classes = (IsAuthenticatedOrReadOnly, )
+
+    def get_queryset(self):
+        placeholders = getattr(settings, 'ADS_PLACEHOLDERS', None)
+
+        if placeholders is not None:
+            return [
+                Placeholder(code=code, label=label)
+                for code, label in placeholders
+            ]
+        else:
+            return []
 
 
 class AdsReadUpdateDeleteView(RetrieveUpdateDestroyAPIView):

python_core/settings.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Django settings for python_core project.
 
@@ -61,6 +62,7 @@
     'skiclubs',
     'widgets',
     'angulation',
+    'multiselectfield',
 )
 
 MIDDLEWARE_CLASSES = (
@@ -142,3 +144,65 @@
 # https://docs.djangoproject.com/en/1.6/howto/static-files/
 
 STATIC_URL = '/static/'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'spider': {
+            'format': '%(asctime)s - %(levelname)s - %(message)s'
+        }
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
+        'spider': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(BASE_DIR, 'logs', 'spider'),
+            'when': 'midnight',
+            'formatter': 'spider',
+            'backupCount': 5,
+        },
+    },
+    'loggers': {
+        'dev': {
+            'handlers': ['console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'spider': {
+            'handlers': ['console', 'spider'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+    },
+}
+
+
+ADS_PLACEHOLDERS = (
+    ('sides', "Toutes les pages, de chaque côté"),
+    ('actu-top', "Page Actu, haut de page"),
+    ('actu-side', "Page Actu, colonne droite"),
+    ('actu-bottom', "Page Actu, base de page"),
+    ('results-top', "Pages Résultats, haut de page"),
+    ('results-side', "Pages Résultats, colonne droite"),
+    ('results-bottom', "Pages Résultats, base de page"),
+    ('blog-top', "Pages Blog, haut de page"),
+    ('blog-side', "Pages Blog, colonne droite"),
+    ('blog-bottom', "Pages Blog, base de page"),
+    ('shop-top', "Page Angulation, haut de page"),
+    ('shop-bottom', "Page Angulation, base de page"),
+    ('sponsors-top', "Page Mentors & Sponsors, haut de page"),
+    ('sponsors-side', "Page Mentors & Sponsors, colonne droite"),
+    ('sponsors-bottom', "Page Mentors & Sponsors, base de page"),
+)

python_core/settings_server.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Note: Make sure that wsgi.py is referencing this file, in case of error 400.
 
@@ -144,3 +145,55 @@
 # https://docs.djangoproject.com/en/1.6/howto/static-files/
 
 STATIC_URL = '/static/'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'spider': {
+            'format': '%(asctime)s - %(levelname)s - %(message)s'
+        }
+    },
+    'handlers': {
+        'spider': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(BASE_DIR, 'logs', 'spider'),
+            'when': 'midnight',
+            'formatter': 'spider',
+            'backupCount': 15,
+        },
+    },
+    'loggers': {
+        'spider': {
+            'handlers': ['spider'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+    },
+}
+
+
+ADS_PLACEHOLDERS = (
+    ('sides', "Toutes les pages, de chaque côté"),
+    ('actu-top', "Page Actu, haut de page"),
+    ('actu-side', "Page Actu, colonne droite"),
+    ('actu-bottom', "Page Actu, base de page"),
+    ('results-top', "Pages Résultats, haut de page"),
+    ('results-side', "Pages Résultats, colonne droite"),
+    ('results-bottom', "Pages Résultats, base de page"),
+    ('blog-top', "Pages Blog, haut de page"),
+    ('blog-side', "Pages Blog, colonne droite"),
+    ('blog-bottom', "Pages Blog, base de page"),
+    ('shop-top', "Page Angulation, haut de page"),
+    ('shop-bottom', "Page Angulation, base de page"),
+    ('sponsors-top', "Page Mentors & Sponsors, haut de page"),
+    ('sponsors-side', "Page Mentors & Sponsors, colonne droite"),
+    ('sponsors-bottom', "Page Mentors & Sponsors, base de page"),
+)

rankings/fis/fis/items.py

@@ -17,6 +17,19 @@ class FisRaces(Item):
     discipline = Field()
     table = Field()
 
+    # def __repr__(self):
+    #     """only print out few data after exiting the Pipeline"""
+    #     return repr({
+    #         "id": self["id"],
+    #         "category": self["category"],
+    #         "date": self["date"],
+    #         "discipline": self["discipline"],
+    #         "genre": self["genre"],
+    #         "info": self["info"],
+    #         "location": self["location"],
+    #         "link": self["link"],
+    #     })
+
 
 class FisRanking(Item):
     id = Field()