diff --git a/openprocurement/chronograph/__init__.py b/openprocurement/chronograph/__init__.py index 1e354dd..c5f627b 100644 --- a/openprocurement/chronograph/__init__.py +++ b/openprocurement/chronograph/__init__.py @@ -2,15 +2,10 @@ gevent.monkey.patch_all() import os from logging import getLogger -#from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor from apscheduler.schedulers.gevent import GeventScheduler as Scheduler from couchdb import Server, Session from couchdb.http import Unauthorized, extract_credentials from datetime import datetime, timedelta -#from openprocurement.chronograph.jobstores import CouchDBJobStore -from openprocurement.chronograph.constants import AUCTIONS -from openprocurement.chronograph.design import sync_design -from openprocurement.chronograph.managers import MANAGERS_MAPPING from openprocurement.chronograph.scheduler import push from openprocurement.chronograph.utils import add_logging_context, get_full_url from pyramid.config import Configurator @@ -100,23 +95,14 @@ def main(global_config, **settings): LOGGER.info("Updating chronograph db validate doc", extra={'MESSAGE_ID': 'update_chronograph_validate_doc'}) db.save(auth_doc) # sync couchdb views - sync_design(db) db = server[db_name] else: if db_name not in server: server.create(db_name) db = server[db_name] # sync couchdb views - sync_design(db) config.registry.db = db - config.registry.manager_mapper = {'types': {}, 'pmts': {}} - for auction in AUCTIONS: - auction_manager = MANAGERS_MAPPING[auction['type']]() - config.registry.manager_mapper['types'][auction['type']] = auction_manager - if auction.get('pmts', []): - config.registry.manager_mapper['pmts'].update({pmt: auction_manager for pmt in auction.get('pmts')}) - jobstores = { #'default': CouchDBJobStore(database=db_name, client=server) } diff --git a/openprocurement/chronograph/constants.py b/openprocurement/chronograph/constants.py index cd55e16..cda16b8 100644 --- a/openprocurement/chronograph/constants.py +++ b/openprocurement/chronograph/constants.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- from datetime import timedelta, time -CALENDAR_ID = 'calendar' -STREAMS_ID = 'streams' -WORKING_DAY_START = time(11, 0) -INSIDER_WORKING_DAY_START = time(9, 30) -TEXAS_WORKING_DAY_START = time(10, 0) -WORKING_DAY_END = time(16, 0) -INSIDER_WORKING_DAY_DURATION = timedelta(minutes=480) -TEXAS_WORKING_DAY_DURATION = timedelta(hours=7) # Does not affect anything ROUNDING = timedelta(minutes=29) MIN_PAUSE = timedelta(minutes=3) BIDDER_TIME = timedelta(minutes=6) @@ -18,25 +10,3 @@ SMOOTHING_REMIN = 60 # value should be greater than SMOOTHING_MIN and SMOOTHING_REMIN SMOOTHING_MAX = 300 -NOT_CLASSIC_AUCTIONS = ['dgfInsider', 'sellout.insider', 'appraisal.insider'] -STREAMS_KEYS = ['streams', 'dutch_streams', 'texas_streams'] -DEFAULT_STREAMS_DOC = { - '_id': STREAMS_ID, - 'streams': 10, - 'dutch_streams': 15, - 'texas_streams': 20 -} - -AUCTIONS = [ - { - 'type': 'insider', - 'pmts': ['dgfInsider', 'sellout.insider', 'appraisal.insider'] - }, - { - 'type': 'texas', - 'pmts': ['landLease'] - }, - { - 'type': 'english' - } -] diff --git a/openprocurement/chronograph/design.py b/openprocurement/chronograph/design.py index 65f2f72..e69de29 100644 --- a/openprocurement/chronograph/design.py +++ b/openprocurement/chronograph/design.py @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -from couchdb.design import ViewDefinition -from openprocurement.chronograph.constants import ( - INSIDER_WORKING_DAY_START, TEXAS_WORKING_DAY_START -) - - -def sync_design(db): - views = [j for i, j in globals().items() if "_view" in i] - ViewDefinition.sync_many(db, views) - - -plan_auctions_view = ViewDefinition('plan', 'auctions', '''function(doc) { - if(doc.streams || doc.dutch_streams || doc.texas_streams) { - for (var i in doc) { - if (i.indexOf('stream_') == 0) { - for (var t in doc[i]) { - if (doc[i][t]) { - var x = doc[i][t].split('_') - if (x.length == 2) { - emit(x, doc._id.split('_')[1] + 'T' + t); - } else { - emit([x[0], null], - doc._id.split('_')[1] + 'T' + t); - } - } - } - } - if (i.indexOf('dutch_streams') == 0) { - for (var aid in doc[i]) { - emit([doc[i][aid], null], - doc._id.split('_')[1] + 'T' + '%s'); - } - } - if (i.indexOf('texas_streams') == 0) { - for (var aid in doc[i]) { - emit([doc[i][aid], null], - doc._id.split('_')[1] + 'T' + '%s'); - } - } - }; - } -}''' % (INSIDER_WORKING_DAY_START.isoformat(), TEXAS_WORKING_DAY_START.isoformat())) diff --git a/openprocurement/chronograph/managers.py b/openprocurement/chronograph/managers.py deleted file mode 100644 index 4a56363..0000000 --- a/openprocurement/chronograph/managers.py +++ /dev/null @@ -1,222 +0,0 @@ -# -*- coding: utf-8 -*- -from copy import deepcopy - -from couchdb import ResourceConflict -from datetime import timedelta, datetime -from iso8601 import parse_date - -from openprocurement.chronograph.constants import ( - WORKING_DAY_START, - WORKING_DAY_END, - TEXAS_WORKING_DAY_START, - INSIDER_WORKING_DAY_START, - DEFAULT_STREAMS_DOC, - STREAMS_ID, - INSIDER_WORKING_DAY_DURATION, - TEXAS_WORKING_DAY_DURATION -) -from openprocurement.chronograph.utils import TZ, find_free_slot - - -class BaseAuctionsManager(object): - working_day_start = None - streams_key = None - - def get_date(self, db, mode, date): - """ - - Get hour of auction start and stream in which auction will run for - passed date. - - :param db: chronograph database with plan docs - :type db: couchdb.Database - :param mode: value from auction procedure, indicates mode of auction, used in plan doc id - :type mode: str - :param date: preplanned date for current auction - :type date: datetime.date - - :returns : (time of auction start, stream for auction, plan document) - :rtype: (datetime.time, int, couchdb.client.Document) - """ - plan_id = 'plan{}_{}'.format(mode, date.isoformat()) - plan = db.get(plan_id, {'_id': plan_id}) - plan_date_end, stream = self._get_hours_and_stream(plan) - plan_date = parse_date(date.isoformat() + 'T' + plan_date_end, None) - plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) - return plan_date.time(), stream, plan - - def get_streams(self, db, streams_id=STREAMS_ID): - """ - - Get allowed amount of streams for auction of particular type per day. - - :param db: chronograph database - :type db: couchdb.Database - :param streams_id: id of document with defined stream amounts - :type streams_id: str - - :return: amount of streams for auction per day - :rtype: int - """ - - streams = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) - return streams.get(self.streams_key, DEFAULT_STREAMS_DOC[self.streams_key]) - - def _get_hours_and_stream(self, plan): - """ - Return time of auction start and first available stream from plan doc - - :param plan: document for planned auctions - :type plan: couchdb.client.Document - - :return: (time of auction start, first available stream) - :rtype: (str, int) - """ - raise NotImplementedError - - def set_end_of_auction(self, *args, **kwargs): - """ - Try to find end of auction for passed date, time and plan doc if such - action is possible and return it's value for further auction planning - """ - raise NotImplementedError - - def set_date(self, *args, **kwargs): - """ - Actually plan auction for particular date, time, stream and slot and - save it to chronograph database. - """ - raise NotImplementedError - - def free_slot(self, *args, **kwargs): - """ - Remove particular auction from stream slot in particular plan document. - """ - raise NotImplementedError - - -class ClassicAuctionsManager(BaseAuctionsManager): - working_day_start = WORKING_DAY_START - working_day_end = WORKING_DAY_END - streams_key = 'streams' - - def _get_hours_and_stream(self, plan): - plan_date_end = plan.get('time', self.working_day_start.isoformat()) - stream = plan.get(self.streams_key, 1) - return plan_date_end, stream - - def set_date(self, db, plan, auction_id, end_time, - cur_stream, start_time, new_slot=True): - if new_slot: - plan['time'] = end_time.isoformat() - plan[self.streams_key] = cur_stream - stream_id = 'stream_{}'.format(cur_stream) - stream = plan.get(stream_id, {}) - stream[start_time.isoformat()] = auction_id - plan[stream_id] = stream - db.save(plan) - - def free_slot(self, db, plan_id, auction_id, plan_time): - slot = plan_time.time().isoformat() - done = False - while not done: - try: - plan = db.get(plan_id) - streams = plan[self.streams_key] - for cur_stream in range(1, streams + 1): - stream_id = 'stream_{}'.format(cur_stream) - if plan[stream_id].get(slot) == auction_id: - plan[stream_id][slot] = None - db.save(plan) - done = True - except ResourceConflict: - done = False - except: - done = True - - def set_end_of_auction(self, stream, streams, nextDate, dayStart, plan): - freeSlot = find_free_slot(plan) - if freeSlot: - startDate, stream = freeSlot - start, end, dayStart, new_slot = startDate, startDate, startDate.time(), False - return start, end, dayStart, stream, new_slot - if dayStart >= self.working_day_end and stream < streams: - stream += 1 - dayStart = self.working_day_start - - start = TZ.localize(datetime.combine(nextDate, dayStart)) - end = start + timedelta(minutes=30) - - # end = calc_auction_end_time(auction.get('numberOfBids', len(auction.get('bids', []))), start) - - # TODO: redundant check, which was used with previous end calculation logic: - # if dayStart == self.working_day_start and end > TZ.localize( - # datetime.combine(nextDate, self.working_day_end) - # ) and stream <= streams: - # return start, end, dayStart, stream, True - - if end <= TZ.localize(datetime.combine(nextDate, self.working_day_end)) and stream <= streams: - return start, end, dayStart, stream, True - - -class NonClassicAuctionsManager(BaseAuctionsManager): - working_day_start = None - working_day_duration = None - streams_key = None - - def _get_hours_and_stream(self, plan): - plan_date_end = self.working_day_start.isoformat() - stream = len(plan.get(self.streams_key, [])) - return plan_date_end, stream - - def set_date(self, db, plan, auction_id, *args, **kwargs): - streams = plan.get(self.streams_key, []) - streams.append(auction_id) - plan[self.streams_key] = streams - db.save(plan) - - def free_slot(self, db, plan_id, auction_id, *args, **kwargs): - done = False - while not done: - try: - plan = db.get(plan_id) - slots = plan.get(self.streams_key, []) - pops = [] - for i in xrange(0, len(slots)): - if slots[i] == auction_id: - pops.append(i) - pops.sort(reverse=True) - for p in pops: - slots.pop(p) - plan[self.streams_key] = slots - db.save(plan) - done = True - except ResourceConflict: - done = False - except: - done = True - - def set_end_of_auction(self, stream, streams, nextDate, dayStart, *args, **kwargs): - if stream < streams: - start = TZ.localize(datetime.combine(nextDate, dayStart)) - end = start + self.working_day_duration - return start, end, dayStart, stream, False - - -class TexasAuctionsManager(NonClassicAuctionsManager): - working_day_start = TEXAS_WORKING_DAY_START - working_day_duration = TEXAS_WORKING_DAY_DURATION - streams_key = 'texas_streams' - - -class InsiderAuctionsManager(NonClassicAuctionsManager): - working_day_start = INSIDER_WORKING_DAY_START - working_day_duration = INSIDER_WORKING_DAY_DURATION - streams_key = 'dutch_streams' - - -MANAGERS_MAPPING = { - 'texas': TexasAuctionsManager, - 'insider': InsiderAuctionsManager, - 'english': ClassicAuctionsManager -} diff --git a/openprocurement/chronograph/scheduler.py b/openprocurement/chronograph/scheduler.py index 7c88b67..4b5bdb2 100644 --- a/openprocurement/chronograph/scheduler.py +++ b/openprocurement/chronograph/scheduler.py @@ -1,25 +1,15 @@ # -*- coding: utf-8 -*- import requests -from couchdb.http import ResourceConflict from datetime import timedelta -from iso8601 import parse_date from json import dumps from logging import getLogger from openprocurement.chronograph.utils import ( context_unpack, update_next_check_job, push, - get_request, calc_auction_end_time, get_calendar, - skipped_days, randomize, get_now, get_manager_for_auction -) -from openprocurement.chronograph.design import plan_auctions_view -from openprocurement.chronograph.constants import ( - WORKING_DAY_START, - SMOOTHING_MIN, - SMOOTHING_REMIN, - SMOOTHING_MAX, + get_request, + get_now ) from os import environ from pytz import timezone -from random import randint from time import sleep from urllib import urlencode @@ -37,131 +27,6 @@ PLANNING_OPT_FIELDS = ['status', 'next_check', 'auctionPeriod', 'procurementMethodType', 'lots', 'auctionParameters'] -def planning_auction(auction, mapper, start, db, quick=False, lot_id=None): - tid = auction.get('id', '') - mode = auction.get('mode', '') - manager = get_manager_for_auction(auction, mapper) - skipped_days = 0 - if quick: - quick_start = calc_auction_end_time(0, start) - return quick_start, 0, skipped_days - calendar = get_calendar(db) - streams = manager.get_streams(db) - start += timedelta(hours=1) - if start.time() > manager.working_day_start: - nextDate = start.date() + timedelta(days=1) - else: - nextDate = start.date() - while True: - # skip Saturday and Sunday - if calendar.get(nextDate.isoformat()) or nextDate.weekday() in [5, 6]: - nextDate += timedelta(days=1) - continue - dayStart, stream, plan = manager.get_date(db, mode, nextDate) - result = manager.set_end_of_auction(stream, streams, nextDate, dayStart, plan) - if result: - start, end, dayStart, stream, new_slot = result - break - nextDate += timedelta(days=1) - skipped_days += 1 - manager.set_date(db, plan, "_".join([tid, lot_id]) if lot_id else tid, end.time(), stream, dayStart, new_slot) - return start, stream, skipped_days - - -def check_auction(request, auction, db, mapper): - now = get_now() - quick = environ.get('SANDBOX_MODE', False) and u'quick' in auction.get('submissionMethodDetails', '') - if not auction.get('lots') and 'shouldStartAfter' in auction.get('auctionPeriod', {}) and auction['auctionPeriod']['shouldStartAfter'] > auction['auctionPeriod'].get('startDate'): - period = auction.get('auctionPeriod') - shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) - planned = False - while not planned: - try: - auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick) - planned = True - except ResourceConflict: - planned = False - auctionPeriod = randomize(auctionPeriod).isoformat() - planned = 'replanned' if period.get('startDate') else 'planned' - LOGGER.info('{} auction for auction {} to {}. Stream {}.{}'.format(planned.title(), auction['id'], auctionPeriod, stream, skipped_days(skip_days)), - extra=context_unpack(request, - {'MESSAGE_ID': '{}_auction_auction'.format(planned)}, - {'PLANNED_DATE': auctionPeriod, 'PLANNED_STREAM': stream, 'PLANNED_DAYS_SKIPPED': skip_days})) - return {'auctionPeriod': {'startDate': auctionPeriod}} - elif auction.get('lots'): - lots = [] - for lot in auction.get('lots', []): - if lot['status'] != 'active' or 'shouldStartAfter' not in lot.get('auctionPeriod', {}) or lot['auctionPeriod']['shouldStartAfter'] < lot['auctionPeriod'].get('startDate'): - lots.append({}) - continue - period = lot.get('auctionPeriod') - shouldStartAfter = max(parse_date(period.get('shouldStartAfter'), TZ).astimezone(TZ), now) - lot_id = lot['id'] - planned = False - while not planned: - try: - auctionPeriod, stream, skip_days = planning_auction(auction, mapper, shouldStartAfter, db, quick, lot_id) - planned = True - except ResourceConflict: - planned = False - auctionPeriod = randomize(auctionPeriod).isoformat() - planned = 'replanned' if period.get('startDate') else 'planned' - lots.append({'auctionPeriod': {'startDate': auctionPeriod}}) - LOGGER.info('{} auction for lot {} of auction {} to {}. Stream {}.{}'.format(planned.title(), lot_id, auction['id'], auctionPeriod, stream, skipped_days(skip_days)), - extra=context_unpack(request, - {'MESSAGE_ID': '{}_auction_lot'.format(planned)}, - {'PLANNED_DATE': auctionPeriod, 'PLANNED_STREAM': stream, 'PLANNED_DAYS_SKIPPED': skip_days, 'LOT_ID': lot_id})) - if any(lots): - return {'lots': lots} - return None - - -def resync_auction(request): - auction_id = request.matchdict['auction_id'] - scheduler = request.registry.scheduler - url = '{}/{}'.format(request.registry.full_url, auction_id) - api_token = request.registry.api_token - resync_url = request.registry.callback_url + 'resync/' + auction_id - recheck_url = request.registry.callback_url + 'recheck/' + auction_id - db = request.registry.db - request_id = request.environ.get('REQUEST_ID', '') - next_check = None - next_sync = None - r = get_request( - url, auth=(api_token, ''), session=SESSION, headers={'X-Client-Request-ID': request_id} - ) - if r.status_code != requests.codes.ok: - LOGGER.error("Error {} on getting auction '{}': {}".format(r.status_code, url, r.text), - extra=context_unpack(request, {'MESSAGE_ID': 'error_get_auction'}, {'ERROR_STATUS': r.status_code})) - if r.status_code == requests.codes.not_found: - return - next_sync = get_now() + timedelta(seconds=randint(SMOOTHING_REMIN, SMOOTHING_MAX)) - else: - json = r.json() - auction = json['data'] - changes = check_auction(request, auction, db, request.registry.manager_mapper) - if changes: - data = dumps({'data': changes}) - r = SESSION.patch(url, - data=data, - headers={'Content-Type': 'application/json', 'X-Client-Request-ID': request_id}, - auth=(api_token, '')) - if r.status_code != requests.codes.ok: - LOGGER.error("Error {} on updating auction '{}' with '{}': {}".format(r.status_code, url, data, r.text), - extra=context_unpack(request, {'MESSAGE_ID': 'error_patch_auction'}, {'ERROR_STATUS': r.status_code})) - next_sync = get_now() + timedelta(seconds=randint(SMOOTHING_REMIN, SMOOTHING_MAX)) - elif r.json(): - next_check = r.json()['data'].get('next_check') - if next_check: - update_next_check_job(next_check, scheduler, auction_id, get_now(), recheck_url) - if next_sync: - scheduler.add_job(push, 'date', run_date=next_sync+timedelta(seconds=randint(SMOOTHING_MIN, SMOOTHING_MAX)), timezone=TZ, - id=auction_id, name="Resync {}".format(auction_id), - misfire_grace_time=60 * 60, replace_existing=True, - args=[resync_url, None]) - return next_sync and next_sync.isoformat() - - def recheck_auction(request): auction_id = request.matchdict['auction_id'] scheduler = request.registry.scheduler @@ -186,56 +51,14 @@ def recheck_auction(request): return next_check and next_check.isoformat() -def check_inner_auction(db, auction, mapper): - manager = get_manager_for_auction(auction, mapper) - - auction_time = auction.get('auctionPeriod', {}).get('startDate') and \ - parse_date(auction.get('auctionPeriod', {}).get('startDate')) - lots = dict([ - (i['id'], parse_date(i.get('auctionPeriod', {}).get('startDate'))) - for i in auction.get('lots', []) - if i.get('auctionPeriod', {}).get('startDate') - ]) - auc_list = [ - (x.key[1], TZ.localize(parse_date(x.value, None)), x.id) - for x in plan_auctions_view(db, startkey=[auction['id'], None], - endkey=[auction['id'], 32 * "f"]) - ] - for key, plan_time, plan_doc in auc_list: - if not key and (not auction_time or not - plan_time < auction_time < plan_time + - timedelta(minutes=30)): - manager.free_slot(db, plan_doc, auction['id'], plan_time) - elif key and (not lots.get(key) or lots.get(key) and not - plan_time < lots.get(key) < plan_time + - timedelta(minutes=30)): - manager.free_slot(db, plan_doc, "_".join([auction['id'], key]), plan_time) - - -def process_listing(auctions, scheduler, callback_url, db, mapper, check=True, planning=True): +def process_listing(auctions, scheduler, callback_url): run_date = get_now() for auction in auctions: - if check: - check_inner_auction(db, auction, mapper) tid = auction['id'] next_check = auction.get('next_check') if next_check: recheck_url = ''.join([callback_url, 'recheck/', tid]) update_next_check_job(next_check, scheduler, tid, run_date, recheck_url, True) - if planning: - if any([ - 'shouldStartAfter' in i.get('auctionPeriod', {}) and i['auctionPeriod']['shouldStartAfter'] > i['auctionPeriod'].get('startDate') - for i in auction.get('lots', []) - ]) or ( - 'shouldStartAfter' in auction.get('auctionPeriod', {}) and auction['auctionPeriod']['shouldStartAfter'] > auction['auctionPeriod'].get('startDate') - ): - resync_job = scheduler.get_job(tid) - if not resync_job or resync_job.next_run_time > run_date + timedelta(minutes=1): - scheduler.add_job(push, 'date', run_date=run_date+timedelta(seconds=randint(SMOOTHING_MIN, SMOOTHING_MAX)), timezone=TZ, - id=tid, name="Resync {}".format(tid), - misfire_grace_time=60 * 60, - args=[callback_url + 'resync/' + tid, None], - replace_existing=True) def resync_auctions(request): @@ -268,8 +91,7 @@ def resync_auctions(request): next_url = json['prev_page']['uri'] if not json['data']: break - process_listing(json['data'], scheduler, callback_url, request.registry.db, - request.registry.manager_mapper, planning=request.registry.planning) + process_listing(json['data'], scheduler, callback_url) sleep(0.1) except Exception as e: LOGGER.error("Error on resync all: {}".format(repr(e)), extra=context_unpack(request, {'MESSAGE_ID': 'error_resync_all'})) @@ -307,8 +129,7 @@ def resync_auctions_back(request): if not json['data']: LOGGER.info("Resync back stopped", extra=context_unpack(request, {'MESSAGE_ID': 'resync_back_stoped'})) return next_url - process_listing(json['data'], scheduler, callback_url, request.registry.db, - request.registry.manager_mapper, False, request.registry.planning) + process_listing(json['data'], scheduler, callback_url) sleep(0.1) except Exception as e: LOGGER.error("Error on resync back: {}".format(repr(e)), extra=context_unpack(request, {'MESSAGE_ID': 'error_resync_back'})) diff --git a/openprocurement/chronograph/tests/base.py b/openprocurement/chronograph/tests/base.py index 9ec5485..ae55a3c 100644 --- a/openprocurement/chronograph/tests/base.py +++ b/openprocurement/chronograph/tests/base.py @@ -106,7 +106,6 @@ def request(method, url, **kwargs): self.app.app.registry.full_url = get_full_url(self.app.app.registry) self.couchdb_server = self.app.app.registry.couchdb_server self.db = self.app.app.registry.db - self.mapper = self.app.app.registry.manager_mapper if not self.scheduler: self.app.app.registry.scheduler.shutdown() diff --git a/openprocurement/chronograph/tests/main.py b/openprocurement/chronograph/tests/main.py index f0ff333..02d3dab 100644 --- a/openprocurement/chronograph/tests/main.py +++ b/openprocurement/chronograph/tests/main.py @@ -2,13 +2,12 @@ import unittest -from openprocurement.chronograph.tests import test, test_scheduler +from openprocurement.chronograph.tests import test def suite(): tests = unittest.TestSuite() tests.addTest(test.suite()) - tests.addTest(test_scheduler.suite()) return tests diff --git a/openprocurement/chronograph/tests/test.py b/openprocurement/chronograph/tests/test.py index 7d98f3b..e5c5de9 100644 --- a/openprocurement/chronograph/tests/test.py +++ b/openprocurement/chronograph/tests/test.py @@ -4,16 +4,12 @@ from datetime import datetime, timedelta from logging import getLogger -from openprocurement.chronograph.utils import get_manager_for_auction from time import sleep import requests -from iso8601 import parse_date from openprocurement.chronograph import TZ -from openprocurement.chronograph.constants import DEFAULT_STREAMS_DOC, WORKING_DAY_START from openprocurement.chronograph.tests.utils import update_json -from openprocurement.chronograph.scheduler import planning_auction from openprocurement.chronograph.tests.base import BaseWebTest, BaseAuctionWebTest from openprocurement.chronograph.tests.data import test_bids, test_lots, test_auction_data @@ -51,114 +47,11 @@ def test_resync_back(self): self.assertEqual(response.status, '200 OK') self.assertNotEqual(response.json, None) - def test_resync_one(self): - response = self.app.get('/resync/all') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - def test_recheck_one(self): response = self.app.get('/recheck/all') self.assertEqual(response.status, '200 OK') self.assertEqual(response.json, None) - def test_calendar(self): - response = self.app.get('/calendar') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, []) - - def test_calendar_entry(self): - response = self.app.get('/calendar/2015-04-23') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - response = self.app.post('/calendar/2015-04-23') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, True) - response = self.app.delete('/calendar/2015-04-23') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - response = self.app.get('/calendar/2015-04-23') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - - def test_streams(self): - # GET /streams - response = self.app.get('/streams') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, DEFAULT_STREAMS_DOC['streams']) - - response = self.app.get('/streams?dutch_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, DEFAULT_STREAMS_DOC['dutch_streams']) - - response = self.app.get('/streams?texas_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, DEFAULT_STREAMS_DOC['texas_streams']) - - # POST /streams - response = self.app.post('/streams', {'streams': 20}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, True) - - # GET /streams - response = self.app.get('/streams') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 20) - - # POST /streams - response = self.app.post('/streams', {'dutch_streams': 21}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, True) - - # GET /streams - response = self.app.get('/streams?dutch_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 21) - - # POST /streams - response = self.app.post('/streams', {'texas_streams': 42}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, True) - - # GET /streams - response = self.app.get('/streams?texas_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 42) - - - # POST /streams - response = self.app.post('/streams', {'streams': -20}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - response = self.app.post('/streams', {'dutch_streams': -20}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - response = self.app.post('/streams', {'texas_streams': -20}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - response = self.app.post('/streams', {'streams': 11, - 'dutch_streams': 12, - 'texas_streams': 13}) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, True) - - # GET /streams - response = self.app.get('/streams') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 11) - - response = self.app.get('/streams?dutch_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 12) - - response = self.app.get('/streams?texas_streams=true') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, 13) - - # POST /streams - response = self.app.patch('/streams') - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, False) - class AuctionsTest(BaseAuctionWebTest): @@ -177,17 +70,6 @@ def test_list_jobs(self): self.assertEqual(len(response.json['jobs']), 2) self.assertIn("recheck_{}".format(self.auction_id), response.json['jobs']) - def test_resync_all(self): - response = self.app.get('/resync_all') - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - sleep(0.1) - response = self.app.get('/') - self.assertEqual(response.status, '200 OK') - self.assertIn('jobs', response.json) - self.assertEqual(len(response.json['jobs']), 2) - self.assertIn("recheck_{}".format(self.auction_id), response.json['jobs']) - class AuctionTest(BaseAuctionWebTest): scheduler = False @@ -262,220 +144,6 @@ def test_wait_for_tenderPeriod(self): auction = response.json['data'] self.assertEqual(auction['status'], 'active.enquiries') - def test_set_auctionPeriod_jobs(self): - now = datetime.now(TZ) - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': { - "enquiryPeriod": { - "endDate": now.isoformat() - }, - 'tenderPeriod': { - 'startDate': now.isoformat(), - 'endDate': (now + timedelta(days=1)).isoformat() - } - } - }) - for _ in range(100): - self.app.app.registry.scheduler.start() - response = self.app.get('/resync_all') - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - response = self.app.get('/') - self.app.app.registry.scheduler.shutdown() - self.assertEqual(response.status, '200 OK') - self.assertIn('jobs', response.json) - self.assertEqual(len(response.json['jobs']), 2) - if "recheck_{}".format(self.auction_id) in response.json['jobs']: - break - self.assertIn("recheck_{}".format(self.auction_id), response.json['jobs']) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - for _ in range(10): - self.app.app.registry.scheduler.start() - self.app.get('/resync_all') - self.app.app.registry.scheduler.shutdown() - - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - if 'startDate' in auction['lots'][0]['auctionPeriod']: - break - else: - self.assertIn('auctionPeriod', auction) - if 'startDate' in auction['auctionPeriod']: - break - else: - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - if self.initial_lots: - self.assertIn('startDate', auction['lots'][0]['auctionPeriod']) - else: - self.assertIn('startDate', auction['auctionPeriod']) - - def test_set_auctionPeriod_nextday(self): - now = datetime.now(TZ) - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': { - "enquiryPeriod": { - "endDate": now.isoformat() - }, - 'tenderPeriod': { - 'startDate': now.isoformat(), - 'endDate': (now + timedelta(days=7 - now.weekday())).replace(hour=13).isoformat() - } - } - }) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - self.assertEqual(parse_date(auction['lots'][0]['auctionPeriod']['startDate'], TZ).weekday(), 1) - else: - self.assertIn('auctionPeriod', auction) - self.assertEqual(parse_date(auction['auctionPeriod']['startDate'], TZ).weekday(), 1) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - self.app.app.registry.scheduler.start() - response = self.app.get('/') - self.assertEqual(response.status, '200 OK') - self.assertIn('jobs', response.json) - self.assertIn('recheck_{}'.format(self.auction_id), response.json['jobs']) - self.assertGreaterEqual(parse_date(response.json['jobs']["recheck_{}".format(self.auction_id)]).utctimetuple(), parse_date(auction['tenderPeriod']['endDate']).utctimetuple()) - self.assertLessEqual(parse_date(response.json['jobs']["recheck_{}".format(self.auction_id)]).utctimetuple(), (parse_date(auction['tenderPeriod']['endDate']) + timedelta(minutes=5)).utctimetuple()) - - def test_set_auctionPeriod_skip_weekend(self): - now = datetime.now(TZ) - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': { - "enquiryPeriod": { - "endDate": now.isoformat() - }, - 'tenderPeriod': { - 'startDate': now.isoformat(), - 'endDate': (now + timedelta(days=5 - now.weekday(), hours=1)).isoformat() - } - } - }) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - self.assertEqual(parse_date(auction['lots'][0]['auctionPeriod']['startDate'], TZ).weekday(), 0) - else: - self.assertIn('auctionPeriod', auction) - self.assertEqual(parse_date(auction['auctionPeriod']['startDate'], TZ).weekday(), 0) - - def test_set_auctionPeriod_skip_holidays(self): - now = datetime.now(TZ) - today = now.date() - for i in range(10): - date = today + timedelta(days=i) - self.app.post('/calendar/' + date.isoformat()) - calendar = self.app.get('/calendar').json - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': { - "enquiryPeriod": { - "endDate": now.isoformat() - }, - 'tenderPeriod': { - 'startDate': now.isoformat(), - 'endDate': (now + timedelta(days=1)).isoformat() - } - } - }) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - auctionPeriodstart = parse_date(auction['lots'][0]['auctionPeriod']['startDate'], TZ) - else: - self.assertIn('auctionPeriod', auction) - auctionPeriodstart = parse_date(auction['auctionPeriod']['startDate'], TZ) - self.assertNotIn(auctionPeriodstart.date().isoformat(), calendar) - self.assertTrue(auctionPeriodstart.date() > date) - - def test_set_auctionPeriod_today(self): - now = datetime.now(TZ) - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': { - "enquiryPeriod": { - "endDate": now.isoformat() - }, - 'tenderPeriod': { - 'startDate': now.isoformat(), - 'endDate': (now + timedelta(days=7 - now.weekday())).replace(hour=1).isoformat() - } - } - }) - response = self.app.get('/recheck/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertNotEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.tendering') - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - self.assertEqual(parse_date(auction['lots'][0]['auctionPeriod']['startDate'], TZ).weekday(), 1) - else: - self.assertIn('auctionPeriod', auction) - self.assertEqual(parse_date(auction['auctionPeriod']['startDate'], TZ).weekday(), 1) - def test_switch_to_unsuccessful(self): response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { 'data': { @@ -595,52 +263,6 @@ def test_reschedule_auction(self): response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) auction = response.json()['data'] self.assertEqual(auction['status'], 'active.auction') - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - auction = response.json()['data'] - date_time = TZ.localize(datetime.combine(datetime.now().date(), WORKING_DAY_START)) - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - auctionPeriod = auction['lots'][0]['auctionPeriod']['startDate'] - auction['lots'][0]['auctionPeriod']['startDate'] = date_time.isoformat() - else: - self.assertIn('auctionPeriod', auction) - auctionPeriod = auction['auctionPeriod']['startDate'] - auction['auctionPeriod']['startDate'] = date_time.isoformat() - update_json(self.api, 'auction', self.auction_id, {"data": auction}) - response = requests.patch('{}/{}'.format(self.app.app.registry.full_url, self.auction_id), { - 'data': {"id": "f547ece35436484e8656a2988fb52a44"}}) - self.assertEqual(response.status_code, 200) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - self.assertEqual(auction['status'], 'active.auction') - if self.initial_lots: - self.assertNotIn('auctionPeriod', auction) - self.assertIn('auctionPeriod', auction['lots'][0]) - self.assertIn('shouldStartAfter', auction['lots'][0]['auctionPeriod']) - self.assertIn('startDate', auction['lots'][0]['auctionPeriod']) - self.assertGreater(auction['lots'][0]['auctionPeriod']['shouldStartAfter'], auction['lots'][0]['auctionPeriod'].get('startDate')) - else: - self.assertIn('auctionPeriod', auction) - self.assertIn('shouldStartAfter', auction['auctionPeriod']) - self.assertIn('startDate', auction['auctionPeriod']) - self.assertGreater(auction['auctionPeriod']['shouldStartAfter'], auction['auctionPeriod'].get('startDate')) - response = self.app.get('/resync/' + self.auction_id) - self.assertEqual(response.status, '200 OK') - self.assertEqual(response.json, None) - response = requests.get('{}/{}'.format(self.app.app.registry.full_url, self.auction_id)) - response.json = response.json() - auction = response.json['data'] - if self.initial_lots: - self.assertIn('auctionPeriod', auction['lots'][0]) - self.assertGreater(auction['lots'][0]['auctionPeriod']['startDate'], auctionPeriod) - else: - self.assertIn('auctionPeriod', auction) - self.assertGreater(auction['auctionPeriod']['startDate'], auctionPeriod) class AuctionLotTest3(AuctionTest3): @@ -655,80 +277,12 @@ class AuctionLotTest4(AuctionTest4): initial_lots = test_lots -class AuctionPlanning(BaseWebTest): - scheduler = False - - def test_auction_quick_planning(self): - now = datetime.now(TZ) - auctionPeriodstartDate = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db, True)[0] - self.assertTrue(now < auctionPeriodstartDate < now + timedelta(hours=1)) - - def test_auction_quick_planning_insider(self): - now = datetime.now(TZ) - my_test_auction = deepcopy(test_auction_data_test_quick) - my_test_auction['procurementMethodType'] = 'dgfInsider' - auctionPeriodstartDate = planning_auction( - my_test_auction, self.mapper, now, self.db, True - )[0] - self.assertTrue( - now < auctionPeriodstartDate < now + timedelta(hours=1) - ) - - def test_auction_planning_overlow_insider(self): - now = datetime.now(TZ) - my_test_auction = deepcopy(test_auction_data_test_quick) - my_test_auction['procurementMethodType'] = 'dgfInsider' - res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] - startDate = res.date() - count = 0 - while startDate == res.date(): - count += 1 - res = planning_auction(my_test_auction, self.mapper, now, self.db)[0] - self.assertEqual(count, 15) - - def test_auction_planning_overlow(self): - now = datetime.now(TZ) - res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] - startDate = res.date() - count = 0 - while startDate == res.date(): - count += 1 - res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] - self.assertEqual(count, 100) - - def test_auction_planning_free(self): - now = datetime.now(TZ) - test_auction_data_test_quick.pop("id") - res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] - startDate, startTime = res.date(), res.time() - manager = get_manager_for_auction(test_auction_data, self.mapper) - manager.free_slot(self.db, "plantest_{}".format(startDate.isoformat()), "", res) - res = planning_auction(test_auction_data_test_quick, self.mapper, now, self.db)[0] - self.assertEqual(res.time(), startTime) - - def test_auction_planning_buffer(self): - some_date = datetime(2015, 9, 21, 6, 30) - date = some_date.date() - ndate = (some_date + timedelta(days=1)).date() - res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] - self.assertEqual(res.date(), date) - some_date = some_date.replace(hour=10) - res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] - self.assertNotEqual(res.date(), date) - self.assertEqual(res.date(), ndate) - some_date = some_date.replace(hour=16) - res = planning_auction(test_auction_data_test_quick, self.mapper, some_date, self.db)[0] - self.assertNotEqual(res.date(), date) - self.assertEqual(res.date(), ndate) - - def suite(): tests = unittest.TestSuite() tests.addTest(unittest.makeSuite(AuctionLotTest)) tests.addTest(unittest.makeSuite(AuctionLotTest2)) tests.addTest(unittest.makeSuite(AuctionLotTest3)) tests.addTest(unittest.makeSuite(AuctionLotTest4)) - tests.addTest(unittest.makeSuite(AuctionPlanning)) tests.addTest(unittest.makeSuite(AuctionTest)) tests.addTest(unittest.makeSuite(AuctionTest2)) tests.addTest(unittest.makeSuite(AuctionTest3)) diff --git a/openprocurement/chronograph/tests/test_scheduler.py b/openprocurement/chronograph/tests/test_scheduler.py deleted file mode 100644 index 342d5f7..0000000 --- a/openprocurement/chronograph/tests/test_scheduler.py +++ /dev/null @@ -1,155 +0,0 @@ -import os -import unittest -from ConfigParser import ConfigParser -from couchdb import Server -from datetime import datetime -from iso8601 import parse_date - -from openprocurement.chronograph import MANAGERS_MAPPING -from openprocurement.chronograph.scheduler import check_inner_auction, TZ -from openprocurement.chronograph.tests.data import plantest -from openprocurement.chronograph.design import sync_design - -dir_path = os.path.dirname(os.path.realpath(__file__)) - - -class SchedulerTest(unittest.TestCase): - - def setUp(self): - conf = ConfigParser() - path_to_config = os.path.join(dir_path, 'chronograph.ini') - if os.path.isfile(path_to_config): - conf.read(path_to_config) - settings = {k: v for k, v in conf.items('app:main')} - self.server = Server(settings['couchdb.url']) - self.settings = settings - self.db = self.server[settings['couchdb.db_name']] if \ - settings['couchdb.db_name'] in self.server else \ - self.server.create(settings['couchdb.db_name']) - sync_design(self.db) - plantest['_id'] = 'plantest_{}'.format( - datetime.now().date().isoformat()) - plantest_from_db = self.db.get(plantest['_id'], {}) - plantest_from_db.update(plantest) - self.db.save(plantest_from_db) - - def tearDown(self): - if hasattr(self, 'db'): - del self.server[self.settings['couchdb.db_name']] - - def test_check_inner_auction(self): - insider_auction_id = '01fa8a7dc4b8eac3b5820747efc6fe36' - texas_auction_id = 'dc3d950743304d05adaa1cd5b0475075' - classic_auction_with_lots = 'da8a28ed2bdf73ee1d373e4cadfed4c5' - classic_auction_without_lots = 'e51508cddc2c490005eaecb73c006b72' - lots_ids = ['1c2fb1e496b317b2b87e197e2332da77', - 'b10f9f7f26157ae2f349be8dc2106d6e'] - - today = datetime.now().date().isoformat() - time = '12:15:00' # actually, can be any time between 12:00:00 and 12:30:00 due to existing asserts - raw_time = ''.join([today, 'T', time]) - - # datetime.datetime object prepared in the way scheduler actually does it: - test_time = TZ.localize(parse_date(raw_time, None)).isoformat() - - auction = { - 'id': insider_auction_id, - 'procurementMethodType': 'dgfInsider', - 'auctionPeriod': { - 'startDate': test_time - } - } - mapper = { - 'pmts': { - 'dgfInsider': MANAGERS_MAPPING['insider'](), - 'landLease': MANAGERS_MAPPING['texas']() - }, - 'types': {'english': MANAGERS_MAPPING['english']()} - } - - plantest = self.db.get('plantest_{}'.format(today)) - - # Test insider - self.assertEqual(len(plantest.get('dutch_streams', [])), 15) - self.assertIn(insider_auction_id, plantest.get('dutch_streams')) - - check_inner_auction(self.db, auction, mapper) - plantest = self.db.get('plantest_{}'.format(today)) - self.assertEqual(len(plantest.get('dutch_streams', [])), 6) - self.assertNotIn(insider_auction_id, plantest.get('dutch_streams')) - - # Test texas - auction['id'] = texas_auction_id - auction['procurementMethodType'] = 'landLease' - - self.assertEqual(len(plantest.get('texas_streams', [])), 20) - self.assertIn(texas_auction_id, plantest.get('texas_streams')) - - check_inner_auction(self.db, auction, mapper) - plantest = self.db.get('plantest_{}'.format(today)) - self.assertEqual(len(plantest.get('texas_streams', [])), 15) - self.assertNotIn(texas_auction_id, plantest.get('texas_streams')) - - # Test classic with lots - auction['procurementMethodType'] = 'classic' - auction['id'] = classic_auction_with_lots - auction['lots'] = [ - { - 'id': lots_ids[0], - 'auctionPeriod': {'startDate': test_time} - }, - { - 'id': lots_ids[1], - 'auctionPeriod': {'startDate': test_time} - } - ] - self.assertEqual(len(plantest.get('stream_1')), 10) - self.assertEqual(len(plantest.get('stream_2')), 10) - stream_1_none_count = len( - [v for k, v in plantest.get('stream_1').items() if v is None]) - stream_2_none_count = len( - [v for k, v in plantest.get('stream_2').items() if v is None]) - self.assertEqual(stream_1_none_count, 0) - self.assertEqual(stream_2_none_count, 0) - check_inner_auction(self.db, auction, mapper) - plantest = self.db.get('plantest_{}'.format(today)) - self.assertEqual(len(plantest.get('stream_1')), 10) - self.assertEqual(len(plantest.get('stream_2')), 10) - stream_1_none_count = len( - [v for k, v in plantest.get('stream_1').items() if v is None]) - stream_2_none_count = len( - [v for k, v in plantest.get('stream_2').items() if v is None]) - self.assertEqual(stream_1_none_count, 3) - self.assertEqual(stream_2_none_count, 3) - self.assertNotIn(classic_auction_with_lots, - plantest.get('stream_1', {}).values()) - self.assertNotIn(classic_auction_with_lots, - plantest.get('stream_2', {}).values()) - - # Test classic without lots - del auction['lots'] - auction['id'] = classic_auction_without_lots - check_inner_auction(self.db, auction, mapper) - plantest = self.db.get('plantest_{}'.format(today)) - self.assertEqual(len(plantest.get('stream_1')), 10) - self.assertEqual(len(plantest.get('stream_2')), 10) - stream_1_none_count = len( - [v for k, v in plantest.get('stream_1').items() if v is None]) - stream_2_none_count = len( - [v for k, v in plantest.get('stream_2').items() if v is None]) - self.assertEqual(stream_1_none_count, 7) - self.assertEqual(stream_2_none_count, 6) - self.assertNotIn(classic_auction_without_lots, - plantest.get('stream_1', {}).values()) - self.assertNotIn(classic_auction_without_lots, - plantest.get('stream_2', {}).values()) - - -def suite(): - tests = unittest.TestSuite() - tests.addTest(unittest.makeSuite(SchedulerTest)) - return tests - - -if __name__ == '__main__': - unittest.main(defaultTest='suite', exit=False) diff --git a/openprocurement/chronograph/utils.py b/openprocurement/chronograph/utils.py index 98cf3fd..0ef685e 100644 --- a/openprocurement/chronograph/utils.py +++ b/openprocurement/chronograph/utils.py @@ -12,16 +12,8 @@ from openprocurement.chronograph.constants import ( - CALENDAR_ID, - STREAMS_ID, - WORKING_DAY_START, - ROUNDING, - MIN_PAUSE, - BIDDER_TIME, - SERVICE_TIME, SMOOTHING_MIN, SMOOTHING_MAX, - DEFAULT_STREAMS_DOC ) POOL = Pool(1) @@ -88,60 +80,6 @@ def randomize(dt): return dt + timedelta(seconds=randint(0, 1799)) -def get_calendar(db, calendar_id=CALENDAR_ID): - return db.get(calendar_id, {'_id': calendar_id}) - - -def set_holiday(db, day): - calendar = get_calendar(db) - key = parse_date(day).date().isoformat() - calendar[key] = True - db.save(calendar) - - -def delete_holiday(db, day): - calendar = get_calendar(db) - key = parse_date(day).date().isoformat() - if key in calendar: - calendar.pop(key) - db.save(calendar) - - -def get_streams(db, stream_key='streams', streams_id=STREAMS_ID): - """ - Backward compatibility version of managers.BaseAuctionsManager method, - which is left due to views.streams_view dependency - """ - streams = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) - return streams.get(stream_key, DEFAULT_STREAMS_DOC[stream_key]) - - -def set_streams(db, streams=None, stream_key=None, streams_id=STREAMS_ID): - streams_doc = db.get(streams_id, deepcopy(DEFAULT_STREAMS_DOC)) - if streams is not None and stream_key is not None: - streams_doc[stream_key] = streams - db.save(streams_doc) - - -def calc_auction_end_time(bids, start): - end = start + bids * BIDDER_TIME + SERVICE_TIME + MIN_PAUSE - seconds = (end - TZ.localize(datetime.combine(end, WORKING_DAY_START))).seconds - roundTo = ROUNDING.seconds - rounding = (seconds + roundTo - 1) // roundTo * roundTo - return (end + timedelta(0, rounding - seconds, -end.microsecond)).astimezone(TZ) - - -def find_free_slot(plan): - streams = plan.get('streams', 0) - for cur_stream in range(1, streams + 1): - stream_id = 'stream_{}'.format(cur_stream) - for slot in plan[stream_id]: - if plan[stream_id].get(slot) is None: - plan_date = parse_date(plan['_id'].split('_')[1] + 'T' + slot, None) - plan_date = plan_date.astimezone(TZ) if plan_date.tzinfo else TZ.localize(plan_date) - return plan_date, cur_stream - - def update_next_check_job(next_check, scheduler, auction_id, run_date, recheck_url, check_next_run_time=False): next_check = parse_date(next_check, TZ).astimezone(TZ) @@ -192,14 +130,3 @@ def get_request(url, auth, session, headers=None): sleep(tx) tx, ty = ty, tx + ty return r - - -def get_manager_for_auction(auction, mapper): - default_manager = mapper['types'].get('english', None) - - auction_type = auction.get('auctionParameters', {}).get('type', None) - if auction_type: - return mapper['types'].get(auction_type, default_manager) - else: - pmt = auction.get('procurementMethodType') - return mapper['pmts'].get(pmt, default_manager) diff --git a/openprocurement/chronograph/views.py b/openprocurement/chronograph/views.py index 28f996d..f2ec375 100644 --- a/openprocurement/chronograph/views.py +++ b/openprocurement/chronograph/views.py @@ -1,17 +1,10 @@ from pyramid.view import view_config -from openprocurement.chronograph.constants import STREAMS_KEYS from openprocurement.chronograph.scheduler import ( - get_calendar, recheck_auction, - resync_auction, resync_auctions, resync_auctions_back, ) -from openprocurement.chronograph.utils import ( - set_holiday, delete_holiday, - get_streams, set_streams -) @view_config(route_name='home', renderer='json') @@ -32,53 +25,6 @@ def resync_back(request): return resync_auctions_back(request) -@view_config(route_name='resync', renderer='json') -def resync(request): - return resync_auction(request) - - @view_config(route_name='recheck', renderer='json') def recheck(request): return recheck_auction(request) - - -@view_config(route_name='calendar', renderer='json') -def calendar_view(request): - calendar = get_calendar(request.registry.db) - return sorted([i for i in calendar if not i.startswith('_')]) - - -@view_config(route_name='calendar_entry', renderer='json') -def calendar_entry_view(request): - date = request.matchdict['date'] - if request.method == 'GET': - calendar = get_calendar(request.registry.db) - return calendar.get(date, False) - elif request.method == 'POST': - set_holiday(request.registry.db, date) - return True - elif request.method == 'DELETE': - delete_holiday(request.registry.db, date) - return False - - -@view_config(route_name='streams', renderer='json') -def streams_view(request): - if request.method == 'GET': - for stream_key in STREAMS_KEYS: - if request.params.get(stream_key, '') in [ - 'True', 'true', 'y', 'yes', 'Yes' - ]: - break - else: - stream_key = 'streams' - return get_streams(request.registry.db, stream_key) - elif request.method == 'POST': - result = False - for stream_key in STREAMS_KEYS: - streams = request.params.get(stream_key, '') - if streams and streams.isdigit(): - set_streams(request.registry.db, int(streams), stream_key) - result = True - return result - return False