From 1f80b16d3fc52462af5ea7281500ad9ed3deec11 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 9 Nov 2012 21:40:07 -0600 Subject: [PATCH 01/56] Updated TODO and testing commit --- .gitignore | 2 ++ TODO | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 TODO diff --git a/.gitignore b/.gitignore index 0d20b64..c1d844d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc +conf/config.py + diff --git a/TODO b/TODO new file mode 100644 index 0000000..ce33fc1 --- /dev/null +++ b/TODO @@ -0,0 +1,19 @@ +TODO File (in no particular order): + +Add Database interface in a modular way +- Utilize fifo queue's +- Allow sqlite database by default +- Allow mysql database + - Use bulk loading + +Don't re-invent the wheel.... use m0mchil's code when we can + - I didn't fork from there for a reason though Namely different db outlook + +Ensure that we can connect to bitcoind AND the it can provide getblocktemplate + +Write a Simplified How-to (Since I had a hard time) + +Allow auto-adding of usernames OR restrict +- Add a "script" to add,list,disable users + +Create a "service" (or whatever) that shows pool level stats. From 229a1185743877859ca8d422677426c14d654df1 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 9 Nov 2012 22:43:53 -0600 Subject: [PATCH 02/56] updated config file to handle new features --- conf/config_sample.py | 60 +++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/conf/config_sample.py b/conf/config_sample.py index 02c7d30..b0178e7 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -1,6 +1,6 @@ ''' This is example configuration for Stratum server. -Please rename it to settings.py and fill correct values. +Please rename it to config.py and fill correct values. ''' # ******************** GENERAL SETTINGS *************** @@ -12,7 +12,7 @@ LOGDIR = 'log/' # Main application log file. -LOGFILE = None#'stratum.log' +LOGFILE = None # eg. 'stratum.log' # Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL LOGLEVEL = 'INFO' @@ -20,7 +20,7 @@ # How many threads use for synchronous methods (services). # 30 is enough for small installation, for real usage # it should be slightly more, say 100-300. -THREAD_POOL_SIZE = 10 +THREAD_POOL_SIZE = 30 ENABLE_EXAMPLE_SERVICE = True @@ -58,17 +58,43 @@ IRC_NICK = None -''' -DATABASE_DRIVER = 'MySQLdb' -DATABASE_HOST = 'localhost' -DATABASE_DBNAME = 'pooldb' -DATABASE_USER = 'pooldb' -DATABASE_PASSWORD = '**empty**' -''' - -# Pool related settings -INSTANCE_ID = 31 -CENTRAL_WALLET = 'set_valid_addresss_in_config!' -PREVHASH_REFRESH_INTERVAL = 5 # in sec -MERKLE_REFRESH_INTERVAL = 60 # How often check memorypool -COINBASE_EXTRAS = '/stratum/' +# ******************** Database ********************* + +DATABASE_DRIVER = 'sqlite' # Options: sqlite, postgresql or mysql +# SQLite +DB_SQLITE_File = 'pooldb.sqlite' +# Postgresql +DB_PGSQL_HOST = 'localhost' +DB_PGSQL_DBNAME = 'pooldb' +DB_PGSQL_USER = 'pooldb' +DB_PGSQL_PASS = '**empty**' +DB_PGSQL_SCHEMA = 'public' +# MySQL +DB_MYSQL_HOST = 'localhost' +DB_MYSQL_DBNAME = 'pooldb' +DB_MYSQL_USER = 'pooldb' +DB_MYSQL_PASS = '**empty**' + +# ******************** Pool Settings ********************* + +# User Auth Options +USERS_AUTOADD = True # Automatically add users to db when they connect. + # This basically disables User Auth for the pool. +USERS_CHECK_PASSWORD = False # Check the workers password? (Many pools don't) + +# Transaction Settings +CENTRAL_WALLET = 'set_valid_addresss_in_config!' # local bitcoin address where money goes +COINBASE_EXTRAS = '/stratumPool/' # Extra Descriptive String to incorporate in solved blocks + +# Pool Target +POOL_TARGET = 1 # Pool-wide difficulty target int >= 1 + +# Bitcoind communication polling settings (In Seconds) +PREVHASH_REFRESH_INTERVAL = 5 # How often to check for new Blocks + # If using the blocknotify script (recommended) set = to MERKLE_REFRESH_INTERVAL + # (No reason to poll if we're getting pushed notifications) +MERKLE_REFRESH_INTERVAL = 60 # How often check memorypool + # This effectively resets the template and incorporates new transactions. + # This should be "slow" + +INSTANCE_ID = 31 # Not a clue what this is for... :P From ce58e8ef324f0ffedcb1e77354c593c277435f57 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 9 Nov 2012 22:46:02 -0600 Subject: [PATCH 03/56] Allow the changing of the pool difficulty target --- mining/subscription.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mining/subscription.py b/mining/subscription.py index d04c62e..41c77d0 100644 --- a/mining/subscription.py +++ b/mining/subscription.py @@ -1,6 +1,7 @@ from stratum.pubsub import Pubsub, Subscription from mining.interfaces import Interfaces +from stratum import settings import stratum.logger log = stratum.logger.get_logger('subscription') @@ -37,8 +38,7 @@ def _finish_after_subscribe(self, result): return result # Force set higher difficulty - # TODO - #self.connection_ref().rpc('mining.set_difficulty', [2,], is_notification=True) + self.connection_ref().rpc('mining.set_difficulty', [settings.POOL_TARGET,], is_notification=True) #self.connection_ref().rpc('client.get_version', []) # Force client to remove previous jobs if any (eg. from previous connection) @@ -51,4 +51,5 @@ def after_subscribe(self, *args): '''This will send new job to the client *after* he receive subscription details. on_finish callback solve the issue that job is broadcasted *during* the subscription request and client receive messages in wrong order.''' - self.connection_ref().on_finish.addCallback(self._finish_after_subscribe) \ No newline at end of file + self.connection_ref().on_finish.addCallback(self._finish_after_subscribe) + From 146d89b01203b23b99e0b3b0ccb21f8515dbac08 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 10 Nov 2012 18:23:13 -0600 Subject: [PATCH 04/56] Silence twisted.web and add Block check logging --- lib/bitcoin_rpc.py | 3 ++- lib/block_updater.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/bitcoin_rpc.py b/lib/bitcoin_rpc.py index 731e13a..fb8d33b 100644 --- a/lib/bitcoin_rpc.py +++ b/lib/bitcoin_rpc.py @@ -19,6 +19,7 @@ def __init__(self, host, port, username, password): 'Content-Type': 'text/json', 'Authorization': 'Basic %s' % self.credentials, } + client.HTTPClientFactory.noisy = False def _call_raw(self, data): return client.getPage( @@ -66,4 +67,4 @@ def prevhash(self): @defer.inlineCallbacks def validateaddress(self, address): resp = (yield self._call('validateaddress', [address,])) - defer.returnValue(json.loads(resp)['result']) \ No newline at end of file + defer.returnValue(json.loads(resp)['result']) diff --git a/lib/block_updater.py b/lib/block_updater.py index b80a024..4876c4d 100644 --- a/lib/block_updater.py +++ b/lib/block_updater.py @@ -44,7 +44,8 @@ def run(self): else: current_prevhash = None - prevhash = util.reverse_hash((yield self.bitcoin_rpc.prevhash())) + log.info("Checking for new block.") + prevhash = util.reverse_hash((yield self.bitcoin_rpc.prevhash())) if prevhash and prevhash != current_prevhash: log.info("New block! Prevhash: %s" % prevhash) update = True @@ -61,4 +62,4 @@ def run(self): finally: self.schedule() - \ No newline at end of file + From 91b951d921c7229bf477b9e5d06ee9710ba7f46f Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 10 Nov 2012 18:57:30 -0600 Subject: [PATCH 05/56] Update Documentation. --- INSTALL | 46 +++++++++++++++++++++++++++++++ README.md | 10 ++----- TODO | 2 ++ launcher_demo.tac => launcher.tac | 2 +- 4 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 INSTALL rename launcher_demo.tac => launcher.tac (94%) diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..501a3b7 --- /dev/null +++ b/INSTALL @@ -0,0 +1,46 @@ +Installation Instructions +========================= + +Step 0: Install bitcoind + It MUST be a recent version of bitcoind that support getblocktemplate + Set it up and start it! + Downloading the blockchain can take a few hours to a couple days! + +Step 1: Install the stratum core + git pull https://github.com/slush0/stratum.git + sudo easy_install stratum + (or if using alternate python: sudo /usr/local/bin/easy_install stratum) + +Step 2: Pull a copy of the miner + git pull https://github.com/generalfault/stratum-mining.git + +Step 3: Configure the Miner + cp conf/config_sample.py conf/config.py + make your changes to conf/config.py + Make sure you set: + CENTRAL_WALLET + BITCOIN_TRUSTED_USER + BITCOIN_TRUSTED_PASSWORD + +Step 4: Run the pool + twistd -ny launcher.tac -l - + OR - using alternate python + /usr/local/bin/twistd -ny launcher.tac -l - + +You can now set the URL on your stratum proxy (or miner that supports stratum) to: +http://YOURHOSTNAME:3333 + +Database Setup +========================= + +TODO: Once the DB code exists, Write this. + + +Problems???? +========================= + +Is your firewall off? +Is bitcoind running? + +TODO: are there other problems? + diff --git a/README.md b/README.md index 86b7603..92ac5e0 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,8 @@ stratum-mining ============== -Demo implementation of bitcoin mining pool using Stratum mining protocol. +Basic implementation of bitcoin mining pool using Stratum mining protocol. -For Stratum mining protocol specification, please visit http://mining.bitcoin.cz/stratum-mining. +For more info on Stratum: +http://mining.bitcoin.cz/stratum-mining. -Contact -------- - -This pool implementation is provided by http://mining.bitcoin.cz. You can contact -me by email info(at)bitcoin.cz or on IRC #stratum on freenode. diff --git a/TODO b/TODO index ce33fc1..5c50a8c 100644 --- a/TODO +++ b/TODO @@ -17,3 +17,5 @@ Allow auto-adding of usernames OR restrict - Add a "script" to add,list,disable users Create a "service" (or whatever) that shows pool level stats. + + diff --git a/launcher_demo.tac b/launcher.tac similarity index 94% rename from launcher_demo.tac rename to launcher.tac index 22e3799..06fb136 100644 --- a/launcher_demo.tac +++ b/launcher.tac @@ -1,4 +1,4 @@ -# Run me with "twistd -ny launcher_demo.tac -l -" +# Run me with "twistd -ny launcher.tac -l -" # Add conf directory to python path. # Configuration file is standard python module. From 9288615ca1f40f58cdb8610a93dc5bd4714e4e3b Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 10 Nov 2012 19:20:16 -0600 Subject: [PATCH 06/56] Added blocknotify instructions and generateAdminHash script --- INSTALL | 26 ++++++++++++++++++++++++++ conf/config_sample.py | 2 +- scripts/generateAdminHash.sh | 9 +++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100755 scripts/generateAdminHash.sh diff --git a/INSTALL b/INSTALL index 501a3b7..661fa85 100644 --- a/INSTALL +++ b/INSTALL @@ -30,6 +30,32 @@ Step 4: Run the pool You can now set the URL on your stratum proxy (or miner that supports stratum) to: http://YOURHOSTNAME:3333 +Bitcoind blocknotify Setup +========================= +Although scary (for me), this is actually pretty easy. + +Step 1: Set Admin Password + Ensure that you have set the ADMIN_PASSWORD_SHA256 parameter in conf/config.py + To make life easy you can run the generateAdminHash script to make the hash + ./scripts/generateAdminHash.sh + +Step 2: Test It + Restart the pool if it's already running + run ./scripts/blocknotify.sh --password --host localhost --port 3333 + Ensure everything is ok. + +Step 3: Run bitcoind with blocknotify + Stop bitcoind if it's already running + bitcoind stop + Wait till it ends + bitcoind -blocknotify="/absolute/path/to/scripts/blocknotify.sh --password --host localhost --port 3333" + +Step 4: Adjust pool polling + Now you should be able to watch the pools debug messages for awhile and see the blocknotify come in + once you are sure it's working edit conf/config.py and set + PREVHASH_REFRESH_INTERVAL = to the same value as MERKLE_REFRESH_INTERVAL + restart the pool + Database Setup ========================= diff --git a/conf/config_sample.py b/conf/config_sample.py index b0178e7..e96e423 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -51,7 +51,7 @@ BITCOIN_TRUSTED_USER = 'user' BITCOIN_TRUSTED_PASSWORD = 'somepassword' -# Use "echo -n '' | sha256sum | cut -f1 -d' ' " +# Use scripts/generateAdminHash.sh to generate the hash # for calculating SHA256 of your preferred password ADMIN_PASSWORD_SHA256 = None #ADMIN_PASSWORD_SHA256 = '9e6c0c1db1e0dfb3fa5159deb4ecd9715b3c8cd6b06bd4a3ad77e9a8c5694219' # SHA256 of the password diff --git a/scripts/generateAdminHash.sh b/scripts/generateAdminHash.sh new file mode 100755 index 0000000..a790673 --- /dev/null +++ b/scripts/generateAdminHash.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "x$1" == "x" ]; then + echo " Usage: $0 " + exit + fi + +echo -n "$1" | sha256sum | cut -f1 -d' ' + From 55ec0a8b2ccc87e701fb39ae87244714cfe88231 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 11 Nov 2012 09:17:44 -0600 Subject: [PATCH 07/56] Checking for Bitcoind connect and sanity.. --- TODO | 4 ---- mining/__init__.py | 44 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/TODO b/TODO index 5c50a8c..ccb6f6d 100644 --- a/TODO +++ b/TODO @@ -9,10 +9,6 @@ Add Database interface in a modular way Don't re-invent the wheel.... use m0mchil's code when we can - I didn't fork from there for a reason though Namely different db outlook -Ensure that we can connect to bitcoind AND the it can provide getblocktemplate - -Write a Simplified How-to (Since I had a hard time) - Allow auto-adding of usernames OR restrict - Add a "script" to add,list,disable users diff --git a/mining/__init__.py b/mining/__init__.py index 588bea4..a683246 100644 --- a/mining/__init__.py +++ b/mining/__init__.py @@ -1,6 +1,9 @@ from service import MiningService from subscription import MiningSubscription from twisted.internet import defer +import time +import simplejson as json +from twisted.internet import reactor @defer.inlineCallbacks def setup(on_startup): @@ -11,8 +14,12 @@ def setup(on_startup): *before* you call setup() in the launcher script.''' from stratum import settings - from interfaces import Interfaces - + + # Get logging online as soon as possible + import stratum.logger + log = stratum.logger.get_logger('mining') + + from interfaces import Interfaces # Let's wait until share manager and worker manager boot up (yield Interfaces.share_manager.on_load) (yield Interfaces.worker_manager.on_load) @@ -28,6 +35,34 @@ def setup(on_startup): settings.BITCOIN_TRUSTED_USER, settings.BITCOIN_TRUSTED_PASSWORD) + # Check bitcoind + # Check we can connect (sleep) + # Check the results: + # - getblocktemplate is avalible (Die if not) + # - we are not still downloading the blockchain (Sleep) + log.info("Connecting to bitcoind...") + while True: + try: + result = (yield bitcoin_rpc.getblocktemplate()) + if isinstance(result, dict): + if result['version'] == 2: + break + except Exception, e: + if isinstance(e[2], str): + if isinstance(json.loads(e[2])['error']['message'],str): + error = json.loads(e[2])['error']['message'] + if error == "Method not found": + log.error("Bitcoind does not support getblocktemplate!!! (time to upgrade.)") + reactor.stop() + elif error == "Bitcoin is downloading blocks...": + log.error("Bitcoind downloading blockchain... will check back in 30 sec") + time.sleep(29) + else: + log.error("Bitcoind Error: %s", error) + time.sleep(1) # If we didn't get a result or the connect failed + log.info('Connected to bitcoind - Ready to GO!') + + # Start the coinbaser coinbaser = SimpleCoinbaser(bitcoin_rpc, settings.CENTRAL_WALLET) (yield coinbaser.on_load) @@ -47,7 +82,6 @@ def setup(on_startup): # mechanism is not working properly BlockUpdater(registry, bitcoin_rpc) - import stratum.logger - log = stratum.logger.get_logger('mining') log.info("MINING SERVICE IS READY") - on_startup.callback(True) + on_startup.callback(True) + From 5638eef6c7d40eec2aaa61c35ac112ea65a02c8d Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 16 Nov 2012 09:57:48 -0600 Subject: [PATCH 08/56] Initial Database Code --- .gitignore | 4 +- INSTALL | 31 ++++++++- TODO | 12 +--- conf/config_sample.py | 17 ++++- lib/template_registry.py | 8 +-- lib/util.py | 13 +++- mining/DBInterface.py | 132 +++++++++++++++++++++++++++++++++++++++ mining/DB_Mysql.py | 122 ++++++++++++++++++++++++++++++++++++ mining/DB_None.py | 32 ++++++++++ mining/DB_Postgresql.py | 124 ++++++++++++++++++++++++++++++++++++ mining/DB_Sqlite.py | 94 ++++++++++++++++++++++++++++ mining/interfaces.py | 26 +++++--- mining/service.py | 14 +++-- 13 files changed, 596 insertions(+), 33 deletions(-) create mode 100644 mining/DBInterface.py create mode 100644 mining/DB_Mysql.py create mode 100644 mining/DB_None.py create mode 100644 mining/DB_Postgresql.py create mode 100644 mining/DB_Sqlite.py diff --git a/.gitignore b/.gitignore index c1d844d..39fc08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc conf/config.py - +NOTES +run +pooldb.sqlite diff --git a/INSTALL b/INSTALL index 661fa85..fd7224c 100644 --- a/INSTALL +++ b/INSTALL @@ -58,9 +58,34 @@ Step 4: Adjust pool polling Database Setup ========================= - -TODO: Once the DB code exists, Write this. - +Table Creation: Tables are auto-created if they don't exist + +None: +Well, this doesn't do anything, so there is nothing to set up + +Sqlite: +Just set the file path in the config file. +Support for sqlite3 is built into recent python versions + +Postgresql: +1: Set up your parameters in the config file. +2: Install the postgresql libraries in your os: + Redhat and the like: + yum install postgresql-libs postgresql-devel + Ubuntu and the like: + apt-get install postgresql postgresql-devel +3: Install the python bindings + easy_install psycopg2 + +Mysql: +1: Set up your parameters in the config file. +2: Install the postgresql libraries in your os: + Redhat and the like: + yum install mysql mysql-devel + Ubuntu and the like: + apt-get install mysql mysql-devel +3: Install the python bindings + easy_install mysql-python Problems???? ========================= diff --git a/TODO b/TODO index ccb6f6d..ecf4b39 100644 --- a/TODO +++ b/TODO @@ -1,17 +1,11 @@ TODO File (in no particular order): -Add Database interface in a modular way -- Utilize fifo queue's -- Allow sqlite database by default -- Allow mysql database - - Use bulk loading +SQL Connection pooling: sqlalchemy -Don't re-invent the wheel.... use m0mchil's code when we can - - I didn't fork from there for a reason though Namely different db outlook - -Allow auto-adding of usernames OR restrict - Add a "script" to add,list,disable users Create a "service" (or whatever) that shows pool level stats. +implement a basic share limiter that increases difficulty until share time + is X seconds between shares diff --git a/conf/config_sample.py b/conf/config_sample.py index e96e423..ea6983c 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -60,9 +60,11 @@ # ******************** Database ********************* -DATABASE_DRIVER = 'sqlite' # Options: sqlite, postgresql or mysql +DATABASE_DRIVER = 'sqlite' # Options: none, sqlite, postgresql or mysql +DATABASE_EXTEND = True # False = pushpool db layout, True = pushpool + extra columns + # SQLite -DB_SQLITE_File = 'pooldb.sqlite' +DB_SQLITE_FILE = 'pooldb.sqlite' # Postgresql DB_PGSQL_HOST = 'localhost' DB_PGSQL_DBNAME = 'pooldb' @@ -75,6 +77,17 @@ DB_MYSQL_USER = 'pooldb' DB_MYSQL_PASS = '**empty**' +# ******************** Adv. DB Settings ********************* +# Don't change these unless you know what you are doing + +DB_LOADER_CHECKTIME = 15 # How often we check to see if we should run the loader +DB_LOADER_REC_MIN = 10 # Min Records before the bulk loader fires +DB_LOADER_REC_MAX = 50 # Max Records the bulk loader will commit at a time + +DB_STATS_AVG_TIME = 300 # When using the DATABASE_EXTEND option, average speed over X sec + # Note: this is also how often it updates +DB_USERCACHE_TIME = 600 # How long the usercache is good for before we refresh + # ******************** Pool Settings ********************* # User Auth Options diff --git a/lib/template_registry.py b/lib/template_registry.py index 435aeb4..7e3e33f 100644 --- a/lib/template_registry.py +++ b/lib/template_registry.py @@ -61,7 +61,7 @@ def get_last_broadcast_args(self): from last known template.''' return self.last_block.broadcast_args - def add_template(self, block): + def add_template(self, block,block_height): '''Adds new template to the registry. It also clean up templates which should not be used anymore.''' @@ -94,7 +94,7 @@ def add_template(self, block): if new_block: # Tell the system about new block # It is mostly important for share manager - self.on_block_callback(prevhash) + self.on_block_callback(prevhash, block_height) # Everything is ready, let's broadcast jobs! self.on_template_callback(new_block) @@ -127,7 +127,7 @@ def _update_block(self, data): template = self.block_template_class(Interfaces.timestamper, self.coinbaser, JobIdGenerator.get_new_id()) template.fill_from_rpc(data) - self.add_template(template) + self.add_template(template,data['height']) log.info("Update finished, %.03f sec, %d txes" % \ (Interfaces.timestamper.time() - start, len(template.vtx))) @@ -251,4 +251,4 @@ def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, return (header_hex, block_hash_hex, on_submit) - return (header_hex, block_hash_hex, None) \ No newline at end of file + return (header_hex, block_hash_hex, None) diff --git a/lib/util.py b/lib/util.py index 3e88fd8..d687875 100644 --- a/lib/util.py +++ b/lib/util.py @@ -142,6 +142,17 @@ def b58decode(v, length): return result +def b58encode(value): + """ encode integer 'value' as a base58 string; returns string + """ + encoded = '' + while value >= __b58base: + div, mod = divmod(value, __b58base) + encoded = __b58chars[mod] + encoded # add to left + value = div + encoded = __b58chars[value] + encoded # most significant remainder + return encoded + def reverse_hash(h): # This only revert byte order, nothing more if len(h) != 64: @@ -198,4 +209,4 @@ def script_to_address(addr): if not d: raise ValueError('invalid address') (ver, pubkeyhash) = d - return b'\x76\xa9\x14' + pubkeyhash + b'\x88\xac' \ No newline at end of file + return b'\x76\xa9\x14' + pubkeyhash + b'\x88\xac' diff --git a/mining/DBInterface.py b/mining/DBInterface.py new file mode 100644 index 0000000..fca494c --- /dev/null +++ b/mining/DBInterface.py @@ -0,0 +1,132 @@ +from twisted.internet import reactor, defer +import time +import Queue + +from stratum import settings + +import stratum.logger +log = stratum.logger.get_logger('DBInterface') + +class DBInterface(): + def __init__(self): + self.q = Queue.Queue() + self.queueclock = None + + self.usercache = {} + self.clearusercache() + + self.nextStatsUpdate = 0 + + self.dbi = self.connectDB() + + self.scheduleImport() + + def connectDB(self): + # Choose our database driver and put it in self.dbi + if settings.DATABASE_DRIVER == "sqlite": + log.debug('DB_Sqlite INIT') + import DB_Sqlite + return DB_Sqlite.DB_Sqlite() + elif settings.DATABASE_DRIVER == "mysql": + log.debug('DB_Mysql INIT') + import DB_Mysql + return DB_Mysql.DB_Mysql() + elif settings.DATABASE_DRIVER == "postgresql": + log.debug('DB_Postgresql INIT') + import DB_Postgresql + return DB_Postgresql.DB_Postgresql() + elif settings.DATABASE_DRIVER == "none": + log.debug('DB_None INIT') + import DB_None + return DB_None.DB_None() + else: + log.error('Invalid DATABASE_DRIVER -- using NONE') + log.debug('DB_None INIT') + import DB_None + return DB_None.DB_None() + + def clearusercache(self): + self.usercache = {} + self.usercacheclock = reactor.callLater( settings.DB_USERCACHE_TIME , self.clearusercache) + + def scheduleImport(self): + # This schedule's the Import + # If you don't want to use threads change + # self.run_import_thread to self.run_import + self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import_thread) + + def run_import_thread(self): + if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run + reactor.callInThread(self.import_thread) + self.scheduleImport() + + def run_import(self): + self.do_import(self.dbi) + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + dbi.updateStats(settings.DB_STATS_AVG_TIME) + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + self.scheduleImport() + + def import_thread(self): + # Here we are in the thread. + dbi = self.connectDB() + self.do_import(dbi) + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + dbi.updateStats(settings.DB_STATS_AVG_TIME) + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + + def do_import(self,dbi): + # Only run if we have data + while self.q.qsize() >= settings.DB_LOADER_REC_MIN: + # Put together the data we want to import + sqldata = [] + datacnt = 0 + while self.q.empty() == False and datacnt < settings.DB_LOADER_REC_MAX : + datacnt += 1 + data = self.q.get() + sqldata.append(data) + self.q.task_done() + # try to do the import, if we fail, log the error and put the data back in the queue + try: + log.info("Inserting %s Share Records",datacnt) + dbi.import_shares(sqldata) + except Exception as e: + log.error("Insert Share Records Failed: %s", e.args[0]) + for k,v in enumerate(sqldata): + self.q.put(v) + break # Allows us to sleep a little + + def queue_share(self,data): + self.q.put( data ) + + def found_block(self,data): + try: + log.info("Updating Found Block Share Record") + self.dbi.found_block(data) + except Exception as e: + log.error("Update Found Block Share Record Failed: %s", e.args[0]) + + def check_password(self,username,password): + wid = username+":-:"+password + if wid in self.usercache : + return True + elif self.dbi.check_password(username,password) : + self.usercache[wid] = 1 + return True + elif settings.USERS_AUTOADD == True : + self.insert_user(username,password) + self.usercache[wid] = 1 + return True + return False + + def insert_user(self,username,password): + return self.dbi.insert_user(username,password) + + def delete_user(self,username): + self.usercache = {} + return self.dbi.delete_user(username) + + def update_user(self,username,password): + self.usercache = {} + return self.dbi.update_user(username,password) + diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py new file mode 100644 index 0000000..fbe8843 --- /dev/null +++ b/mining/DB_Mysql.py @@ -0,0 +1,122 @@ +import time +from stratum import settings +import stratum.logger +log = stratum.logger.get_logger('DB_Mysql') + +import MySQLdb + +class DB_Mysql(): + def __init__(self): + log.debug("Connecting to DB") + self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST,settings.DB_MYSQL_USER,settings.DB_MYSQL_PASS,settings.DB_MYSQL_DBNAME) + self.dbc = self.dbh.cursor() + + self.check_tables() + + def updateStats(self,averageOverTime): + log.debug("Updating Stats") + stime = '%.2f' % ( time.time() - averageOverTime ); +# self.dbc.execute("select username,SUM(difficulty) from shares where time > %(time)s group by username", {"time": (stime,)}) + self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) + for name,shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbh.commit() + + def import_shares(self,data): + log.debug("Importing Shares") +# 0 1 2 3 4 5 6 7 8 9 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] + checkin_times = {} + for k,v in enumerate(data): + if settings.DATABASE_EXTEND : + if v[0] in checkin_times: + if v[4] > checkin_times[v[0]] : + checkin_times[v[0]] = v[4] + else: + checkin_times[v[0]] = v[4] + + self.dbc.execute("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + else : + self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ + "(FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'') ) + if settings.DATABASE_EXTEND : + for k,v in checkin_times.items(): + self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s) where username = %s",(v,k)) + + self.dbh.commit() + + + def found_block(self,data): + # Note: difficulty = -1 here + self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", + (data[5],data[2],data[4],data[0])) + self.dbh.commit() + + def delete_user(self,username): + log.debug("Deleting Username") + self.dbc.execute("delete from pool_worker where username = %s", + (username )) + self.dbh.commit() + + def insert_user(self,username,password): + log.debug("Adding Username/Password") + self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", + (username, password )) + self.dbh.commit() + + def update_user(self,username,password): + log.debug("Updating Username/Password") + self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", + (username, password )) + self.dbh.commit() + + def check_password(self,username,password): + log.debug("Checking Username/Password") + self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", + (username, password )) + data = self.dbc.fetchone() + if data[0] > 0 : + return True + return False + + def check_tables(self): + log.debug("Checking Tables") + + shares_exist = False + self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", + {"schema": settings.DB_MYSQL_DBNAME }) + data = self.dbc.fetchone() + if data[0] > 0 : + shares_exist = True + + pool_worker_exist = False + self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS where table_schema = %(schema)s and table_name = 'pool_worker' and index_name = 'pool_worker_username'", + {"schema": settings.DB_MYSQL_DBNAME }) + data = self.dbc.fetchone() + if data[0] > 0 : + pool_worker_exist = True + + if settings.DATABASE_EXTEND == True : + self.dbc.execute("create table if not exists shares " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + if shares_exist == False: + self.dbc.execute("create index shares_username ON shares(username(10))") + self.dbc.execute("create table if not exists pool_worker(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp) ENGINE = MYISAM") + if pool_worker_exist == False: + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + else : + self.dbc.execute("create table if not exists shares" + \ + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") + if shares_exist == False: + self.dbc.execute("create index shares_username ON shares(username(10))") + self.dbc.execute("create table if not exists pool_worker(id serial,username TEXT, password TEXT) ENGINE = MYISAM") + if pool_worker_exist == False: + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + self.dbh.commit() + diff --git a/mining/DB_None.py b/mining/DB_None.py new file mode 100644 index 0000000..59374e0 --- /dev/null +++ b/mining/DB_None.py @@ -0,0 +1,32 @@ +import stratum.logger +log = stratum.logger.get_logger('None') + +class DB_None(): + def __init__(self): + log.debug("Connecting to DB") + + def updateStats(self,averageOverTime): + log.debug("Updating Stats") + + def import_shares(self,data): + log.debug("Importing Shares") + + def found_block(self,data): + log.debug("Found Block") + + def delete_user(self,username): + log.debug("Deleting Username") + + def insert_user(self,username,password): + log.debug("Adding Username/Password") + + def update_user(self,username,password): + log.debug("Updating Username/Password") + + def check_password(self,username,password): + log.debug("Checking Username/Password") + return True + + def check_tables(self): + log.debug("Checking Tables") + diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py new file mode 100644 index 0000000..18a1975 --- /dev/null +++ b/mining/DB_Postgresql.py @@ -0,0 +1,124 @@ +import time +from stratum import settings +import stratum.logger +log = stratum.logger.get_logger('DB_Postgresql') + +import psycopg2 + +class DB_Postgresql(): + def __init__(self): + log.debug("Connecting to DB") + self.dbh = psycopg2.connect("host='"+settings.DB_PGSQL_HOST+"' dbname='"+settings.DB_PGSQL_DBNAME+"' user='"+settings.DB_PGSQL_USER+\ + "' password='"+settings.DB_PGSQL_PASS+"'") + # TODO -- set the schema + self.dbc = self.dbh.cursor() + + self.check_tables() + + def updateStats(self,averageOverTime): + log.debug("Updating Stats") + stime = '%.2f' % ( time.time() - averageOverTime ); +# self.dbc.execute("select username,SUM(difficulty) from shares where time > %(time)s group by username", {"time": (stime,)}) + self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", (stime,)) + for name,shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbh.commit() + + def import_shares(self,data): + log.debug("Importing Shares") +# 0 1 2 3 4 5 6 7 8 9 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] + checkin_times = {} + for k,v in enumerate(data): + if settings.DATABASE_EXTEND : + if v[0] in checkin_times: + if v[4] > checkin_times[v[0]] : + checkin_times[v[0]] = v[4] + else: + checkin_times[v[0]] = v[4] + + self.dbc.execute("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + else : + self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ + "(to_timestamp(%s),%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'') ) + if settings.DATABASE_EXTEND : + for k,v in checkin_times.items(): + self.dbc.execute("update pool_worker set last_checkin = to_timestamp(%s) where username = %s",(v,k)) + + self.dbh.commit() + + + def found_block(self,data): + # Note: difficulty = -1 here + self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", + (data[5],data[2],data[4],data[0])) + self.dbh.commit() + + def delete_user(self,username): + log.debug("Deleting Username") + self.dbc.execute("delete from pool_worker where username = %s", + (username )) + self.dbh.commit() + + def insert_user(self,username,password): + log.debug("Adding Username/Password") + self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", + (username, password )) + self.dbh.commit() + + def update_user(self,username,password): + log.debug("Updating Username/Password") + self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", + (username, password )) + self.dbh.commit() + + def check_password(self,username,password): + log.debug("Checking Username/Password") + self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", + (username, password )) + data = self.dbc.fetchone() + if data[0] > 0 : + return True + return False + + def check_tables(self): + log.debug("Checking Tables") + + shares_exist = False + self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'shares'", + {"schema": settings.DB_PGSQL_SCHEMA }) + data = self.dbc.fetchone() + if data[0] > 0 : + shares_exist = True + + pool_worker_exist = False + self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'pool_worker'", + {"schema": settings.DB_PGSQL_SCHEMA }) + data = self.dbc.fetchone() + if data[0] > 0 : + pool_worker_exist = True + + if settings.DATABASE_EXTEND == True : + if shares_exist == False: + self.dbc.execute("create table shares" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create index shares_username ON shares(username)") + if pool_worker_exist == False: + self.dbc.execute("create table pool_worker(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + else : + if shares_exist == False: + self.dbc.execute("create table shares" + \ + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT)") + self.dbc.execute("create index shares_username ON shares(username)") + if pool_worker_exist == False: + self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + self.dbh.commit() + diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py new file mode 100644 index 0000000..90bd009 --- /dev/null +++ b/mining/DB_Sqlite.py @@ -0,0 +1,94 @@ +import time +from stratum import settings +import stratum.logger +log = stratum.logger.get_logger('DB_Sqlite') + +import sqlite3 + +class DB_Sqlite(): + def __init__(self): + log.debug("Connecting to DB") + self.dbh = sqlite3.connect(settings.DB_SQLITE_FILE) + self.dbc = self.dbh.cursor() + + self.check_tables() + + def updateStats(self,averageOverTime): + log.debug("Updating Stats") + stime = '%.2f' % ( time.time() - averageOverTime ); + self.dbc.execute("select username,SUM(difficulty) from shares where time > ? group by username", (stime,)) + for name,shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + self.dbc.execute("update pool_worker set speed = ? where username = ?",(speed,name)) + self.dbh.commit() + + def import_shares(self,data): + log.debug("Importing Shares") +# 0 1 2 3 4 5 6 7 8 9 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] + checkin_times = {} + for k,v in enumerate(data): + if settings.DATABASE_EXTEND : + if v[0] in checkin_times: + if v[4] > checkin_times[v[0]] : + checkin_times[v[0]] = v[4] + else: + checkin_times[v[0]] = v[4] + + self.dbc.execute("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + else : + self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES (?,?,?,?,?,?,?)", + (v[4],v[6],v[0],v[5],0,v[9],'') ) + if settings.DATABASE_EXTEND : + for k,v in checkin_times.items(): + self.dbc.execute("update pool_worker set last_checkin = ? where username = ?",(v,k)) + + self.dbh.commit() + + + def found_block(self,data): + # Note: difficulty = -1 here + self.dbc.execute("update shares set upstream_result = ?, solution = ? where time = ? and username = ?", + (data[5],data[2],data[4],data[0])) + self.dbh.commit() + + def delete_user(self,username): + log.debug("Deleting Username") + self.dbc.execute("delete from pool_worker where username = ?", (username)) + self.dbh.commit() + + def insert_user(self,username,password): + log.debug("Adding Username/Password") + self.dbc.execute("insert into pool_worker (username,password) VALUES (?,?)", (username,password)) + self.dbh.commit() + + def update_user(self,username,password): + log.debug("Updating Username/Password") + self.dbc.execute("update pool_worker set password = ? where username = ?", (password,username)) + self.dbh.commit() + + def check_password(self,username,password): + log.debug("Checking Username/Password") + self.dbc.execute("select COUNT(*) from pool_worker where username = ? and password = ?",(username,password)) + data = self.dbc.fetchone() + if data[0] > 0 : + return True + return False + + def check_tables(self): + log.debug("Checking Tables") + if settings.DATABASE_EXTEND == True : + self.dbc.execute("create table if not exists shares" +\ + "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create table if not exists pool_worker(username TEXT, password TEXT, speed INTEGER, last_checkin DATETIME)") + else : + self.dbc.execute("create table if not exists shares" + \ + "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT)") + self.dbc.execute("create table if not exists pool_worker(username TEXT, password TEXT)") + self.dbc.execute("create index if not exists shares_username ON shares(username)") + self.dbc.execute("create index if not exists pool_worker_username ON pool_worker(username)") + diff --git a/mining/interfaces.py b/mining/interfaces.py index 1fef21a..f89c6d2 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -3,13 +3,16 @@ and customize references to interface instances in your launcher. (see launcher_demo.tac for an example). ''' - import time from twisted.internet import reactor, defer +from lib.util import b58encode import stratum.logger log = stratum.logger.get_logger('interfaces') +import DBInterface +dbi = DBInterface.DBInterface() + class WorkerManagerInterface(object): def __init__(self): # Fire deferred when manager is ready @@ -17,7 +20,9 @@ def __init__(self): self.on_load.callback(True) def authorize(self, worker_name, worker_password): - return True + # Important NOTE: This is called on EVERY submitted share. So you'll need caching!!! + return dbi.check_password(worker_name,worker_password) + class ShareLimiterInterface(object): '''Implement difficulty adjustments here''' @@ -29,6 +34,7 @@ def submit(self, connection_ref, current_difficulty, timestamp): - raise SubmitException for stop processing this request - call mining.set_difficulty on connection to adjust the difficulty''' + pass class ShareManagerInterface(object): @@ -36,16 +42,22 @@ def __init__(self): # Fire deferred when manager is ready self.on_load = defer.Deferred() self.on_load.callback(True) + self.block_height = 0 + self.prev_hash = 0 - def on_network_block(self, prevhash): + def on_network_block(self, prevhash, block_height): '''Prints when there's new block coming from the network (possibly new round)''' + self.block_height = block_height + self.prev_hash = b58encode(int(prevhash,16)) pass - def on_submit_share(self, worker_name, block_header, block_hash, shares, timestamp, is_valid): + def on_submit_share(self, worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, invalid_reason ): log.info("%s %s %s" % (block_hash, 'valid' if is_valid else 'INVALID', worker_name)) - - def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp): + dbi.queue_share([worker_name,block_header,block_hash,difficulty,timestamp,is_valid, ip, self.block_height, self.prev_hash, invalid_reason ]) + + def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip ): log.info("Block %s %s" % (block_hash, 'ACCEPTED' if is_accepted else 'REJECTED')) + dbi.found_block([worker_name,block_header,block_hash,-1,timestamp,is_accepted,ip,self.block_height, self.prev_hash]) class TimestamperInterface(object): '''This is the only source for current time in the application. @@ -87,4 +99,4 @@ def set_timestamper(cls, manager): @classmethod def set_template_registry(cls, registry): - cls.template_registry = registry \ No newline at end of file + cls.template_registry = registry diff --git a/mining/service.py b/mining/service.py index ff86d96..7642ef0 100644 --- a/mining/service.py +++ b/mining/service.py @@ -1,6 +1,7 @@ import binascii from twisted.internet import defer +from stratum import settings from stratum.services import GenericService, admin from stratum.pubsub import Pubsub from interfaces import Interfaces @@ -56,7 +57,7 @@ def subscribe(self): session = self.connection_ref().get_session() session['extranonce1'] = extranonce1 - session['difficulty'] = 1 # Following protocol specs, default diff is 1 + session['difficulty'] = settings.POOL_TARGET # Following protocol specs, default diff is 1 return Pubsub.subscribe(self.connection_ref(), MiningSubscription()) + (extranonce1_hex, extranonce2_size) @@ -93,6 +94,7 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): difficulty = session['difficulty'] submit_time = Interfaces.timestamper.time() + ip = self.connection_ref()._get_ip() Interfaces.share_limiter.submit(self.connection_ref, difficulty, submit_time) @@ -101,21 +103,21 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): try: (block_header, block_hash, on_submit) = Interfaces.template_registry.submit_share(job_id, worker_name, extranonce1_bin, extranonce2, ntime, nonce, difficulty) - except SubmitException: + except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, - submit_time, False) + submit_time, False, ip, e[0]) raise Interfaces.share_manager.on_submit_share(worker_name, block_header, block_hash, difficulty, - submit_time, True) + submit_time, True, ip, '') if on_submit != None: # Pool performs submitblock() to bitcoind. Let's hook # to result and report it to share manager on_submit.addCallback(Interfaces.share_manager.on_submit_block, - worker_name, block_header, block_hash, submit_time) + worker_name, block_header, block_hash, submit_time,ip) return True @@ -138,4 +140,4 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): ('extranonce2', 'string', 'hex-encoded big-endian extranonce2, length depends on extranonce2_size from mining.notify.'), ('ntime', 'string', 'UNIX timestamp (32bit integer, big-endian, hex-encoded), must be >= ntime provided by mining,notify and <= current time'), ('nonce', 'string', '32bit integer, hex-encoded, big-endian'),] - \ No newline at end of file + From 3107aad9674566fb68d3492b2b37dea72d8927a1 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 16 Nov 2012 10:01:10 -0600 Subject: [PATCH 09/56] Doc update --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 92ac5e0..4a1be19 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,16 @@ stratum-mining Basic implementation of bitcoin mining pool using Stratum mining protocol. +This fork includes a database implementation for: + None + Sqlite + Mysql + Postgresql + +Basic worker stats are provided (and updated) + +See the INSTALL file for install instructions. + For more info on Stratum: http://mining.bitcoin.cz/stratum-mining. From 247158f33b7c5213bef45e6ffc89d2d25692be17 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Mon, 19 Nov 2012 10:33:08 -0600 Subject: [PATCH 10/56] Reset speed to 0 on inactive miners --- mining/DB_Mysql.py | 3 ++- mining/DB_Postgresql.py | 3 ++- mining/DB_Sqlite.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index fbe8843..2a70d60 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -15,8 +15,9 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") + # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. + self.dbc.execute("update pool_worker set speed = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); -# self.dbc.execute("select username,SUM(difficulty) from shares where time > %(time)s group by username", {"time": (stime,)}) self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 18a1975..a217f74 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -17,8 +17,9 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") + # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. + self.dbc.execute("update pool_worker set speed = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); -# self.dbc.execute("select username,SUM(difficulty) from shares where time > %(time)s group by username", {"time": (stime,)}) self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", (stime,)) for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 90bd009..33aace3 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -15,6 +15,8 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") + # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. + self.dbc.execute("update pool_worker set speed = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); self.dbc.execute("select username,SUM(difficulty) from shares where time > ? group by username", (stime,)) for name,shares in self.dbc.fetchall(): From 24b117a4c0671bd902e905fc621688b9d7dce7f0 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Mon, 3 Dec 2012 20:58:44 -0600 Subject: [PATCH 11/56] Fixed speed not resetting to 0, Added a bunch of stats to the database, Added basic stats page --- lib/basic_stats.py | 134 ++++++++++++++++++++++++++++++++++++++++ statics/basic_stats.css | 4 ++ statics/favicon.ico | Bin 0 -> 2550 bytes 3 files changed, 138 insertions(+) create mode 100644 lib/basic_stats.py create mode 100644 statics/basic_stats.css create mode 100644 statics/favicon.ico diff --git a/lib/basic_stats.py b/lib/basic_stats.py new file mode 100644 index 0000000..dfe0f19 --- /dev/null +++ b/lib/basic_stats.py @@ -0,0 +1,134 @@ +from twisted.internet import reactor +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.web import static + +import time +from datetime import timedelta +import json +from stratum import settings + +import stratum.logger +log = stratum.logger.get_logger('Basic Stats') + +import mining.DBInterface +dbi = mining.DBInterface.DBInterface() + +import locale +locale.setlocale(locale.LC_ALL, '') + +class StatsPage(Resource): + isLeaf = False + cache_html = "" + cache_json = "" + last_update = 0 + + def getChild(self, name, request): + if name == '' or name == 'stats': + return self + return Resource.getChild(self, name, request) + + def render_GET(self, request): + # Cache the results for 30 seconds, takes care of high volume/ddos + if time.time() - self.last_update < 30 : + if(request.path == "/stats"): + return self.cache_json + return self.cache_html + + # Get info + pool_stats = dbi.get_pool_stats() + workers_stats = dbi.get_workers_stats() + + # Send in JSON? + self.cache_json = json.dumps({"pool":pool_stats,"workers":workers_stats}) + + #Otherwise build webpage + r= "" + r+= "" + r+= "" + r+="" + r+="

Stratum Mining Server

" + r+="" + + pool_color="#A00" + if int(pool_stats['pool_speed']) > 100 and float(pool_stats['round_progress']) < 200: + pool_color="#AA0" + if int(pool_stats['pool_speed']) > 100 and float(pool_stats['round_progress']) < 150: + pool_color="#0A0" + r+="" + + bitcoin_color="#A00" + if int(pool_stats['bitcoin_connections']) > 0: + bitcoin_color="yellow" + if int(pool_stats['bitcoin_connections']) > 10: + bitcoin_color="#0A0" + r+="
Pool Stats:" + r+="Speed: " + pool_stats['pool_speed'] + "Mhs" + r+="Round Shares: " + format(int(pool_stats['round_shares']),"n") + " (%.2f" % float(pool_stats['round_progress']) + "%)" + r+="Round Duration: " + str(timedelta(0,(time.time() - float(pool_stats['round_start'])))).split(".")[0] + r+="Total Found: " + format(int(pool_stats['pool_total_found']),"n") + r+="
Bitcoin Stats:" + r+="Connections :"+pool_stats['bitcoin_connections'] + r+="Difficulty :"+format(int(float(pool_stats['bitcoin_difficulty'])),"n") + r+="Height :"+format(int(pool_stats['bitcoin_blocks']),"n") + r+="Balance :"+pool_stats['bitcoin_balance'] + r+="
" + + r+="
" + + r+="" + size = len(workers_stats) + if size <= 10: + r+="" + else : + colcnt = int(size/3) + 1 + colt = colcnt + r+="" + r+="
" + r+="" + r+="" + for (w, wi) in enumerate(workers_stats): + wd = workers_stats[wi] + wc = "#A00" + if wd["speed"] > 0: + wc = "yellow" + if wd["speed"] > 100: + wc = "#0A0" + r+=""%( + wc,wi,format(int(wd["speed"]),"n"),format(int(wd["total_shares"]),"n"),format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + r+="
WorkerSpeedShares/RejFound
%s%s%s/%s%s
" + r+="" + r+="" + for (w, wd) in enumerate(workers_stats): + wc = "#A00" + if wd["speed"] > 0: + wc = "yellow" + if wd["speed"] > 100: + wc = "#0A0" + r+=""%( + wc,wi,format(int(wd["speed"]),"n"),format(int(wd["total_shares"]),"n"),format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + colt = colt - 1 + if colt <= 0: + r+="
WorkerSpeedShares/RejFound
%s%s%s/%s%s
" + r+="" + r+="" + r+="
WorkerSpeedShares/RejFound
" + self.cache_html = str(r) + self.last_update = time.time() + + # Send Results + if(request.path == "/stats"): + return self.cache_json + return self.cache_html + + +def BasicStats(start_event): + start_event.addCallback(BasicStats_start) + +def BasicStats_start(cb): + root = StatsPage() + root.putChild('favicon.ico', static.File('statics/favicon.ico', defaultType='image/vnd.microsoft.icon') ) + root.putChild('basic_stats.css', static.File('statics/basic_stats.css') ) + factory = Site(root) + reactor.listenTCP(settings.BASIC_STATS_PORT, factory) + diff --git a/statics/basic_stats.css b/statics/basic_stats.css new file mode 100644 index 0000000..9ce8cd7 --- /dev/null +++ b/statics/basic_stats.css @@ -0,0 +1,4 @@ +H1 { + font-size: 18pt; + color: #999; + } diff --git a/statics/favicon.ico b/statics/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..14f09cd4c5cab38ea79051328c1107483f0063c4 GIT binary patch literal 2550 zcmeH}dr*{B7>9p?x1>}ux|vFnXeml2U|Me8O$=BS5d<~RKtWMp*@ayl5Efeq;$i`X z84!3|2AM@i5e!ihG;WSSArdaj)ifp*MPc9GU3A7v{89c{@67vs-#L5UbH3d@^E?L_ z00WF13BC`7*C4wHwCaw$%k-X0p@&H0b%MH1gGa>R%#UlX%(26ehKr_tKgV< z7Q&2D2-0id#?N;9&tXwU4eSq=!tp>koDP*B>`)%ul_l_27GrzXPgr>P3S6?wFfXef zE=S7Xo>K`IRTVa=$`Pb0#mbxpZr8#s_ZMtAT8X8qCJ0q(cpj~WBj3G`U4$@K1N);b z@H=r4k$GiUal91~CoABZr-ple9bEI(*qUDt=acOa=C?y~sunH<9dJ9<2KQ6#@I2iJ z?;>8O@CMcubz;Mr-{Eog8brm-@F{Lb=(z@jl{8{|X)_e(o8W(;4N(^~i21n*k!4MY zDsM)7MKh$8*AP^71JPA&kX*Wr#Oe;nYHlH^<{BdEZX&d<3*nr}b$>v9`7Y%3T}Y|# zg#5}qC>pvEd9?@8>K>%2yAY$%BCc7B11os!7y64@;+j)si zrtk}z`}lu6>y`HxZw-9=Mg13;CiD*s2ngITO5f65Y%C^Dp4!*CI}Yd^0u&koAp`od zzpe1_bKTg+5x`x+lbki}uk@*Y<}LM?W}acTPZd1qa7)fHunbKNT9$VDPv!;1YI-TY zg(#s#OUwvz$tfoj?`RtCEIDQ)iZOX~o((CvhRk;QCplqL_^5|-jF;|{GrK5C-9yYb zVs;brjF|7lJR{~KF%RkUEB7esw{BvG&@e~Q5na7NalnjSLjNYmc!bO`56x7BU+yC2 zA~C~>IZw^EYT6Z4kXhr|w}kYBrKTZx7SuM&5!$hvSdzLm&<>zDKUvwMi0 zqOWJ45Hp{c(Zr4<_93y?NLGED{LZ(~5T~G9cA3Ss30%)Uqs19DoYlk*)7P_~iG54# z0%F$?yO7vt#9kow7Y%iiR9eRzNsiU@5c`PO!Nfi#_Kv=uy+`aP{rc=fV#m>1{vW?= z){-DOkA|%hMOw)-tl4o|_6xDoiQP}^abh=-hq8p&RmAQl*B{%6JxF1f?-2Wmj6EZ} zCvKLT>GptSRCcDk{toS|y+wR(AUcq_g$@b zr5aD$t;}L)^FHpSXpNSld3;}LiTz8~v8wB)K2cj8B2t(Le;<}1sq?2Rl Date: Fri, 7 Dec 2012 08:09:15 -0600 Subject: [PATCH 12/56] Stats changes that (for some reason) did not get commited last time... grrrr --- README.md | 3 + TODO | 14 ++-- conf/config_sample.py | 10 +++ launcher.tac | 5 ++ mining/DBInterface.py | 16 +++++ mining/DB_Mysql.py | 110 +++++++++++++++++++++++++++-- mining/DB_None.py | 13 ++++ mining/DB_Postgresql.py | 119 +++++++++++++++++++++++++++++--- mining/DB_Sqlite.py | 148 ++++++++++++++++++++++++++++++++++------ mining/interfaces.py | 1 + 10 files changed, 401 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 4a1be19..5af2122 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,6 @@ See the INSTALL file for install instructions. For more info on Stratum: http://mining.bitcoin.cz/stratum-mining. +Original version by Slush + +This version by GeneralFault (Tips Welcome: 15Zk7DoFYJ7hESpZzmix1WLkomTMGW81c2 ) diff --git a/TODO b/TODO index ecf4b39..38f1632 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,17 @@ TODO File (in no particular order): SQL Connection pooling: sqlalchemy +Share Archiving in DB + - Perhaps separate table for found shares? -- Add a "script" to add,list,disable users +Add a "script" to add,list,disable users -Create a "service" (or whatever) that shows pool level stats. - -implement a basic share limiter that increases difficulty until share time +Implement a basic share limiter that increases difficulty until share time is X seconds between shares +Flush all Pending shares on shutdown (I really don't know how to do this.) + +Figure out "Best" submitted difficulty + +Handle bitcoind going away (Make sure stats reflect problems) + diff --git a/conf/config_sample.py b/conf/config_sample.py index ea6983c..a7fd68a 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -111,3 +111,13 @@ # This should be "slow" INSTANCE_ID = 31 # Not a clue what this is for... :P + +# ******************** Pool Settings ********************* + +BASIC_STATS = True # Enable basic stats page. This has stats for ALL users. + # (Requires advanced database to be enabled) + # Human : http://:/ + # JSON : http://:/stats + # (Disable if you have your own frontend) + +BASIC_STATS_PORT = 8889 # Port to listen on diff --git a/launcher.tac b/launcher.tac index 06fb136..e7835e4 100644 --- a/launcher.tac +++ b/launcher.tac @@ -28,3 +28,8 @@ Interfaces.set_worker_manager(WorkerManagerInterface()) Interfaces.set_timestamper(TimestamperInterface()) mining.setup(on_startup) + +if settings.DATABASE_EXTEND == True and settings.BASIC_STATS == True : + from lib.basic_stats import BasicStats + BasicStats(on_startup) + diff --git a/mining/DBInterface.py b/mining/DBInterface.py index fca494c..ae2529a 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -21,6 +21,9 @@ def __init__(self): self.scheduleImport() + def set_bitcoinrpc(self,bitcoinrpc): + self.bitcoinrpc=bitcoinrpc + def connectDB(self): # Choose our database driver and put it in self.dbi if settings.DATABASE_DRIVER == "sqlite": @@ -64,6 +67,8 @@ def run_import(self): self.do_import(self.dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) + d = self.bitcoinrpc.getinfo() + d.addCallback(self._update_pool_info) self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME self.scheduleImport() @@ -73,8 +78,14 @@ def import_thread(self): self.do_import(dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) + d = self.bitcoinrpc.getinfo() + d.addCallback(self._update_pool_info) self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + def _update_pool_info(self,data): + self.dbi.update_pool_info({ 'blocks' : data['blocks'], 'balance' : data['balance'], + 'connections' : data['connections'], 'difficulty' : data['difficulty'] }) + def do_import(self,dbi): # Only run if we have data while self.q.qsize() >= settings.DB_LOADER_REC_MIN: @@ -130,3 +141,8 @@ def update_user(self,username,password): self.usercache = {} return self.dbi.update_user(username,password) + def get_pool_stats(self): + return self.dbi.get_pool_stats() + + def get_workers_stats(self): + return self.dbi.get_workers_stats() diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 2a70d60..1d94fef 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -17,11 +17,14 @@ def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. self.dbc.execute("update pool_worker set speed = 0"); - stime = '%.2f' % ( time.time() - averageOverTime ); + stime = '%.0f' % ( time.time() - averageOverTime ); self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) + total_speed = 0 for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + total_speed += speed self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() def import_shares(self,data): @@ -29,13 +32,20 @@ def import_shares(self,data): # 0 1 2 3 4 5 6 7 8 9 # data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] checkin_times = {} + total_shares = 0 for k,v in enumerate(data): if settings.DATABASE_EXTEND : + total_shares += v[3] if v[0] in checkin_times: if v[4] > checkin_times[v[0]] : - checkin_times[v[0]] = v[4] + checkin_times[v[0]]["time"] = v[4] else: - checkin_times[v[0]] = v[4] + checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } + + if v[5] == True : + checkin_times[v[0]]["shares"] += v[3] + else : + checkin_times[v[0]]["rejects"] += v[3] self.dbc.execute("insert into shares " +\ "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ @@ -45,9 +55,21 @@ def import_shares(self,data): self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ "(FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s)", (v[4],v[6],v[0],v[5],0,v[9],'') ) + if settings.DATABASE_EXTEND : + self.dbc.execute("select value from pool where parameter = 'round_shares'") + round_shares = int(self.dbc.fetchone()[0]) + total_shares + self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + difficulty = float(self.dbc.fetchone()[0]) + + progress = (round_shares/difficulty)*100 + self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) + for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s) where username = %s",(v,k)) + self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", + (v["time"],v["shares"],v["rejects"],k)) self.dbh.commit() @@ -56,6 +78,16 @@ def found_block(self,data): # Note: difficulty = -1 here self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", (data[5],data[2],data[4],data[0])) + if settings.DATABASE_EXTEND : + if data[5] == True: + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) + self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + total_found = int(self.dbc.fetchone()[0]) + 1 + self.dbc.executemany("update pool set value = %s where paramter = %s",[(0,'round_shares'), + (0,'round_progress'), + (time.time(),'round_start'), + ([total_found],'pool_total_found') + ]) self.dbh.commit() def delete_user(self,username): @@ -85,6 +117,33 @@ def check_password(self,username,password): return True return False + def update_pool_info(self,pi): + self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), + (pi['balance'],"bitcoin_balance"), + (pi['connections'],"bitcoin_connections"), + (pi['difficulty'],"bitcoin_difficulty") + ]) + self.dbh.commit() + + def get_pool_stats(self): + self.dbc.execute("select * from pool") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = data[1] + return ret + + def get_workers_stats(self): + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = { "username" : data[0], + "speed" : data[1], + "last_checkin" : time.mktime(data[2].timetuple()), + "total_shares" : data[3], + "total_rejects" : data[4], + "total_found" : data[5] } + return ret + def check_tables(self): log.debug("Checking Tables") @@ -108,9 +167,23 @@ def check_tables(self): "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") if shares_exist == False: self.dbc.execute("create index shares_username ON shares(username(10))") - self.dbc.execute("create table if not exists pool_worker(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp) ENGINE = MYISAM") + + self.dbc.execute("create table if not exists pool_worker" +\ + "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp" +\ + ") ENGINE = MYISAM") if pool_worker_exist == False: self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + + self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") + self.dbc.execute("select COUNT(*) from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + if data[0] <= 0: + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + + self.update_tables() else : self.dbc.execute("create table if not exists shares" + \ "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") @@ -121,3 +194,30 @@ def check_tables(self): self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") self.dbh.commit() + def update_tables(self): + version = 0 + current_version = 3 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() + + + def update_version_2(self): + log.info("running update 2") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), + ('bitcoin_balance',0), + ('bitcoin_connections',0), + ('bitcoin_difficulty',0), + ('pool_speed',0), + ('pool_total_found',0), + ('round_shares',0), + ('round_progress',0), + ('round_start',time.time()) + ]) + self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_None.py b/mining/DB_None.py index 59374e0..5d14c5d 100644 --- a/mining/DB_None.py +++ b/mining/DB_None.py @@ -26,6 +26,19 @@ def update_user(self,username,password): def check_password(self,username,password): log.debug("Checking Username/Password") return True + + def update_pool_info(self,pi): + log.debug("Update Pool Info") + + def get_pool_stats(self): + log.debug("Get Pool Stats") + ret = {} + return ret + + def get_workers_stats(self): + log.debug("Get Workers Stats") + ret = {} + return ret def check_tables(self): log.debug("Checking Tables") diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index a217f74..09cacc3 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -20,10 +20,13 @@ def updateStats(self,averageOverTime): # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. self.dbc.execute("update pool_worker set speed = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); - self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", (stime,)) + self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", [stime]) + total_speed = 0 for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + total_speed += speed self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() def import_shares(self,data): @@ -31,25 +34,44 @@ def import_shares(self,data): # 0 1 2 3 4 5 6 7 8 9 # data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] checkin_times = {} + total_shares = 0 for k,v in enumerate(data): if settings.DATABASE_EXTEND : + total_shares += v[3] if v[0] in checkin_times: if v[4] > checkin_times[v[0]] : - checkin_times[v[0]] = v[4] + checkin_times[v[0]]["time"] = v[4] else: - checkin_times[v[0]] = v[4] + checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } + + if v[5] == True : + checkin_times[v[0]]["shares"] += v[3] + else : + checkin_times[v[0]]["rejects"] += v[3] self.dbc.execute("insert into shares " +\ "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ "VALUES (to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + (v[4],v[6],v[0],bool(v[5]),False,v[9],'',v[7],v[8],'',v[3]) ) else : self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ "(to_timestamp(%s),%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'') ) + (v[4],v[6],v[0],bool(v[5]),False,v[9],'') ) + if settings.DATABASE_EXTEND : + self.dbc.execute("select value from pool where parameter = 'round_shares'") + round_shares = int(self.dbc.fetchone()[0]) + total_shares + self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + difficulty = float(self.dbc.fetchone()[0]) + + progress = (round_shares/difficulty)*100 + self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) + for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = to_timestamp(%s) where username = %s",(v,k)) + self.dbc.execute("update pool_worker set last_checkin = to_timestamp(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", + (v["time"],v["shares"],v["rejects"],k)) self.dbh.commit() @@ -57,7 +79,17 @@ def import_shares(self,data): def found_block(self,data): # Note: difficulty = -1 here self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", - (data[5],data[2],data[4],data[0])) + (bool(data[5]),data[2],data[4],data[0])) + if settings.DATABASE_EXTEND : + if data[5] == True: + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) + self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + total_found = int(self.dbc.fetchone()[0]) + 1 + self.dbc.executemany("update pool set value = %s where paramter = %s",[(0,'round_shares'), + (0,'round_progress'), + (time.time(),'round_start'), + ([total_found],'pool_total_found') + ]) self.dbh.commit() def delete_user(self,username): @@ -87,6 +119,33 @@ def check_password(self,username,password): return True return False + def update_pool_info(self,pi): + self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), + (pi['balance'],"bitcoin_balance"), + (pi['connections'],"bitcoin_connections"), + (pi['difficulty'],"bitcoin_difficulty") + ]) + self.dbh.commit() + + def get_pool_stats(self): + self.dbc.execute("select * from pool") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = data[1] + return ret + + def get_workers_stats(self): + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = { "username" : data[0], + "speed" : data[1], + "last_checkin" : time.mktime(data[2].timetuple()), + "total_shares" : data[3], + "total_rejects" : data[4], + "total_found" : data[5] } + return ret + def check_tables(self): log.debug("Checking Tables") @@ -103,6 +162,13 @@ def check_tables(self): data = self.dbc.fetchone() if data[0] > 0 : pool_worker_exist = True + + pool_exist = False + self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'pool'", + {"schema": settings.DB_PGSQL_SCHEMA }) + data = self.dbc.fetchone() + if data[0] > 0 : + pool_exist = True if settings.DATABASE_EXTEND == True : if shares_exist == False: @@ -111,15 +177,50 @@ def check_tables(self): "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") self.dbc.execute("create index shares_username ON shares(username)") if pool_worker_exist == False: - self.dbc.execute("create table pool_worker(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") + self.dbc.execute("create table pool_worker" +\ + "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + if pool_exist == False: + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("create table pool(parameter TEXT, value TEXT)") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + self.update_tables() else : if shares_exist == False: self.dbc.execute("create table shares" + \ - "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT)") + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT)") self.dbc.execute("create index shares_username ON shares(username)") if pool_worker_exist == False: self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") self.dbc.execute("create index pool_worker_username ON pool_worker(username)") self.dbh.commit() + def update_tables(self): + version = 0 + current_version = 3 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() + + + def update_version_2(self): + log.info("running update 2") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), + ('bitcoin_balance',0), + ('bitcoin_connections',0), + ('bitcoin_difficulty',0), + ('pool_speed',0), + ('pool_total_found',0), + ('round_shares',0), + ('round_progress',0), + ('round_start',time.time()) + ]) + self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 33aace3..8802dec 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -4,7 +4,9 @@ log = stratum.logger.get_logger('DB_Sqlite') import sqlite3 - +from sqlalchemy.orm import sessionmaker +from sqlalchemy import create_engine + class DB_Sqlite(): def __init__(self): log.debug("Connecting to DB") @@ -18,10 +20,15 @@ def updateStats(self,averageOverTime): # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. self.dbc.execute("update pool_worker set speed = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); - self.dbc.execute("select username,SUM(difficulty) from shares where time > ? group by username", (stime,)) + self.dbc.execute("select username,SUM(difficulty) from shares where time > :time group by username", {'time':stime}) + total_speed = 0 + sqldata = [] for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) - self.dbc.execute("update pool_worker set speed = ? where username = ?",(speed,name)) + total_speed += speed + sqldata.append({'speed':speed,'user':name}) + self.dbc.executemany("update pool_worker set speed = :speed where username = :user",sqldata) + self.dbc.execute("update pool set value = :val where parameter = 'pool_speed'",{'val':total_speed}) self.dbh.commit() def import_shares(self,data): @@ -29,68 +36,169 @@ def import_shares(self,data): # 0 1 2 3 4 5 6 7 8 9 # data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] checkin_times = {} + total_shares = 0 + sqldata = [] for k,v in enumerate(data): if settings.DATABASE_EXTEND : + total_shares += v[3] if v[0] in checkin_times: if v[4] > checkin_times[v[0]] : - checkin_times[v[0]] = v[4] + checkin_times[v[0]]["time"] = v[4] else: - checkin_times[v[0]] = v[4] + checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } - self.dbc.execute("insert into shares " +\ - "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ - "VALUES (?,?,?,?,?,?,?,?,?,?,?)", - (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + if v[5] == True : + checkin_times[v[0]]["shares"] += v[3] + else : + checkin_times[v[0]]["rejects"] += v[3] + + sqldata.append({'time':v[4],'rem_host':v[6],'username':v[0],'our_result':v[5],'upstream_result':0,'reason':v[9],'solution':'', + 'block_num':v[7],'prev_block_hash':v[8],'ua':'','diff':v[3]} ) else : - self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES (?,?,?,?,?,?,?)", - (v[4],v[6],v[0],v[5],0,v[9],'') ) + sqldata.append({'time':v[4],'rem_host':v[6],'username':v[0],'our_result':v[5],'upstream_result':0,'reason':v[9],'solution':''} ) + if settings.DATABASE_EXTEND : + self.dbc.executemany("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (:time,:rem_host,:username,:our_result,:upstream_result,:reason,:solution,:block_num,:prev_block_hash,:ua,:diff)",sqldata) + + + self.dbc.execute("select value from pool where parameter = 'round_shares'") + round_shares = int(self.dbc.fetchone()[0]) + total_shares + self.dbc.execute("update pool set value = :val where parameter = 'round_shares'",{'val':round_shares}) + + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + difficulty = float(self.dbc.fetchone()[0]) + + progress = (round_shares/difficulty)*100 + self.dbc.execute("update pool set value = :val where parameter = 'round_progress'",{'val':progress}) + + sqldata = [] for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = ? where username = ?",(v,k)) + sqldata.append({'last_checkin':v["time"],'addshares':v["shares"],'addrejects':v["rejects"],'user':k}) + self.dbc.executemany("update pool_worker set last_checkin = :last_checkin, total_shares = total_shares + :addshares, " +\ + "total_rejects = total_rejects + :addrejects where username = :user",sqldata) + else: + self.dbc.executemany("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) " +\ + "VALUES (:time,:rem_host,:username,:our_result,:upstream_result,:reason,:solution)",sqldata) self.dbh.commit() - def found_block(self,data): # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = ?, solution = ? where time = ? and username = ?", - (data[5],data[2],data[4],data[0])) + self.dbc.execute("update shares set upstream_result = :usr, solution = :sol where time = :time and username = :user", + {'usr':data[5],'sol':data[2],'time':data[4],'user':data[0]}) + if settings.DATABASE_EXTEND : + if data[5] == True: + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = :user",{'user':data[0]}) + self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + total_found = int(self.dbc.fetchone()[0]) + 1 + self.dbc.execute("update pool set value = :val where paramter = :parm", [{'val':0,'parm':'round_shares'}, + {'val':0,'parm':'round_progress'}, + {'val':time.time(),'parm':'round_start'}, + {'val':[total_found],'parm':'pool_total_found'} + ]) self.dbh.commit() def delete_user(self,username): log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = ?", (username)) + self.dbc.execute("delete from pool_worker where username = :user", {'user':username}) self.dbh.commit() def insert_user(self,username,password): log.debug("Adding Username/Password") - self.dbc.execute("insert into pool_worker (username,password) VALUES (?,?)", (username,password)) + self.dbc.execute("insert into pool_worker (username,password) VALUES (:user,:pass)", {'user':username,'pass':password}) self.dbh.commit() def update_user(self,username,password): log.debug("Updating Username/Password") - self.dbc.execute("update pool_worker set password = ? where username = ?", (password,username)) + self.dbc.execute("update pool_worker set password = :pass where username = :user", {'pass':password,'user':username}) self.dbh.commit() def check_password(self,username,password): log.debug("Checking Username/Password") - self.dbc.execute("select COUNT(*) from pool_worker where username = ? and password = ?",(username,password)) + self.dbc.execute("select COUNT(*) from pool_worker where username = :user and password = :pass", {'user':username,'pass':password}) data = self.dbc.fetchone() if data[0] > 0 : return True return False + def update_pool_info(self,pi): + self.dbc.executemany("update pool set value = :val where parameter = :parm",[{'val':pi['blocks'],'parm':"bitcoin_blocks"}, + {'val':pi['balance'],'parm':"bitcoin_balance"}, + {'val':pi['connections'],'parm':"bitcoin_connections"}, + {'val':pi['difficulty'],'parm':"bitcoin_difficulty"} + ]) + self.dbh.commit() + + def get_pool_stats(self): + self.dbc.execute("select * from pool") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = data[1] + return ret + + def get_workers_stats(self): + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = { "username" : data[0], + "speed" : data[1], + "last_checkin" : data[2], + "total_shares" : data[3], + "total_rejects" : data[4], + "total_found" : data[5] } + return ret + def check_tables(self): log.debug("Checking Tables") if settings.DATABASE_EXTEND == True : self.dbc.execute("create table if not exists shares" +\ "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT, " +\ "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") - self.dbc.execute("create table if not exists pool_worker(username TEXT, password TEXT, speed INTEGER, last_checkin DATETIME)") + self.dbc.execute("create table if not exists pool_worker" +\ + "(username TEXT, password TEXT, speed INTEGER, last_checkin DATETIME)") + self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") + + self.dbc.execute("select COUNT(*) from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + if data[0] <= 0: + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + self.update_tables() else : self.dbc.execute("create table if not exists shares" + \ "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT)") self.dbc.execute("create table if not exists pool_worker(username TEXT, password TEXT)") self.dbc.execute("create index if not exists shares_username ON shares(username)") self.dbc.execute("create index if not exists pool_worker_username ON pool_worker(username)") + + def update_tables(self): + version = 0 + current_version = 3 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() + + + def update_version_2(self): + log.info("running update 2") + self.dbc.executemany("insert into pool (parameter,value) VALUES (?,?)",[('bitcoin_blocks',0), + ('bitcoin_balance',0), + ('bitcoin_connections',0), + ('bitcoin_difficulty',0), + ('pool_speed',0), + ('pool_total_found',0), + ('round_shares',0), + ('round_progress',0), + ('round_start',time.time()) + ]) + self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + self.dbh.commit() diff --git a/mining/interfaces.py b/mining/interfaces.py index f89c6d2..eabd157 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -99,4 +99,5 @@ def set_timestamper(cls, manager): @classmethod def set_template_registry(cls, registry): + dbi.set_bitcoinrpc(registry.bitcoin_rpc) cls.template_registry = registry From 5fa9adff8432501f1b80c4a1559a783d47529768 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 7 Dec 2012 08:27:32 -0600 Subject: [PATCH 13/56] Fix of >10 worker section of basic stats --- lib/basic_stats.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/basic_stats.py b/lib/basic_stats.py index dfe0f19..442a59d 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -95,10 +95,11 @@ def render_GET(self, request): else : colcnt = int(size/3) + 1 colt = colcnt - r+="" + r+="" r+="" r+="" - for (w, wd) in enumerate(workers_stats): + for (w, wi) in enumerate(workers_stats): + wd = workers_stats[wi] wc = "#A00" if wd["speed"] > 0: wc = "yellow" @@ -108,9 +109,10 @@ def render_GET(self, request): wc,wi,format(int(wd["speed"]),"n"),format(int(wd["total_shares"]),"n"),format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) colt = colt - 1 if colt <= 0: - r+="
WorkerSpeedShares/RejFound
" + r+="" r+="" r+="" + colt = colcnt r+="
WorkerSpeedShares/RejFound
" r+="" self.cache_html = str(r) From 14a0013f818e415a24984662a905ca36dea0fb0e Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 8 Dec 2012 07:52:43 -0600 Subject: [PATCH 14/56] Missed removal of a dep --- mining/DB_Sqlite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 8802dec..9653285 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -4,8 +4,6 @@ log = stratum.logger.get_logger('DB_Sqlite') import sqlite3 -from sqlalchemy.orm import sessionmaker -from sqlalchemy import create_engine class DB_Sqlite(): def __init__(self): From aeca8a382654cecdf1820ce531d0efe159166648 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 8 Dec 2012 10:00:01 -0600 Subject: [PATCH 15/56] added "best_share" stat, indicate bitcoind prob on stats page, added "alive" to workers --- TODO | 4 ---- lib/basic_stats.py | 7 +++++-- lib/template_registry.py | 7 +++++-- mining/DB_Mysql.py | 37 +++++++++++++++++++++++++++++-------- mining/DB_Postgresql.py | 37 +++++++++++++++++++++++++++++-------- mining/DB_Sqlite.py | 37 +++++++++++++++++++++++++++++-------- mining/interfaces.py | 11 ++++++----- mining/service.py | 8 ++++---- 8 files changed, 107 insertions(+), 41 deletions(-) diff --git a/TODO b/TODO index 38f1632..77673bf 100644 --- a/TODO +++ b/TODO @@ -11,7 +11,3 @@ Implement a basic share limiter that increases difficulty until share time Flush all Pending shares on shutdown (I really don't know how to do this.) -Figure out "Best" submitted difficulty - -Handle bitcoind going away (Make sure stats reflect problems) - diff --git a/lib/basic_stats.py b/lib/basic_stats.py index 442a59d..f6906ed 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -59,19 +59,22 @@ def render_GET(self, request): r+="Speed: " + pool_stats['pool_speed'] + "Mhs" r+="Round Shares: " + format(int(pool_stats['round_shares']),"n") + " (%.2f" % float(pool_stats['round_progress']) + "%)" r+="Round Duration: " + str(timedelta(0,(time.time() - float(pool_stats['round_start'])))).split(".")[0] + r+="Best Share: " + format(int(pool_stats['round_best_share']),"n") r+="Total Found: " + format(int(pool_stats['pool_total_found']),"n") r+="" + last_update = time.time() - float(pool_stats['bitcoin_infotime']) bitcoin_color="#A00" - if int(pool_stats['bitcoin_connections']) > 0: + if int(pool_stats['bitcoin_connections']) > 0 and last_update < 660: bitcoin_color="yellow" - if int(pool_stats['bitcoin_connections']) > 10: + if int(pool_stats['bitcoin_connections']) > 10 and last_update < 330: bitcoin_color="#0A0" r+="Bitcoin Stats:" r+="Connections :"+pool_stats['bitcoin_connections'] r+="Difficulty :"+format(int(float(pool_stats['bitcoin_difficulty'])),"n") r+="Height :"+format(int(pool_stats['bitcoin_blocks']),"n") r+="Balance :"+pool_stats['bitcoin_balance'] + r+="" r+="" r+="
" diff --git a/lib/template_registry.py b/lib/template_registry.py index 7e3e33f..8841151 100644 --- a/lib/template_registry.py +++ b/lib/template_registry.py @@ -233,6 +233,9 @@ def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, if hash_int <= target_info: log.info("Yay, share with diff above 100000") + # Algebra tells us the diff_to_target is the same as hash_to_diff + share_diff = int(self.diff_to_target(hash_int)) + # 5. Compare hash with target of the network if hash_int <= job.target: # Yay! It is block candidate! @@ -249,6 +252,6 @@ def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, serialized = binascii.hexlify(job.serialize()) on_submit = self.bitcoin_rpc.submitblock(serialized) - return (header_hex, block_hash_hex, on_submit) + return (header_hex, block_hash_hex, share_diff, on_submit) - return (header_hex, block_hash_hex, None) + return (header_hex, block_hash_hex, share_diff, None) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 1d94fef..dc12567 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -16,23 +16,24 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0"); + self.dbc.execute("update pool_worker set speed = 0, alive = 0"); stime = '%.0f' % ( time.time() - averageOverTime ); self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) total_speed = 0 for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) total_speed += speed - self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbc.execute("update pool_worker set speed = %s, alive = 1 where username = %s", (speed,name)) self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() def import_shares(self,data): log.debug("Importing Shares") -# 0 1 2 3 4 5 6 7 8 9 -# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] +# 0 1 2 3 4 5 6 7 8 9 10 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] checkin_times = {} total_shares = 0 + best_diff = 0 for k,v in enumerate(data): if settings.DATABASE_EXTEND : total_shares += v[3] @@ -47,6 +48,9 @@ def import_shares(self,data): else : checkin_times[v[0]]["rejects"] += v[3] + if v[10] > best_diff: + best_diff = v[10] + self.dbc.execute("insert into shares " +\ "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ "VALUES (FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", @@ -61,6 +65,11 @@ def import_shares(self,data): round_shares = int(self.dbc.fetchone()[0]) + total_shares self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + self.dbc.execute("select value from pool where parameter = 'round_best_share'") + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: + self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) @@ -121,7 +130,8 @@ def update_pool_info(self,pi): self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), (pi['balance'],"bitcoin_balance"), (pi['connections'],"bitcoin_connections"), - (pi['difficulty'],"bitcoin_difficulty") + (pi['difficulty'],"bitcoin_difficulty"), + (time.time(),"bitcoin_infotime") ]) self.dbh.commit() @@ -133,7 +143,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -141,7 +151,8 @@ def get_workers_stats(self): "last_checkin" : time.mktime(data[2].timetuple()), "total_shares" : data[3], "total_rejects" : data[4], - "total_found" : data[5] } + "total_found" : data[5], + "alive" : data[6] } return ret def check_tables(self): @@ -196,7 +207,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 3 + current_version = 4 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -221,3 +232,13 @@ def update_version_2(self): self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") self.dbh.commit() + def update_version_3(self): + log.info("running update 3") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ + ('round_best_share',0), + ('bitcoin_infotime',0) + ]) + self.dbc.execute("alter table pool_worker add alive BOOLEAN") + self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 09cacc3..93a7978 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -18,23 +18,24 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0"); + self.dbc.execute("update pool_worker set speed = 0, alive = 'f'"); stime = '%.2f' % ( time.time() - averageOverTime ); self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", [stime]) total_speed = 0 for name,shares in self.dbc.fetchall(): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) total_speed += speed - self.dbc.execute("update pool_worker set speed = %s where username = %s", (speed,name)) + self.dbc.execute("update pool_worker set speed = %s, alive = 't' where username = %s", (speed,name)) self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() def import_shares(self,data): log.debug("Importing Shares") -# 0 1 2 3 4 5 6 7 8 9 -# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] +# 0 1 2 3 4 5 6 7 8 9 10 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] checkin_times = {} total_shares = 0 + best_diff = 0 for k,v in enumerate(data): if settings.DATABASE_EXTEND : total_shares += v[3] @@ -49,6 +50,9 @@ def import_shares(self,data): else : checkin_times[v[0]]["rejects"] += v[3] + if v[10] > best_diff: + best_diff = v[10] + self.dbc.execute("insert into shares " +\ "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ "VALUES (to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", @@ -63,6 +67,11 @@ def import_shares(self,data): round_shares = int(self.dbc.fetchone()[0]) + total_shares self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + self.dbc.execute("select value from pool where parameter = 'round_best_share'") + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: + self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) @@ -123,7 +132,8 @@ def update_pool_info(self,pi): self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), (pi['balance'],"bitcoin_balance"), (pi['connections'],"bitcoin_connections"), - (pi['difficulty'],"bitcoin_difficulty") + (pi['difficulty'],"bitcoin_difficulty"), + (time.time(),"bitcoin_infotime") ]) self.dbh.commit() @@ -135,7 +145,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -143,7 +153,8 @@ def get_workers_stats(self): "last_checkin" : time.mktime(data[2].timetuple()), "total_shares" : data[3], "total_rejects" : data[4], - "total_found" : data[5] } + "total_found" : data[5], + "alive" : data[6] } return ret def check_tables(self): @@ -199,7 +210,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 3 + current_version = 4 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -224,3 +235,13 @@ def update_version_2(self): self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") self.dbh.commit() + def update_version_3(self): + log.info("running update 3") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ + ('round_best_share',0), + ('bitcoin_infotime',0) + ]) + self.dbc.execute("alter table pool_worker add alive BOOLEAN") + self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 9653285..ef695c0 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -16,7 +16,7 @@ def __init__(self): def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0"); + self.dbc.execute("update pool_worker set speed = 0, alive = 0"); stime = '%.2f' % ( time.time() - averageOverTime ); self.dbc.execute("select username,SUM(difficulty) from shares where time > :time group by username", {'time':stime}) total_speed = 0 @@ -25,16 +25,17 @@ def updateStats(self,averageOverTime): speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) total_speed += speed sqldata.append({'speed':speed,'user':name}) - self.dbc.executemany("update pool_worker set speed = :speed where username = :user",sqldata) + self.dbc.executemany("update pool_worker set speed = :speed, alive = 1 where username = :user",sqldata) self.dbc.execute("update pool set value = :val where parameter = 'pool_speed'",{'val':total_speed}) self.dbh.commit() def import_shares(self,data): log.debug("Importing Shares") -# 0 1 2 3 4 5 6 7 8 9 -# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason] +# 0 1 2 3 4 5 6 7 8 9 10 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,share_diff] checkin_times = {} total_shares = 0 + best_diff = 0 sqldata = [] for k,v in enumerate(data): if settings.DATABASE_EXTEND : @@ -50,6 +51,9 @@ def import_shares(self,data): else : checkin_times[v[0]]["rejects"] += v[3] + if v[10] > best_diff: + best_diff = v[10] + sqldata.append({'time':v[4],'rem_host':v[6],'username':v[0],'our_result':v[5],'upstream_result':0,'reason':v[9],'solution':'', 'block_num':v[7],'prev_block_hash':v[8],'ua':'','diff':v[3]} ) else : @@ -65,6 +69,11 @@ def import_shares(self,data): round_shares = int(self.dbc.fetchone()[0]) + total_shares self.dbc.execute("update pool set value = :val where parameter = 'round_shares'",{'val':round_shares}) + self.dbc.execute("select value from pool where parameter = 'round_best_share'") + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: + self.dbc.execute("update pool set value = :val where parameter = 'round_best_share'",{'val':best_diff}) + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) @@ -125,7 +134,8 @@ def update_pool_info(self,pi): self.dbc.executemany("update pool set value = :val where parameter = :parm",[{'val':pi['blocks'],'parm':"bitcoin_blocks"}, {'val':pi['balance'],'parm':"bitcoin_balance"}, {'val':pi['connections'],'parm':"bitcoin_connections"}, - {'val':pi['difficulty'],'parm':"bitcoin_difficulty"} + {'val':pi['difficulty'],'parm':"bitcoin_difficulty"}, + {'val':time.time(),'parm':"bitcoin_infotime"} ]) self.dbh.commit() @@ -137,7 +147,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -145,7 +155,8 @@ def get_workers_stats(self): "last_checkin" : data[2], "total_shares" : data[3], "total_rejects" : data[4], - "total_found" : data[5] } + "total_found" : data[5], + "alive" : data[6] } return ret def check_tables(self): @@ -175,7 +186,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 3 + current_version = 4 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -199,4 +210,14 @@ def update_version_2(self): ]) self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") self.dbh.commit() + + def update_version_3(self): + log.info("running update 3") + self.dbc.executemany("insert into pool (parameter,value) VALUES (?,?)",[ + ('round_best_share',0), + ('bitcoin_infotime',0), + ]) + self.dbc.execute("alter table pool_worker add alive INTEGER default 0") + self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + self.dbh.commit() diff --git a/mining/interfaces.py b/mining/interfaces.py index eabd157..e0404e3 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -51,13 +51,14 @@ def on_network_block(self, prevhash, block_height): self.prev_hash = b58encode(int(prevhash,16)) pass - def on_submit_share(self, worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, invalid_reason ): - log.info("%s %s %s" % (block_hash, 'valid' if is_valid else 'INVALID', worker_name)) - dbi.queue_share([worker_name,block_header,block_hash,difficulty,timestamp,is_valid, ip, self.block_height, self.prev_hash, invalid_reason ]) + def on_submit_share(self, worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, invalid_reason, share_diff ): + log.info("%s (%s) %s %s" % (block_hash, share_diff, 'valid' if is_valid else 'INVALID', worker_name)) + dbi.queue_share([worker_name,block_header,block_hash,difficulty,timestamp,is_valid, ip, self.block_height, self.prev_hash, + invalid_reason, share_diff ]) - def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip ): + def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip, share_diff ): log.info("Block %s %s" % (block_hash, 'ACCEPTED' if is_accepted else 'REJECTED')) - dbi.found_block([worker_name,block_header,block_hash,-1,timestamp,is_accepted,ip,self.block_height, self.prev_hash]) + dbi.found_block([worker_name,block_header,block_hash,-1,timestamp,is_accepted,ip,self.block_height, self.prev_hash, share_diff ]) class TimestamperInterface(object): '''This is the only source for current time in the application. diff --git a/mining/service.py b/mining/service.py index 7642ef0..4eb05e0 100644 --- a/mining/service.py +++ b/mining/service.py @@ -101,23 +101,23 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): # This checks if submitted share meet all requirements # and it is valid proof of work. try: - (block_header, block_hash, on_submit) = Interfaces.template_registry.submit_share(job_id, + (block_header, block_hash, share_diff, on_submit) = Interfaces.template_registry.submit_share(job_id, worker_name, extranonce1_bin, extranonce2, ntime, nonce, difficulty) except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, - submit_time, False, ip, e[0]) + submit_time, False, ip, e[0], share_diff) raise Interfaces.share_manager.on_submit_share(worker_name, block_header, block_hash, difficulty, - submit_time, True, ip, '') + submit_time, True, ip, '', share_diff) if on_submit != None: # Pool performs submitblock() to bitcoind. Let's hook # to result and report it to share manager on_submit.addCallback(Interfaces.share_manager.on_submit_block, - worker_name, block_header, block_hash, submit_time,ip) + worker_name, block_header, block_hash, submit_time,ip,share_diff) return True From 7dd0a706eb322a826d925c7b4c8fbc1f09f0ae44 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 8 Dec 2012 10:03:47 -0600 Subject: [PATCH 16/56] Fix share_diff on rejected share --- mining/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mining/service.py b/mining/service.py index 4eb05e0..2fa9798 100644 --- a/mining/service.py +++ b/mining/service.py @@ -106,7 +106,7 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, - submit_time, False, ip, e[0], share_diff) + submit_time, False, ip, e[0], 0) raise From 42deb3afb51413b1000a52a9916ef395882b0743 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 9 Dec 2012 22:07:33 -0600 Subject: [PATCH 17/56] Basic Share limiter ... i.e. Basic Variable Difficulty code --- conf/config_sample.py | 19 ++++++-- launcher.tac | 8 ++- mining/basic_share_limiter.py | 92 +++++++++++++++++++++++++++++++++++ mining/interfaces.py | 2 +- mining/service.py | 2 +- 5 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 mining/basic_share_limiter.py diff --git a/conf/config_sample.py b/conf/config_sample.py index a7fd68a..3d300bc 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -99,9 +99,6 @@ CENTRAL_WALLET = 'set_valid_addresss_in_config!' # local bitcoin address where money goes COINBASE_EXTRAS = '/stratumPool/' # Extra Descriptive String to incorporate in solved blocks -# Pool Target -POOL_TARGET = 1 # Pool-wide difficulty target int >= 1 - # Bitcoind communication polling settings (In Seconds) PREVHASH_REFRESH_INTERVAL = 5 # How often to check for new Blocks # If using the blocknotify script (recommended) set = to MERKLE_REFRESH_INTERVAL @@ -112,7 +109,21 @@ INSTANCE_ID = 31 # Not a clue what this is for... :P -# ******************** Pool Settings ********************* +# ******************** Pool Difficulty Settings ********************* +# Again, Don't change unless you know what this is for. + +# Pool Target (Base Difficulty) +POOL_TARGET = 1 # Pool-wide difficulty target int >= 1 + +# Variable Difficulty Enable +VARIABLE_DIFF = False # Master variable difficulty enable + +# Variable diff tuning variables +VDIFF_TARGET = 30 # Target time per share (i.e. try to get 1 share per this many seconds) +VDIFF_RETARGET = 300 # Check to see if we should retarget this often +VDIFF_VARIANCE_PERCENT = 50 # Allow average time to very this % from target without retarget + +# ******************** Stats Settings ********************* BASIC_STATS = True # Enable basic stats page. This has stats for ALL users. # (Requires advanced database to be enabled) diff --git a/launcher.tac b/launcher.tac index e7835e4..c9ec635 100644 --- a/launcher.tac +++ b/launcher.tac @@ -22,8 +22,14 @@ from mining.interfaces import Interfaces from mining.interfaces import WorkerManagerInterface, TimestamperInterface, \ ShareManagerInterface, ShareLimiterInterface +if settings.VARIABLE_DIFF == True: + from mining.basic_share_limiter import BasicShareLimiter + Interfaces.set_share_limiter(BasicShareLimiter()) +else: + from mining.interfaces import ShareLimiterInterface + Interfaces.set_share_limiter(ShareLimiterInterface()) + Interfaces.set_share_manager(ShareManagerInterface()) -Interfaces.set_share_limiter(ShareLimiterInterface()) Interfaces.set_worker_manager(WorkerManagerInterface()) Interfaces.set_timestamper(TimestamperInterface()) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py new file mode 100644 index 0000000..88b69da --- /dev/null +++ b/mining/basic_share_limiter.py @@ -0,0 +1,92 @@ +from stratum import settings + +import stratum.logger +log = stratum.logger.get_logger('BasicShareLimiter') + + +''' This is just a cusomized ring buffer ''' +class SpeedBuffer: + def __init__(self,size_max): + self.max = size_max + self.data = [] + self.cur = 0 + def append(self,x): + self.data.append(x) + self.cur += 1 + if len(self.data) == self.max: + self.cur=0 + self.__class__ = SpeedBufferFull + def avg(self): + return sum(self.data)/self.cur + def pos(self): + return self.cur + def clear(self): + self.data=[] + self.cur=0 + +class SpeedBufferFull: + def __init__(self,n): + raise "you should use SpeedBuffer" + def append(self,x): + self.data[self.cur]=x + self.cur=(self.cur+1) % self.max + def avg(self): + return sum(self.data)/self.max + def pos(self): + return self.cur + def clear(self): + self.data=[] + self.cur=0 + self.__class__ = SpeedBuffer + +class BasicShareLimiter(object): + def __init__(self): + self.worker_stats = {} + self.target = settings.VDIFF_TARGET + self.retarget = settings.VDIFF_RETARGET + self.variance = self.target * (float(settings.VDIFF_VARIANCE_PERCENT)/float(100)) + self.tmin = self.target - self.variance + self.tmax = self.target + self.variance + self.buffersize = self.retarget / self.target *4 + # TODO: trim the hash of inactive workers + + def submit(self, connection_ref, current_difficulty, timestamp, worker_name): + ts = int(timestamp) + + # Init the stats for this worker if it isn't set. + if worker_name not in self.worker_stats : + self.worker_stats[worker_name] = {'last_rtc': (ts - self.retarget/2), 'last_ts': ts, 'buffer': SpeedBuffer(self.buffersize) } + return + + # Standard share update of data + self.worker_stats[worker_name]['buffer'].append(ts - self.worker_stats[worker_name]['last_ts']) + self.worker_stats[worker_name]['last_ts'] = ts + + # Do We retarget? If not, we're done. + if ts - self.worker_stats[worker_name]['last_rtc'] < self.retarget: + return + + # Set up and log our check + self.worker_stats[worker_name]['last_rtc'] = ts + avg = self.worker_stats[worker_name]['buffer'].avg() + log.info("Checking Retarget for %s avg. %i target %i+-%i" % (worker_name,avg,self.target,self.variance) ) + + # Figure out our Delta-Diff + ddiff = current_difficulty * (self.target / avg) + if (avg > self.tmax and current_difficulty > settings.POOL_TARGET): + if ddiff > -1: + ddiff = -1 + elif avg < self.tmin: + if ddiff < 1: + ddiff = 1 + else: # If we are here, then we should not be retargeting. + return + + # At this point we are retargeting this worker + new_diff = current_difficulty + ddiff + log.info("Retarget for %s %i old: %i new: %i" % (worker_name,ddiff,current_difficulty,new_diff)) + + self.worker_stats[worker_name]['buffer'].clear() + connection_ref().get_session()['difficulty'] = new_diff + connection_ref().rpc('mining.set_difficulty', [new_diff,], is_notification=True) + diff --git a/mining/interfaces.py b/mining/interfaces.py index e0404e3..5185876 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -27,7 +27,7 @@ def authorize(self, worker_name, worker_password): class ShareLimiterInterface(object): '''Implement difficulty adjustments here''' - def submit(self, connection_ref, current_difficulty, timestamp): + def submit(self, connection_ref, current_difficulty, timestamp, worker_name): '''connection - weak reference to Protocol instance current_difficulty - difficulty of the connection timestamp - submission time of current share diff --git a/mining/service.py b/mining/service.py index 2fa9798..5ce96f4 100644 --- a/mining/service.py +++ b/mining/service.py @@ -96,7 +96,7 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): submit_time = Interfaces.timestamper.time() ip = self.connection_ref()._get_ip() - Interfaces.share_limiter.submit(self.connection_ref, difficulty, submit_time) + Interfaces.share_limiter.submit(self.connection_ref, difficulty, submit_time, worker_name) # This checks if submitted share meet all requirements # and it is valid proof of work. From b52cbd940e3f46385eace15d0a9405555d4c20bf Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 15 Dec 2012 13:52:48 -0600 Subject: [PATCH 18/56] A couple changes to eliminate rare div/0 --- mining/basic_share_limiter.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 88b69da..8051dba 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -23,6 +23,8 @@ def pos(self): def clear(self): self.data=[] self.cur=0 + def size(self): + return self.cur class SpeedBufferFull: def __init__(self,n): @@ -38,6 +40,8 @@ def clear(self): self.data=[] self.cur=0 self.__class__ = SpeedBuffer + def size(self): + return self.max class BasicShareLimiter(object): def __init__(self): @@ -63,13 +67,16 @@ def submit(self, connection_ref, current_difficulty, timestamp, worker_name): self.worker_stats[worker_name]['last_ts'] = ts # Do We retarget? If not, we're done. - if ts - self.worker_stats[worker_name]['last_rtc'] < self.retarget: + if ts - self.worker_stats[worker_name]['last_rtc'] < self.retarget and self.worker_stats[worker_name]['buffer'].size() > 0: return # Set up and log our check self.worker_stats[worker_name]['last_rtc'] = ts avg = self.worker_stats[worker_name]['buffer'].avg() log.info("Checking Retarget for %s avg. %i target %i+-%i" % (worker_name,avg,self.target,self.variance) ) + if avg < 1: + log.info("Reseting avg = 1 since it's SOOO low") + avg = 1 # Figure out our Delta-Diff ddiff = current_difficulty * (self.target / avg) From b62e07673fe94dac43dab7bb9b6e91e17770c3be Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 15 Dec 2012 16:24:26 -0600 Subject: [PATCH 19/56] Variable diff changes, accept shares > prev diff after a retarget until job change --- lib/template_registry.py | 6 ++++-- mining/basic_share_limiter.py | 7 +++++-- mining/interfaces.py | 2 +- mining/service.py | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/template_registry.py b/lib/template_registry.py index 8841151..5742cde 100644 --- a/lib/template_registry.py +++ b/lib/template_registry.py @@ -161,7 +161,7 @@ def get_job(self, job_id): return j - def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, nonce, + def submit_share(self, job_id, worker_name, session, extranonce1_bin, extranonce2, ntime, nonce, difficulty): '''Check parameters and finalize block template. If it leads to valid block candidate, asynchronously submits the block @@ -225,7 +225,9 @@ def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, header_hex = binascii.hexlify(header_bin) target_user = self.diff_to_target(difficulty) - if hash_int > target_user: + if hash_int > target_user and \ + ( 'prev_jobid' not in session or session['prev_jobid'] < job_id \ + or 'prev_diff' not in session or hash_int > self.diff_to_target(session['prev_diff']) ): raise SubmitException("Share is above target") # Mostly for debugging purposes diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 8051dba..a81ccec 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -54,7 +54,7 @@ def __init__(self): self.buffersize = self.retarget / self.target *4 # TODO: trim the hash of inactive workers - def submit(self, connection_ref, current_difficulty, timestamp, worker_name): + def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_name): ts = int(timestamp) # Init the stats for this worker if it isn't set. @@ -94,6 +94,9 @@ def submit(self, connection_ref, current_difficulty, timestamp, worker_name): log.info("Retarget for %s %i old: %i new: %i" % (worker_name,ddiff,current_difficulty,new_diff)) self.worker_stats[worker_name]['buffer'].clear() - connection_ref().get_session()['difficulty'] = new_diff + session = connection_ref().get_session() + session['prev_diff'] = session['difficulty'] + session['prev_jobid'] = job_id + session['difficulty'] = new_diff connection_ref().rpc('mining.set_difficulty', [new_diff,], is_notification=True) diff --git a/mining/interfaces.py b/mining/interfaces.py index 5185876..cda3c42 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -27,7 +27,7 @@ def authorize(self, worker_name, worker_password): class ShareLimiterInterface(object): '''Implement difficulty adjustments here''' - def submit(self, connection_ref, current_difficulty, timestamp, worker_name): + def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_name): '''connection - weak reference to Protocol instance current_difficulty - difficulty of the connection timestamp - submission time of current share diff --git a/mining/service.py b/mining/service.py index 5ce96f4..922358d 100644 --- a/mining/service.py +++ b/mining/service.py @@ -96,13 +96,13 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): submit_time = Interfaces.timestamper.time() ip = self.connection_ref()._get_ip() - Interfaces.share_limiter.submit(self.connection_ref, difficulty, submit_time, worker_name) + Interfaces.share_limiter.submit(self.connection_ref, job_id, difficulty, submit_time, worker_name) # This checks if submitted share meet all requirements # and it is valid proof of work. try: (block_header, block_hash, share_diff, on_submit) = Interfaces.template_registry.submit_share(job_id, - worker_name, extranonce1_bin, extranonce2, ntime, nonce, difficulty) + worker_name, session, extranonce1_bin, extranonce2, ntime, nonce, difficulty) except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, From a858442a33f078030c5f16aba88506c1490c835f Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 16 Dec 2012 16:38:01 -0600 Subject: [PATCH 20/56] Important: Fixed errors when finding a block. There were a couple problems oops... --- TODO | 8 +++++--- mining/DBInterface.py | 10 ++++++---- mining/DB_Mysql.py | 12 ++++++------ mining/DB_Postgresql.py | 12 ++++++------ mining/DB_Sqlite.py | 10 +++++----- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/TODO b/TODO index 77673bf..afd5083 100644 --- a/TODO +++ b/TODO @@ -6,8 +6,10 @@ Share Archiving in DB Add a "script" to add,list,disable users -Implement a basic share limiter that increases difficulty until share time - is X seconds between shares - Flush all Pending shares on shutdown (I really don't know how to do this.) +create index if not exists shares_time_username ON shares(time,username) + +Variable difficulty should not be able to go higher than current difficulty + +Possibly allow non-local Coinbase (payaddress). (If we do, ensure we have to Disable checking in config) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index ae2529a..812b376 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -64,7 +64,7 @@ def run_import_thread(self): self.scheduleImport() def run_import(self): - self.do_import(self.dbi) + self.do_import(self.dbi,False) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() @@ -75,7 +75,7 @@ def run_import(self): def import_thread(self): # Here we are in the thread. dbi = self.connectDB() - self.do_import(dbi) + self.do_import(dbi,False) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() @@ -86,9 +86,10 @@ def _update_pool_info(self,data): self.dbi.update_pool_info({ 'blocks' : data['blocks'], 'balance' : data['balance'], 'connections' : data['connections'], 'difficulty' : data['difficulty'] }) - def do_import(self,dbi): + def do_import(self,dbi,force): # Only run if we have data - while self.q.qsize() >= settings.DB_LOADER_REC_MIN: + while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN: + force = False # Put together the data we want to import sqldata = [] datacnt = 0 @@ -113,6 +114,7 @@ def queue_share(self,data): def found_block(self,data): try: log.info("Updating Found Block Share Record") + self.do_import(self.dbi,True) # We can't Update if the record is not there. self.dbi.found_block(data) except Exception as e: log.error("Update Found Block Share Record Failed: %s", e.args[0]) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index dc12567..8a17aa6 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -85,17 +85,17 @@ def import_shares(self,data): def found_block(self,data): # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", + self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = FROM_UNIXTIME(%s) and username = %s limit 1", (data[5],data[2],data[4],data[0])) - if settings.DATABASE_EXTEND : - if data[5] == True: - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) + if settings.DATABASE_EXTEND and data[5] == True: + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) self.dbc.execute("select value from pool where parameter = 'pool_total_found'") total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.executemany("update pool set value = %s where paramter = %s",[(0,'round_shares'), + self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), (0,'round_progress'), + (0,'round_best_share'), (time.time(),'round_start'), - ([total_found],'pool_total_found') + (total_found,'pool_total_found') ]) self.dbh.commit() diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 93a7978..9c0a71e 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -87,17 +87,17 @@ def import_shares(self,data): def found_block(self,data): # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = %s and username = %s", + self.dbc.execute("update shares set upstream_result = %s, solution = %s where id in (select id from shares where time = to_timestamp(%s) and username = %s limit 1)", (bool(data[5]),data[2],data[4],data[0])) - if settings.DATABASE_EXTEND : - if data[5] == True: - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) + if settings.DATABASE_EXTEND and data[5] == True : + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0],)) self.dbc.execute("select value from pool where parameter = 'pool_total_found'") total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.executemany("update pool set value = %s where paramter = %s",[(0,'round_shares'), + self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), (0,'round_progress'), + (0,'round_best_share'), (time.time(),'round_start'), - ([total_found],'pool_total_found') + (total_found,'pool_total_found') ]) self.dbh.commit() diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index ef695c0..b286c35 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -95,15 +95,15 @@ def found_block(self,data): # Note: difficulty = -1 here self.dbc.execute("update shares set upstream_result = :usr, solution = :sol where time = :time and username = :user", {'usr':data[5],'sol':data[2],'time':data[4],'user':data[0]}) - if settings.DATABASE_EXTEND : - if data[5] == True: - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = :user",{'user':data[0]}) + if settings.DATABASE_EXTEND and data[5] == True : + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = :user",{'user':data[0]}) self.dbc.execute("select value from pool where parameter = 'pool_total_found'") total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.execute("update pool set value = :val where paramter = :parm", [{'val':0,'parm':'round_shares'}, + self.dbc.executemany("update pool set value = :val where parameter = :parm", [{'val':0,'parm':'round_shares'}, {'val':0,'parm':'round_progress'}, + {'val':0,'parm':'round_best_share'}, {'val':time.time(),'parm':'round_start'}, - {'val':[total_found],'parm':'pool_total_found'} + {'val':total_found,'parm':'pool_total_found'} ]) self.dbh.commit() From 28503e2451aaceeedc88c759cea299c520add02b Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 16 Dec 2012 17:05:21 -0600 Subject: [PATCH 21/56] Adding allowing of Non-Local Wallet address --- TODO | 2 +- conf/config_sample.py | 1 + lib/coinbaser.py | 11 ++++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/TODO b/TODO index afd5083..d06b4ae 100644 --- a/TODO +++ b/TODO @@ -12,4 +12,4 @@ create index if not exists shares_time_username ON shares(time,username) Variable difficulty should not be able to go higher than current difficulty -Possibly allow non-local Coinbase (payaddress). (If we do, ensure we have to Disable checking in config) +Test NON-Local Coinbase with testnet in a box diff --git a/conf/config_sample.py b/conf/config_sample.py index 3d300bc..f87eb05 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -98,6 +98,7 @@ # Transaction Settings CENTRAL_WALLET = 'set_valid_addresss_in_config!' # local bitcoin address where money goes COINBASE_EXTRAS = '/stratumPool/' # Extra Descriptive String to incorporate in solved blocks +ALLOW_NONLOCAL_WALLET = False # Allow valid, but NON-Local wallet's # Bitcoind communication polling settings (In Seconds) PREVHASH_REFRESH_INTERVAL = 5 # How often to check for new Blocks diff --git a/lib/coinbaser.py b/lib/coinbaser.py index f3fd14b..7647594 100644 --- a/lib/coinbaser.py +++ b/lib/coinbaser.py @@ -1,6 +1,8 @@ import util from twisted.internet import defer +from stratum import settings + import stratum.logger log = stratum.logger.get_logger('coinbaser') @@ -32,6 +34,13 @@ def _address_check(self, result): if not self.on_load.called: self.on_load.callback(True) + + elif result['isvalid'] and settings.ALLOW_NONLOCAL_WALLET == True : + self.is_valid = True + log.warning("!!! Coinbase address '%s' is valid BUT it is not local" % self.address) + + if not self.on_load.called: + self.on_load.callback(True) else: self.is_valid = False @@ -55,4 +64,4 @@ def get_script_pubkey(self): return util.script_to_address(self.address) def get_coinbase_data(self): - return '' \ No newline at end of file + return '' From 73fca6cd054a79ebfffa5db514be481cd59d5fe8 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Tue, 18 Dec 2012 09:33:35 -0600 Subject: [PATCH 22/56] Added getwork proxy into code... :) --- .gitmodules | 3 ++ conf/config_sample.py | 10 ++++ externals/stratum-mining-proxy | 1 + launcher.tac | 6 ++- lib/getwork_proxy.py | 84 ++++++++++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 externals/stratum-mining-proxy create mode 100755 lib/getwork_proxy.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a247f03 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "externals/stratum-mining-proxy"] + path = externals/stratum-mining-proxy + url = https://github.com/generalfault/stratum-mining-proxy.git diff --git a/conf/config_sample.py b/conf/config_sample.py index f87eb05..29e09b5 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -133,3 +133,13 @@ # (Disable if you have your own frontend) BASIC_STATS_PORT = 8889 # Port to listen on + +# ******************** Getwork Proxy Settings ********************* +# This enables a copy of slush's getwork proxy for old clients +# It will also auto-redirect new clients to the stratum interface +# so you can point ALL clients to: http://: + +GW_ENABLE = False # Enable the Proxy +GW_PORT = 8331 # Getwork Proxy Port +GW_DISABLE_MIDSTATE = False # Disable midstate's (Faster but breaks some clients) +GW_SEND_REAL_TARGET = False # Propigate >1 difficulty to Clients (breaks some clients) diff --git a/externals/stratum-mining-proxy b/externals/stratum-mining-proxy new file mode 160000 index 0000000..9b893fa --- /dev/null +++ b/externals/stratum-mining-proxy @@ -0,0 +1 @@ +Subproject commit 9b893fab15df318670cd137636df758dfcffa005 diff --git a/launcher.tac b/launcher.tac index c9ec635..3c2e448 100644 --- a/launcher.tac +++ b/launcher.tac @@ -3,7 +3,7 @@ # Add conf directory to python path. # Configuration file is standard python module. import os, sys -sys.path = [os.path.join(os.getcwd(), 'conf'),] + sys.path +sys.path = [os.path.join(os.getcwd(), 'conf'),os.path.join(os.getcwd(), 'externals', 'stratum-mining-proxy'),] + sys.path from twisted.internet import defer @@ -39,3 +39,7 @@ if settings.DATABASE_EXTEND == True and settings.BASIC_STATS == True : from lib.basic_stats import BasicStats BasicStats(on_startup) +if settings.GW_ENABLE == True : + from lib.getwork_proxy import GetworkProxy + GetworkProxy(on_startup) + diff --git a/lib/getwork_proxy.py b/lib/getwork_proxy.py new file mode 100755 index 0000000..7a76182 --- /dev/null +++ b/lib/getwork_proxy.py @@ -0,0 +1,84 @@ +import stratum.logger +log = stratum.logger.get_logger('Getwork Proxy') + +from stratum import settings +from stratum.socket_transport import SocketTransportClientFactory + +from twisted.internet import reactor, defer +from twisted.web import server + +from mining_libs import getwork_listener +from mining_libs import client_service +from mining_libs import jobs +from mining_libs import worker_registry +from mining_libs import version + +class Site(server.Site): + def log(self, request): + pass + +def on_shutdown(f): + log.info("Shutting down proxy...") + f.is_reconnecting = False # Don't let stratum factory to reconnect again + +@defer.inlineCallbacks +def on_connect(f, workers, job_registry): + log.info("Connected to Stratum pool at %s:%d" % f.main_host) + + # Hook to on_connect again + f.on_connect.addCallback(on_connect, workers, job_registry) + + # Every worker have to re-autorize + workers.clear_authorizations() + + # Subscribe for receiving jobs + log.info("Subscribing for mining jobs") + (_, extranonce1, extranonce2_size) = (yield f.rpc('mining.subscribe', [])) + job_registry.set_extranonce(extranonce1, extranonce2_size) + + defer.returnValue(f) + +def on_disconnect(f, workers, job_registry): + log.info("Disconnected from Stratum pool at %s:%d" % f.main_host) + f.on_disconnect.addCallback(on_disconnect, workers, job_registry) + + # Reject miners because we don't give a *job :-) + workers.clear_authorizations() + return f + +@defer.inlineCallbacks +def GetworkProxy_main(cb): + log.info("Stratum proxy version %s Connecting to Pool..." % version.VERSION) + + # Connect to Stratum pool + f = SocketTransportClientFactory(settings.HOSTNAME, settings.LISTEN_SOCKET_TRANSPORT, + debug=False, proxy=None, event_handler=client_service.ClientMiningService) + + job_registry = jobs.JobRegistry(f, cmd='', no_midstate=settings.GW_DISABLE_MIDSTATE, real_target=settings.GW_SEND_REAL_TARGET) + client_service.ClientMiningService.job_registry = job_registry + client_service.ClientMiningService.reset_timeout() + + workers = worker_registry.WorkerRegistry(f) + f.on_connect.addCallback(on_connect, workers, job_registry) + f.on_disconnect.addCallback(on_disconnect, workers, job_registry) + + # Cleanup properly on shutdown + reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown, f) + + # Block until proxy connects to the pool + yield f.on_connect + + # Setup getwork listener + gw_site = Site(getwork_listener.Root(job_registry, workers, + stratum_host=settings.HOSTNAME, stratum_port=settings.LISTEN_SOCKET_TRANSPORT, + custom_lp=False, custom_stratum=False, + custom_user=False, custom_password=False + )) + gw_site.noisy = False + reactor.listenTCP(settings.GW_PORT, gw_site, interface='0.0.0.0') + + log.info("Getwork Proxy is online, Port: %d" % (settings.GW_PORT)) + +def GetworkProxy(start_event): + start_event.addCallback(GetworkProxy_main) + From e7a2d5adf5fa3ea0d40d2c1e031e827ccea89495 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Tue, 18 Dec 2012 09:34:17 -0600 Subject: [PATCH 23/56] Silence the log lines from the stats pages --- lib/basic_stats.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/basic_stats.py b/lib/basic_stats.py index f6906ed..4a68344 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -1,7 +1,6 @@ from twisted.internet import reactor -from twisted.web.server import Site from twisted.web.resource import Resource -from twisted.web import static +from twisted.web import static,server import time from datetime import timedelta @@ -17,6 +16,10 @@ import locale locale.setlocale(locale.LC_ALL, '') +class Site(server.Site): + def log(self, request): + pass + class StatsPage(Resource): isLeaf = False cache_html = "" From 7ead758994a0b4a75fcea307d0a085f7ad018417 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Tue, 18 Dec 2012 09:49:24 -0600 Subject: [PATCH 24/56] Added script to update the submodules --- conf/config_sample.py | 2 +- update_submodules | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100755 update_submodules diff --git a/conf/config_sample.py b/conf/config_sample.py index 29e09b5..2298684 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -139,7 +139,7 @@ # It will also auto-redirect new clients to the stratum interface # so you can point ALL clients to: http://: -GW_ENABLE = False # Enable the Proxy +GW_ENABLE = False # Enable the Proxy (If enabled you MUST run update_submodules) GW_PORT = 8331 # Getwork Proxy Port GW_DISABLE_MIDSTATE = False # Disable midstate's (Faster but breaks some clients) GW_SEND_REAL_TARGET = False # Propigate >1 difficulty to Clients (breaks some clients) diff --git a/update_submodules b/update_submodules new file mode 100755 index 0000000..a57cd81 --- /dev/null +++ b/update_submodules @@ -0,0 +1,3 @@ +#!/bin/sh + +git submodule foreach git pull origin master From 1894176740eefc5c2ffe0db978ad3e1762c95e59 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Tue, 18 Dec 2012 22:10:56 -0600 Subject: [PATCH 25/56] Fixed vardiff adjusting down too slow --- mining/basic_share_limiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index a81ccec..250f1c7 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -79,7 +79,7 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n avg = 1 # Figure out our Delta-Diff - ddiff = current_difficulty * (self.target / avg) + ddiff = current_difficulty - (current_difficulty * (self.target / avg)) if (avg > self.tmax and current_difficulty > settings.POOL_TARGET): if ddiff > -1: ddiff = -1 From 9475884c600532aa1f0fda76d0ea542256db36c2 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sat, 22 Dec 2012 11:32:07 -0600 Subject: [PATCH 26/56] Variable difficulty should finally be fixed --- mining/basic_share_limiter.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 250f1c7..3d410ee 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -79,13 +79,20 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n avg = 1 # Figure out our Delta-Diff - ddiff = current_difficulty - (current_difficulty * (self.target / avg)) + ddiff = int( (float(current_difficulty) * (float(self.target) / float(avg))) - current_difficulty ) if (avg > self.tmax and current_difficulty > settings.POOL_TARGET): + # For fractional -0.1 ddiff's just drop by 1 if ddiff > -1: ddiff = -1 + # Don't drop below POOL_TARGET + if (ddiff + current_difficulty) < settings.POOL_TARGET: + ddiff = settings.POOL_TARGET - current_difficulty elif avg < self.tmin: + # For fractional 0.1 ddiff's just up by 1 if ddiff < 1: ddiff = 1 + # Don't go above BITCOIN_DIFF + #TODO else: # If we are here, then we should not be retargeting. return From 4f3afdd1e7cc7b26543bcd08f901b570c09f14d3 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Mon, 24 Dec 2012 08:44:58 -0600 Subject: [PATCH 27/56] Updated logging for variable diff --- mining/basic_share_limiter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 3d410ee..063d692 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -73,7 +73,8 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n # Set up and log our check self.worker_stats[worker_name]['last_rtc'] = ts avg = self.worker_stats[worker_name]['buffer'].avg() - log.info("Checking Retarget for %s avg. %i target %i+-%i" % (worker_name,avg,self.target,self.variance) ) + log.info("Checking Retarget for %s (%i) avg. %i target %i+-%i" % (worker_name,current_difficulty,avg, + self.target,self.variance) ) if avg < 1: log.info("Reseting avg = 1 since it's SOOO low") avg = 1 From e51222bfed37cfa8b5f9e6b0e888aa2bb6d65b8d Mon Sep 17 00:00:00 2001 From: Generalfault Date: Thu, 27 Dec 2012 13:14:36 -0600 Subject: [PATCH 28/56] Stats now show Variable difficulty, Share Archiving --- .gitignore | 1 + TODO | 6 ++-- conf/config_sample.py | 11 +++++++ lib/basic_stats.py | 12 ++++--- mining/DBInterface.py | 61 +++++++++++++++++++++++++++++++++-- mining/DB_Mysql.py | 54 ++++++++++++++++++++++++++++--- mining/DB_Postgresql.py | 54 ++++++++++++++++++++++++++++--- mining/DB_Sqlite.py | 54 ++++++++++++++++++++++++++++--- mining/basic_share_limiter.py | 5 +++ mining/interfaces.py | 6 ++-- 10 files changed, 236 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 39fc08d..2dd4327 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ conf/config.py NOTES run pooldb.sqlite +archives/* diff --git a/TODO b/TODO index d06b4ae..8094b0e 100644 --- a/TODO +++ b/TODO @@ -1,15 +1,15 @@ TODO File (in no particular order): SQL Connection pooling: sqlalchemy -Share Archiving in DB - - Perhaps separate table for found shares? Add a "script" to add,list,disable users Flush all Pending shares on shutdown (I really don't know how to do this.) -create index if not exists shares_time_username ON shares(time,username) +Verify and create indicies on the shares, pool, and pool_worker tables Variable difficulty should not be able to go higher than current difficulty Test NON-Local Coinbase with testnet in a box + +verify settings diff --git a/conf/config_sample.py b/conf/config_sample.py index 2298684..3ca0593 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -143,3 +143,14 @@ GW_PORT = 8331 # Getwork Proxy Port GW_DISABLE_MIDSTATE = False # Disable midstate's (Faster but breaks some clients) GW_SEND_REAL_TARGET = False # Propigate >1 difficulty to Clients (breaks some clients) + +# ******************** Archival Settings ********************* + +ARCHIVE_SHARES = False # Use share archiving? +ARCHIVE_DELAY = 86400 # Seconds after finding a share to archive all previous shares +ARCHIVE_MODE = 'file' # Do we archive to a file (file) , or to a database table (db) + +# Archive file options +ARCHIVE_FILE = 'archives/share_archive' # Name of the archive file ( .csv extension will be appended) +ARCHIVE_FILE_APPEND_TIME = True # Append the Date/Time to the end of the filename (must be true for bzip2 compress) +ARCHIVE_FILE_COMPRESS = 'none' # Method to compress file (none,gzip,bzip2) diff --git a/lib/basic_stats.py b/lib/basic_stats.py index 4a68344..8bc06a3 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -87,7 +87,7 @@ def render_GET(self, request): if size <= 10: r+="" r+="" - r+="" + r+="" for (w, wi) in enumerate(workers_stats): wd = workers_stats[wi] wc = "#A00" @@ -95,8 +95,9 @@ def render_GET(self, request): wc = "yellow" if wd["speed"] > 100: wc = "#0A0" - r+=""%( - wc,wi,format(int(wd["speed"]),"n"),format(int(wd["total_shares"]),"n"),format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + r+=""%( + wc,wi,format(int(wd["speed"]),"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), + format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) r+="
WorkerSpeedShares/RejFound
WorkerSpeed/DiffShares/RejFound
%s%s%s/%s%s
%s%s/%s%s/%s%s
" else : colcnt = int(size/3) + 1 @@ -111,8 +112,9 @@ def render_GET(self, request): wc = "yellow" if wd["speed"] > 100: wc = "#0A0" - r+="%s%s%s/%s%s"%( - wc,wi,format(int(wd["speed"]),"n"),format(int(wd["total_shares"]),"n"),format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + r+="%s%s/%s%s/%s%s"%( + wc,wi,format(int(wd["speed"]),"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), + format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) colt = colt - 1 if colt <= 0: r+="" diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 812b376..d91aab9 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -1,5 +1,6 @@ from twisted.internet import reactor, defer import time +from datetime import datetime import Queue from stratum import settings @@ -9,6 +10,11 @@ class DBInterface(): def __init__(self): + self.dbi = self.connectDB() + + def init_main(self): + self.dbi.check_tables() + self.q = Queue.Queue() self.queueclock = None @@ -17,8 +23,6 @@ def __init__(self): self.nextStatsUpdate = 0 - self.dbi = self.connectDB() - self.scheduleImport() def set_bitcoinrpc(self,bitcoinrpc): @@ -65,6 +69,8 @@ def run_import_thread(self): def run_import(self): self.do_import(self.dbi,False) + if settings.ARCHIVE_SHARES : + self.archive_shares(dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() @@ -76,6 +82,8 @@ def import_thread(self): # Here we are in the thread. dbi = self.connectDB() self.do_import(dbi,False) + if settings.ARCHIVE_SHARES : + self.archive_shares(dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() @@ -108,6 +116,45 @@ def do_import(self,dbi,force): self.q.put(v) break # Allows us to sleep a little + def archive_shares(self,dbi): + found_time = dbi.archive_check() + if found_time == 0: + return + log.info("Archiving shares newer than timestamp %f " % found_time) + dbi.archive_found(found_time) + if settings.ARCHIVE_MODE == 'db': + dbi.archive_to_db(found_time) + dbi.archive_cleanup(found_time) + elif settings.ARCHIVE_MODE == 'file': + shares = dbi.archive_get_shares(found_time) + + filename = settings.ARCHIVE_FILE + if settings.ARCHIVE_FILE_APPEND_TIME : + filename = filename + "-" + datetime.fromtimestamp(found_time).strftime("%Y-%m-%d-%H-%M-%S") + filename = filename + ".csv" + + if settings.ARCHIVE_FILE_COMPRESS == 'gzip' : + import gzip + filename = filename + ".gz" + filehandle = gzip.open(filename, 'a') + elif settings.ARCHIVE_FILE_COMPRESS == 'bzip2' and settings.ARCHIVE_FILE_APPEND_TIME : + import bz2 + filename = filename + ".bz2" + filehandle = bz2.BZFile(filename, mode='wb', buffering=4096 ) + else: + filehandle = open(filename, "a") + + while True: + row = shares.fetchone() + if row == None: + break + str1 = '","'.join([str(x) for x in row]) + filehandle.write('"%s"\n' % str1) + filehandle.close() + + + dbi.archive_cleanup(found_time) + def queue_share(self,data): self.q.put( data ) @@ -120,6 +167,9 @@ def found_block(self,data): log.error("Update Found Block Share Record Failed: %s", e.args[0]) def check_password(self,username,password): + if username == "": + log.info("Rejected worker for blank username") + return False wid = username+":-:"+password if wid in self.usercache : return True @@ -143,8 +193,15 @@ def update_user(self,username,password): self.usercache = {} return self.dbi.update_user(username,password) + def update_worker_diff(self,username,diff): + return self.dbi.update_worker_diff(username,diff) + def get_pool_stats(self): return self.dbi.get_pool_stats() def get_workers_stats(self): return self.dbi.get_workers_stats() + + def clear_worker_diff(self): + return self.dbi.clear_worker_diff() + diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 8a17aa6..99d619f 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -11,8 +11,6 @@ def __init__(self): self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST,settings.DB_MYSQL_USER,settings.DB_MYSQL_PASS,settings.DB_MYSQL_DBNAME) self.dbc = self.dbh.cursor() - self.check_tables() - def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. @@ -26,6 +24,30 @@ def updateStats(self,averageOverTime): self.dbc.execute("update pool_worker set speed = %s, alive = 1 where username = %s", (speed,name)) self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() + + def archive_check(self): + # Check for found shares to archive + self.dbc.execute("select time from shares where upstream_result = 1 order by time limit 1") + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : + return False + return data[0] + + def archive_found(self,found_time): + self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = 1 and time <= FROM_UNIXTIME(%s)", (found_time,)) + self.dbh.commit() + + def archive_to_db(self,found_time): + self.dbc.execute("insert into shares_archive select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + self.dbh.commit() + + def archive_cleanup(self,found_time): + self.dbc.execute("delete from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + self.dbh.commit() + + def archive_get_shares(self,found_time): + self.dbc.execute("select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + return self.dbc def import_shares(self,data): log.debug("Importing Shares") @@ -117,6 +139,15 @@ def update_user(self,username,password): (username, password )) self.dbh.commit() + def update_worker_diff(self,username,diff): + self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) + self.dbh.commit() + + def clear_worker_diff(self): + if settings.DATABASE_EXTEND == True : + self.dbc.execute("update pool_worker set difficulty = 0") + self.dbh.commit() + def check_password(self,username,password): log.debug("Checking Username/Password") self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", @@ -143,7 +174,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -152,7 +183,8 @@ def get_workers_stats(self): "total_shares" : data[3], "total_rejects" : data[4], "total_found" : data[5], - "alive" : data[6] } + "alive" : data[6], + "difficulty" : data[7] } return ret def check_tables(self): @@ -207,7 +239,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 4 + current_version = 5 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -242,3 +274,15 @@ def update_version_3(self): self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") self.dbh.commit() + def update_version_4(self): + log.info("running update 4") + self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") + self.dbc.execute("create table if not exists shares_archive " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + self.dbc.execute("create table if not exists shares_archive_found " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 9c0a71e..bd06a21 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -13,8 +13,6 @@ def __init__(self): # TODO -- set the schema self.dbc = self.dbh.cursor() - self.check_tables() - def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. @@ -28,6 +26,30 @@ def updateStats(self,averageOverTime): self.dbc.execute("update pool_worker set speed = %s, alive = 't' where username = %s", (speed,name)) self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) self.dbh.commit() + + def archive_check(self): + # Check for found shares to archive + self.dbc.execute("select time from shares where upstream_result = true order by time limit 1") + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : + return False + return data[0] + + def archive_found(self,found_time): + self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = true and time <= to_timestamp(%s)", [found_time]) + self.dbh.commit() + + def archive_to_db(self,found_time): + self.dbc.execute("insert into shares_archive select * from shares where time <= to_timestamp(%s)",[found_time]) + self.dbh.commit() + + def archive_cleanup(self,found_time): + self.dbc.execute("delete from shares where time <= to_timestamp(%s)",[found_time]) + self.dbh.commit() + + def archive_get_shares(self,found_time): + self.dbc.execute("select * from shares where time <= to_timestamp(%s)",[found_time]) + return self.dbc def import_shares(self,data): log.debug("Importing Shares") @@ -119,6 +141,15 @@ def update_user(self,username,password): (username, password )) self.dbh.commit() + def update_worker_diff(self,username,diff): + self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) + self.dbh.commit() + + def clear_worker_diff(self): + if settings.DATABASE_EXTEND == True : + self.dbc.execute("update pool_worker set difficulty = 0") + self.dbh.commit() + def check_password(self,username,password): log.debug("Checking Username/Password") self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", @@ -145,7 +176,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -154,7 +185,8 @@ def get_workers_stats(self): "total_shares" : data[3], "total_rejects" : data[4], "total_found" : data[5], - "alive" : data[6] } + "alive" : data[6], + "difficulty" : data[7] } return ret def check_tables(self): @@ -210,7 +242,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 4 + current_version = 5 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -245,3 +277,15 @@ def update_version_3(self): self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") self.dbh.commit() + def update_version_4(self): + log.info("running update 4") + self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") + self.dbc.execute("create table shares_archive" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create table shares_archive_found" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index b286c35..2ba2714 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -11,8 +11,6 @@ def __init__(self): self.dbh = sqlite3.connect(settings.DB_SQLITE_FILE) self.dbc = self.dbh.cursor() - self.check_tables() - def updateStats(self,averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. @@ -29,6 +27,30 @@ def updateStats(self,averageOverTime): self.dbc.execute("update pool set value = :val where parameter = 'pool_speed'",{'val':total_speed}) self.dbh.commit() + def archive_check(self): + # Check for found shares to archive + self.dbc.execute("select time from shares where upstream_result = 1 order by time limit 1") + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : + return False + return data[0] + + def archive_found(self,found_time): + self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = 1 and time <= :time",{'time':found_time}) + self.dbh.commit() + + def archive_to_db(self,found_time): + self.dbc.execute("insert into shares_archive select * from shares where time <= :time",{'time':found_time}) + self.dbh.commit() + + def archive_cleanup(self,found_time): + self.dbc.execute("delete from shares where time <= :time",{'time':found_time}) + self.dbh.commit() + + def archive_get_shares(self,found_time): + self.dbc.execute("select * from shares where time <= :time",{'time':found_time}) + return self.dbc + def import_shares(self,data): log.debug("Importing Shares") # 0 1 2 3 4 5 6 7 8 9 10 @@ -130,6 +152,15 @@ def check_password(self,username,password): return True return False + def update_worker_diff(self,username,diff): + self.dbc.execute("update pool_worker set difficulty = :diff where username = :user",{'diff':diff,'user':username}) + self.dbh.commit() + + def clear_worker_diff(self): + if settings.DATABASE_EXTEND == True : + self.dbc.execute("update pool_worker set difficulty = 0") + self.dbh.commit() + def update_pool_info(self,pi): self.dbc.executemany("update pool set value = :val where parameter = :parm",[{'val':pi['blocks'],'parm':"bitcoin_blocks"}, {'val':pi['balance'],'parm':"bitcoin_balance"}, @@ -147,7 +178,7 @@ def get_pool_stats(self): return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive from pool_worker") + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") ret = {} for data in self.dbc.fetchall(): ret[data[0]] = { "username" : data[0], @@ -156,7 +187,8 @@ def get_workers_stats(self): "total_shares" : data[3], "total_rejects" : data[4], "total_found" : data[5], - "alive" : data[6] } + "alive" : data[6], + "difficulty" : data[7] } return ret def check_tables(self): @@ -186,7 +218,7 @@ def check_tables(self): def update_tables(self): version = 0 - current_version = 4 + current_version = 5 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -221,3 +253,15 @@ def update_version_3(self): self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") self.dbh.commit() + def update_version_4(self): + log.info("running update 4") + self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") + self.dbc.execute("create table if not exists shares_archive" +\ + "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create table if not exists shares_archive_found" +\ + "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 063d692..882f777 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -3,6 +3,9 @@ import stratum.logger log = stratum.logger.get_logger('BasicShareLimiter') +import DBInterface +dbi = DBInterface.DBInterface() +dbi.clear_worker_diff() ''' This is just a cusomized ring buffer ''' class SpeedBuffer: @@ -60,6 +63,7 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n # Init the stats for this worker if it isn't set. if worker_name not in self.worker_stats : self.worker_stats[worker_name] = {'last_rtc': (ts - self.retarget/2), 'last_ts': ts, 'buffer': SpeedBuffer(self.buffersize) } + dbi.update_worker_diff(worker_name,settings.POOL_TARGET) return # Standard share update of data @@ -106,5 +110,6 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n session['prev_diff'] = session['difficulty'] session['prev_jobid'] = job_id session['difficulty'] = new_diff + dbi.update_worker_diff(worker_name,new_diff) connection_ref().rpc('mining.set_difficulty', [new_diff,], is_notification=True) diff --git a/mining/interfaces.py b/mining/interfaces.py index cda3c42..c7e2a79 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -12,6 +12,7 @@ import DBInterface dbi = DBInterface.DBInterface() +dbi.init_main() class WorkerManagerInterface(object): def __init__(self): @@ -34,9 +35,8 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n - raise SubmitException for stop processing this request - call mining.set_difficulty on connection to adjust the difficulty''' - - pass - + return dbi.update_worker_diff(worker_name,settings.POOL_TARGET) + class ShareManagerInterface(object): def __init__(self): # Fire deferred when manager is ready From deb93caf183c97294f26aef04ebc7c63c49255de Mon Sep 17 00:00:00 2001 From: Generalfault Date: Fri, 4 Jan 2013 12:49:33 -0600 Subject: [PATCH 29/56] fix of inital div/0 error, updating of indicies --- mining/DBInterface.py | 13 +++--- mining/DB_Mysql.py | 87 +++++++++++++++++++++++------------------ mining/DB_None.py | 3 ++ mining/DB_Postgresql.py | 87 +++++++++++++++++++++-------------------- mining/DB_Sqlite.py | 37 ++++++++++++++++-- 5 files changed, 137 insertions(+), 90 deletions(-) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index d91aab9..18d69f9 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -69,26 +69,27 @@ def run_import_thread(self): def run_import(self): self.do_import(self.dbi,False) - if settings.ARCHIVE_SHARES : - self.archive_shares(dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) - self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + if settings.ARCHIVE_SHARES : + self.archive_shares(dbi) self.scheduleImport() def import_thread(self): # Here we are in the thread. dbi = self.connectDB() self.do_import(dbi,False) - if settings.ARCHIVE_SHARES : - self.archive_shares(dbi) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) - self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + if settings.ARCHIVE_SHARES : + self.archive_shares(dbi) + dbi.close() def _update_pool_info(self,data): self.dbi.update_pool_info({ 'blocks' : data['blocks'], 'balance' : data['balance'], diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 99d619f..5489795 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -95,7 +95,10 @@ def import_shares(self,data): self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) - progress = (round_shares/difficulty)*100 + if difficulty == 0: + progress = 0 + else: + progress = (round_shares/difficulty)*100 self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) for k,v in checkin_times.items(): @@ -187,66 +190,60 @@ def get_workers_stats(self): "difficulty" : data[7] } return ret + def close(self): + self.dbh.close() + def check_tables(self): log.debug("Checking Tables") + # Do we have our tables? shares_exist = False - self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", - {"schema": settings.DB_MYSQL_DBNAME }) - data = self.dbc.fetchone() - if data[0] > 0 : - shares_exist = True - - pool_worker_exist = False - self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS where table_schema = %(schema)s and table_name = 'pool_worker' and index_name = 'pool_worker_username'", + self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS " +\ + "where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", {"schema": settings.DB_MYSQL_DBNAME }) data = self.dbc.fetchone() - if data[0] > 0 : - pool_worker_exist = True + if data[0] <= 0 : + self.update_version_1() # no, we don't, so create them + + if settings.DATABASE_EXTEND == True : + self.update_tables() + + def update_tables(self): + version = 0 + current_version = 6 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() + def update_version_1(self): if settings.DATABASE_EXTEND == True : self.dbc.execute("create table if not exists shares " +\ "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - if shares_exist == False: - self.dbc.execute("create index shares_username ON shares(username(10))") + self.dbc.execute("create index shares_username ON shares(username(10))") self.dbc.execute("create table if not exists pool_worker" +\ "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp" +\ ") ENGINE = MYISAM") - if pool_worker_exist == False: - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") - self.dbc.execute("select COUNT(*) from pool where parameter = 'DB Version'") - data = self.dbc.fetchone() - if data[0] <= 0: - self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") - self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") - self.update_tables() else : self.dbc.execute("create table if not exists shares" + \ "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") - if shares_exist == False: - self.dbc.execute("create index shares_username ON shares(username(10))") + self.dbc.execute("create index shares_username ON shares(username(10))") self.dbc.execute("create table if not exists pool_worker(id serial,username TEXT, password TEXT) ENGINE = MYISAM") - if pool_worker_exist == False: - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") self.dbh.commit() - - def update_tables(self): - version = 0 - current_version = 5 - while version < current_version : - self.dbc.execute("select value from pool where parameter = 'DB Version'") - data = self.dbc.fetchone() - version = int(data[0]) - if version < current_version : - log.info("Updating Database from %i to %i" % (version, version +1)) - getattr(self, 'update_version_' + str(version) )() def update_version_2(self): @@ -285,4 +282,18 @@ def update_version_4(self): "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") self.dbh.commit() + + def update_version_5(self): + log.info("running update 5") + # Adding Primary key to table: pool + self.dbc.execute("alter table pool add primary key (parameter(100))") + self.dbh.commit() + # Adjusting indicies on table: shares + self.dbc.execute("DROP INDEX shares_username ON shares") + self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username(10))") + self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") + self.dbh.commit() + self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") + self.dbh.commit() + diff --git a/mining/DB_None.py b/mining/DB_None.py index 5d14c5d..eda5121 100644 --- a/mining/DB_None.py +++ b/mining/DB_None.py @@ -42,4 +42,7 @@ def get_workers_stats(self): def check_tables(self): log.debug("Checking Tables") + + def close(self): + log.debug("Close Connection") diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index bd06a21..959586c 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -97,7 +97,10 @@ def import_shares(self,data): self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) - progress = (round_shares/difficulty)*100 + if difficulty == 0: + progress = 0 + else: + progress = (round_shares/difficulty)*100 self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) for k,v in checkin_times.items(): @@ -189,6 +192,9 @@ def get_workers_stats(self): "difficulty" : data[7] } return ret + def close(self): + self.dbh.close() + def check_tables(self): log.debug("Checking Tables") @@ -196,53 +202,16 @@ def check_tables(self): self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'shares'", {"schema": settings.DB_PGSQL_SCHEMA }) data = self.dbc.fetchone() - if data[0] > 0 : - shares_exist = True - - pool_worker_exist = False - self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'pool_worker'", - {"schema": settings.DB_PGSQL_SCHEMA }) - data = self.dbc.fetchone() - if data[0] > 0 : - pool_worker_exist = True + if data[0] <= 0 : + self.update_version_1() - pool_exist = False - self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'pool'", - {"schema": settings.DB_PGSQL_SCHEMA }) - data = self.dbc.fetchone() - if data[0] > 0 : - pool_exist = True - if settings.DATABASE_EXTEND == True : - if shares_exist == False: - self.dbc.execute("create table shares" +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") - self.dbc.execute("create index shares_username ON shares(username)") - if pool_worker_exist == False: - self.dbc.execute("create table pool_worker" +\ - "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") - self.dbc.execute("create index pool_worker_username ON pool_worker(username)") - if pool_exist == False: - self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") - self.dbc.execute("create table pool(parameter TEXT, value TEXT)") - self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") self.update_tables() - else : - if shares_exist == False: - self.dbc.execute("create table shares" + \ - "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT)") - self.dbc.execute("create index shares_username ON shares(username)") - if pool_worker_exist == False: - self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") - self.dbc.execute("create index pool_worker_username ON pool_worker(username)") - self.dbh.commit() + def update_tables(self): version = 0 - current_version = 5 + current_version = 6 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -251,6 +220,27 @@ def update_tables(self): log.info("Updating Database from %i to %i" % (version, version +1)) getattr(self, 'update_version_' + str(version) )() + def update_version_1(self): + if settings.DATABASE_EXTEND == True : + self.dbc.execute("create table shares" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create index shares_username ON shares(username)") + self.dbc.execute("create table pool_worker" +\ + "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("create table pool(parameter TEXT, value TEXT)") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + else : + self.dbc.execute("create table shares" + \ + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT)") + self.dbc.execute("create index shares_username ON shares(username)") + self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + self.dbh.commit() def update_version_2(self): log.info("running update 2") @@ -289,3 +279,16 @@ def update_version_4(self): self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") self.dbh.commit() + def update_version_5(self): + log.info("running update 5") + # Adding Primary key to table: pool + self.dbc.execute("alter table pool add primary key (parameter)") + self.dbh.commit() + # Adjusting indicies on table: shares + self.dbc.execute("DROP INDEX shares_username") + self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username)") + self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") + self.dbh.commit() + + self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") + self.dbh.commit() diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 2ba2714..41ea34e 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -99,7 +99,10 @@ def import_shares(self,data): self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") difficulty = float(self.dbc.fetchone()[0]) - progress = (round_shares/difficulty)*100 + if difficulty == 0: + progress = 0 + else: + progress = (round_shares/difficulty)*100 self.dbc.execute("update pool set value = :val where parameter = 'round_progress'",{'val':progress}) sqldata = [] @@ -191,6 +194,9 @@ def get_workers_stats(self): "difficulty" : data[7] } return ret + def close(self): + self.dbh.close() + def check_tables(self): log.debug("Checking Tables") if settings.DATABASE_EXTEND == True : @@ -213,12 +219,11 @@ def check_tables(self): self.dbc.execute("create table if not exists shares" + \ "(time DATETIME,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT)") self.dbc.execute("create table if not exists pool_worker(username TEXT, password TEXT)") - self.dbc.execute("create index if not exists shares_username ON shares(username)") - self.dbc.execute("create index if not exists pool_worker_username ON pool_worker(username)") + self.dbc.execute("create index if not exists pool_worker_username ON pool_worker(username)") def update_tables(self): version = 0 - current_version = 5 + current_version = 6 while version < current_version : self.dbc.execute("select value from pool where parameter = 'DB Version'") data = self.dbc.fetchone() @@ -240,6 +245,8 @@ def update_version_2(self): ('round_progress',0), ('round_start',time.time()) ]) + self.dbc.execute("create index if not exists shares_username ON shares(username)") + self.dbc.execute("create index if not exists pool_worker_username ON pool_worker(username)") self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") self.dbh.commit() @@ -265,3 +272,25 @@ def update_version_4(self): self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") self.dbh.commit() + def update_version_5(self): + log.info("running update 5") + # Adding Primary key to table: pool + self.dbc.execute("alter table pool rename to pool_old") + self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT, primary key(parameter))") + self.dbc.execute("insert into pool select * from pool_old") + self.dbc.execute("drop table pool_old") + self.dbh.commit() + # Adding Primary key to table: pool_worker + self.dbc.execute("alter table pool_worker rename to pool_worker_old") + self.dbc.execute("CREATE TABLE pool_worker(username TEXT, password TEXT, speed INTEGER, last_checkin DATETIME, total_shares INTEGER default 0, total_rejects INTEGER default 0, total_found INTEGER default 0, alive INTEGER default 0, difficulty INTEGER default 0, primary key(username))") + self.dbc.execute("insert into pool_worker select * from pool_worker_old") + self.dbc.execute("drop table pool_worker_old") + self.dbh.commit() + # Adjusting indicies on table: shares + self.dbc.execute("DROP INDEX shares_username") + self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username)") + self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") + self.dbh.commit() + + self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") + self.dbh.commit() From 27d25a879887b30d6dd81c927ed05c4920f5b06b Mon Sep 17 00:00:00 2001 From: Generalfault Date: Mon, 14 Jan 2013 18:58:00 -0600 Subject: [PATCH 30/56] Vardif should notify miner before db (in case db is misbehaving) --- mining/basic_share_limiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 882f777..1dc3450 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -110,6 +110,6 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n session['prev_diff'] = session['difficulty'] session['prev_jobid'] = job_id session['difficulty'] = new_diff - dbi.update_worker_diff(worker_name,new_diff) connection_ref().rpc('mining.set_difficulty', [new_diff,], is_notification=True) + dbi.update_worker_diff(worker_name,new_diff) From 233d797e4a480aa69174a470c32bdc95b1ca8792 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 20 Jan 2013 18:55:45 -0600 Subject: [PATCH 31/56] Basic E-Mail notifications. --- TODO | 2 ++ conf/config_sample.py | 11 +++++++++++ lib/notify_email.py | 39 +++++++++++++++++++++++++++++++++++++++ mining/interfaces.py | 11 +++++++++++ 4 files changed, 63 insertions(+) create mode 100644 lib/notify_email.py diff --git a/TODO b/TODO index 8094b0e..8b612dc 100644 --- a/TODO +++ b/TODO @@ -13,3 +13,5 @@ Variable difficulty should not be able to go higher than current difficulty Test NON-Local Coinbase with testnet in a box verify settings + +send e-mail on dead miner diff --git a/conf/config_sample.py b/conf/config_sample.py index 3ca0593..ea72537 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -154,3 +154,14 @@ ARCHIVE_FILE = 'archives/share_archive' # Name of the archive file ( .csv extension will be appended) ARCHIVE_FILE_APPEND_TIME = True # Append the Date/Time to the end of the filename (must be true for bzip2 compress) ARCHIVE_FILE_COMPRESS = 'none' # Method to compress file (none,gzip,bzip2) + +# ******************** E-Mail Notification Settings ********************* + +NOTIFY_EMAIL_TO = '' # Where to send Start/Found block notifications +NOTIFY_EMAIL_TO_DEADMINER = '' # Where to send dead miner notifications +NOTIFY_EMAIL_FROM = 'root@localhost' # Sender address +NOTIFY_EMAIL_SERVER = 'localhost' # E-Mail Sender +NOTIFY_EMAIL_USERNAME = '' # E-Mail server SMTP Logon +NOTIFY_EMAIL_PASSWORD = '' +NOTIFY_EMAIL_USETLS = True + diff --git a/lib/notify_email.py b/lib/notify_email.py new file mode 100644 index 0000000..4c70a4f --- /dev/null +++ b/lib/notify_email.py @@ -0,0 +1,39 @@ +import smtplib +from email.mime.text import MIMEText + +from stratum import settings + +import stratum.logger +log = stratum.logger.get_logger('Notify_Email') + +class NOTIFY_EMAIL(): + + def notify_start(self): + if settings.NOTIFY_EMAIL_TO != '': + self.send_email(settings.NOTIFY_EMAIL_TO,'Stratum Server Started','Stratum server has started!') + + def notify_found_block(self,worker_name): + if settings.NOTIFY_EMAIL_TO != '': + text = '%s on Stratum server found a block!' % worker_name + self.send_email(settings.NOTIFY_EMAIL_TO,'Stratum Server Found Block',text) + + def send_email(self,to,subject,message): + msg = MIMEText(message) + msg['Subject'] = subject + msg['From'] = settings.NOTIFY_EMAIL_FROM + msg['To'] = to + try: + s = smtplib.SMTP(settings.NOTIFY_EMAIL_SERVER) + if settings.NOTIFY_EMAIL_USERNAME != '': + if settings.NOTIFY_EMAIL_USETLS: + s.ehlo() + s.starttls() + s.ehlo() + s.login(settings.NOTIFY_EMAIL_USERNAME, settings.NOTIFY_EMAIL_PASSWORD) + s.sendmail(settings.NOTIFY_EMAIL_FROM,to,msg.as_string()) + s.quit() + except smtplib.SMTPAuthenticationError as e: + log.error('Error sending Email: %s' % e[1]) + except Exception as e: + log.error('Error sending Email: %s' % e[0]) + diff --git a/mining/interfaces.py b/mining/interfaces.py index c7e2a79..fcabf04 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -10,6 +10,8 @@ import stratum.logger log = stratum.logger.get_logger('interfaces') +import lib.notify_email + import DBInterface dbi = DBInterface.DBInterface() dbi.init_main() @@ -44,6 +46,10 @@ def __init__(self): self.on_load.callback(True) self.block_height = 0 self.prev_hash = 0 + + # Send out the e-mail saying we are starting. + notify_email = lib.notify_email.NOTIFY_EMAIL() + notify_email.notify_start() def on_network_block(self, prevhash, block_height): '''Prints when there's new block coming from the network (possibly new round)''' @@ -59,6 +65,11 @@ def on_submit_share(self, worker_name, block_header, block_hash, difficulty, tim def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip, share_diff ): log.info("Block %s %s" % (block_hash, 'ACCEPTED' if is_accepted else 'REJECTED')) dbi.found_block([worker_name,block_header,block_hash,-1,timestamp,is_accepted,ip,self.block_height, self.prev_hash, share_diff ]) + + # Send out the e-mail saying we found a block. + if is_accepted: + notify_email = lib.notify_email.NOTIFY_EMAIL() + notify_email.notify_found_block(worker_name) class TimestamperInterface(object): '''This is the only source for current time in the application. From 3315f290295e3ab68d6f29035d7a0ab9369b32c8 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 20 Jan 2013 19:06:20 -0600 Subject: [PATCH 32/56] fix multi-recipient e-mails --- lib/notify_email.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/notify_email.py b/lib/notify_email.py index 4c70a4f..2e8867f 100644 --- a/lib/notify_email.py +++ b/lib/notify_email.py @@ -10,12 +10,17 @@ class NOTIFY_EMAIL(): def notify_start(self): if settings.NOTIFY_EMAIL_TO != '': - self.send_email(settings.NOTIFY_EMAIL_TO,'Stratum Server Started','Stratum server has started!') + self.send_emails(settings.NOTIFY_EMAIL_TO,'Stratum Server Started','Stratum server has started!') def notify_found_block(self,worker_name): if settings.NOTIFY_EMAIL_TO != '': text = '%s on Stratum server found a block!' % worker_name - self.send_email(settings.NOTIFY_EMAIL_TO,'Stratum Server Found Block',text) + self.send_emails(settings.NOTIFY_EMAIL_TO,'Stratum Server Found Block',text) + + def send_emails(self,to,subject,message): + tos = to.split(";") + for tov in tos: + self.send_email(tov,subject,message) def send_email(self,to,subject,message): msg = MIMEText(message) From 622d4857b0d6ed3673cc0b06a7ffb0332b061197 Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 27 Jan 2013 10:35:43 -0600 Subject: [PATCH 33/56] Sqlite threading breaks things, so disable it. Archiving becomes stable. Doc updates. --- INSTALL | 15 +++++++++------ TODO | 8 ++++++-- conf/config_sample.py | 22 ++++++++++++++++------ mining/DBInterface.py | 30 ++++++++++++++++++++++-------- mining/DB_Sqlite.py | 1 + 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/INSTALL b/INSTALL index fd7224c..281a544 100644 --- a/INSTALL +++ b/INSTALL @@ -17,10 +17,8 @@ Step 2: Pull a copy of the miner Step 3: Configure the Miner cp conf/config_sample.py conf/config.py make your changes to conf/config.py - Make sure you set: - CENTRAL_WALLET - BITCOIN_TRUSTED_USER - BITCOIN_TRUSTED_PASSWORD + Make sure you set the values in BASIC SETTINGS! These are how to connect to bitcoind + and where your money goes! Step 4: Run the pool twistd -ny launcher.tac -l - @@ -64,8 +62,13 @@ None: Well, this doesn't do anything, so there is nothing to set up Sqlite: -Just set the file path in the config file. -Support for sqlite3 is built into recent python versions +THIS IS THE DEFAULT! +Just set the file path in the config file (or keep the default.) +Support for sqlite3 is built into recent python versions. +A couple notes for Sqlite: + - Sqlite and threading/concurancy just doesn't work right for that reason it is disabled. + - Since threading is disabled, The server will "pause" when archiving happens, this will affect + your miners. However this will not happen often (24 hours after finding a share) Postgresql: 1: Set up your parameters in the config file. diff --git a/TODO b/TODO index 8b612dc..6bd8f47 100644 --- a/TODO +++ b/TODO @@ -6,8 +6,6 @@ Add a "script" to add,list,disable users Flush all Pending shares on shutdown (I really don't know how to do this.) -Verify and create indicies on the shares, pool, and pool_worker tables - Variable difficulty should not be able to go higher than current difficulty Test NON-Local Coinbase with testnet in a box @@ -15,3 +13,9 @@ Test NON-Local Coinbase with testnet in a box verify settings send e-mail on dead miner + +multiple bitcoind + +send e-mail on dead bitcoind + + diff --git a/conf/config_sample.py b/conf/config_sample.py index ea72537..bd84f36 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -1,8 +1,21 @@ ''' This is example configuration for Stratum server. Please rename it to config.py and fill correct values. + +This is already setup with sane values for solomining. +You NEED to set the parameters in BASIC SETTINGS ''' +# ******************** BASIC SETTINGS *************** +# These are the MUST BE SET parameters! + +CENTRAL_WALLET = 'set_valid_addresss_in_config!' # local bitcoin address where money goes + +BITCOIN_TRUSTED_HOST = 'localhost' +BITCOIN_TRUSTED_PORT = 8332 +BITCOIN_TRUSTED_USER = 'user' +BITCOIN_TRUSTED_PASSWORD = 'somepassword' + # ******************** GENERAL SETTINGS *************** # Enable some verbose debug (logging requests and responses). @@ -46,10 +59,7 @@ # Hostname and credentials for one trusted Bitcoin node ("Satoshi's client"). # Stratum uses both P2P port (which is 8333 already) and RPC port -BITCOIN_TRUSTED_HOST = 'localhost' -BITCOIN_TRUSTED_PORT = 8332 -BITCOIN_TRUSTED_USER = 'user' -BITCOIN_TRUSTED_PASSWORD = 'somepassword' +# BITCOIN_TRUSTED_* -- in basic settings above # Use scripts/generateAdminHash.sh to generate the hash # for calculating SHA256 of your preferred password @@ -96,7 +106,7 @@ USERS_CHECK_PASSWORD = False # Check the workers password? (Many pools don't) # Transaction Settings -CENTRAL_WALLET = 'set_valid_addresss_in_config!' # local bitcoin address where money goes +# CENTRAL_WALLET ---- In basic settings at top COINBASE_EXTRAS = '/stratumPool/' # Extra Descriptive String to incorporate in solved blocks ALLOW_NONLOCAL_WALLET = False # Allow valid, but NON-Local wallet's @@ -117,7 +127,7 @@ POOL_TARGET = 1 # Pool-wide difficulty target int >= 1 # Variable Difficulty Enable -VARIABLE_DIFF = False # Master variable difficulty enable +VARIABLE_DIFF = True # Master variable difficulty enable # Variable diff tuning variables VDIFF_TARGET = 30 # Target time per share (i.e. try to get 1 share per this many seconds) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 18d69f9..4152e6e 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -58,9 +58,14 @@ def clearusercache(self): def scheduleImport(self): # This schedule's the Import - # If you don't want to use threads change - # self.run_import_thread to self.run_import - self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import_thread) + use_thread = True + if settings.DATABASE_DRIVER == "sqlite": + use_thread = False + + if use_thread: + self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import_thread) + else: + self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import) def run_import_thread(self): if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run @@ -71,11 +76,11 @@ def run_import(self): self.do_import(self.dbi,False) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME - dbi.updateStats(settings.DB_STATS_AVG_TIME) + self.dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) if settings.ARCHIVE_SHARES : - self.archive_shares(dbi) + self.archive_shares(self.dbi) self.scheduleImport() def import_thread(self): @@ -120,7 +125,7 @@ def do_import(self,dbi,force): def archive_shares(self,dbi): found_time = dbi.archive_check() if found_time == 0: - return + return False log.info("Archiving shares newer than timestamp %f " % found_time) dbi.archive_found(found_time) if settings.ARCHIVE_MODE == 'db': @@ -152,9 +157,18 @@ def archive_shares(self,dbi): str1 = '","'.join([str(x) for x in row]) filehandle.write('"%s"\n' % str1) filehandle.close() - - dbi.archive_cleanup(found_time) + clean = False + while not clean: + try: + dbi.archive_cleanup(found_time) + clean = True + except Exception as e: + clean = False + log.error("Archive Cleanup Failed... will retry to cleanup in 30 seconds") + sleep(30) + + return True def queue_share(self,data): self.q.put( data ) diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 41ea34e..9e1d1d9 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -45,6 +45,7 @@ def archive_to_db(self,found_time): def archive_cleanup(self,found_time): self.dbc.execute("delete from shares where time <= :time",{'time':found_time}) + self.dbc.execute("vacuum") self.dbh.commit() def archive_get_shares(self,found_time): From bf789ef493acca8eb3d7a1cc77396da647aff04c Mon Sep 17 00:00:00 2001 From: Generalfault Date: Sun, 24 Feb 2013 16:17:14 -0600 Subject: [PATCH 34/56] Added support for backup bitcoind backends (yes, I have had bitcoind crash a couple times) --- TODO | 2 - conf/config_sample.py | 14 +++++ lib/bitcoin_rpc_manager.py | 123 +++++++++++++++++++++++++++++++++++++ mining/__init__.py | 21 +------ 4 files changed, 139 insertions(+), 21 deletions(-) create mode 100644 lib/bitcoin_rpc_manager.py diff --git a/TODO b/TODO index 6bd8f47..a6318db 100644 --- a/TODO +++ b/TODO @@ -14,8 +14,6 @@ verify settings send e-mail on dead miner -multiple bitcoind - send e-mail on dead bitcoind diff --git a/conf/config_sample.py b/conf/config_sample.py index bd84f36..fa2deba 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -16,6 +16,20 @@ BITCOIN_TRUSTED_USER = 'user' BITCOIN_TRUSTED_PASSWORD = 'somepassword' +# ******************** BASIC SETTINGS *************** +# Backup Bitcoind connections (consider having at least 1 backup) +# You can have up to 99 + +#BITCOIN_TRUSTED_HOST_1 = 'localhost' +#BITCOIN_TRUSTED_PORT_1 = 8332 +#BITCOIN_TRUSTED_USER_1 = 'user' +#BITCOIN_TRUSTED_PASSWORD_1 = 'somepassword' + +#BITCOIN_TRUSTED_HOST_2 = 'localhost' +#BITCOIN_TRUSTED_PORT_2 = 8332 +#BITCOIN_TRUSTED_USER_2 = 'user' +#BITCOIN_TRUSTED_PASSWORD_2 = 'somepassword' + # ******************** GENERAL SETTINGS *************** # Enable some verbose debug (logging requests and responses). diff --git a/lib/bitcoin_rpc_manager.py b/lib/bitcoin_rpc_manager.py new file mode 100644 index 0000000..257a949 --- /dev/null +++ b/lib/bitcoin_rpc_manager.py @@ -0,0 +1,123 @@ +''' + Implements simple interface to bitcoind's RPC. +''' + + +import simplejson as json +from twisted.internet import defer + +from stratum import settings + +import time + +import stratum.logger +log = stratum.logger.get_logger('bitcoin_rpc_manager') + +from lib.bitcoin_rpc import BitcoinRPC + + +class BitcoinRPCManager(object): + + def __init__(self): + self.conns = {} + self.conns[0] = BitcoinRPC(settings.BITCOIN_TRUSTED_HOST, + settings.BITCOIN_TRUSTED_PORT, + settings.BITCOIN_TRUSTED_USER, + settings.BITCOIN_TRUSTED_PASSWORD) + self.curr_conn = 0 + for x in range (1, 99): + if hasattr(settings, 'BITCOIN_TRUSTED_HOST_' + str(x)) and hasattr(settings, 'BITCOIN_TRUSTED_PORT_' + str(x)) and hasattr(settings, 'BITCOIN_TRUSTED_USER_' + str(x)) and hasattr(settings, 'BITCOIN_TRUSTED_PASSWORD_' + str(x)): + self.conns[len(self.conns)] = BitcoinRPC(settings.__dict__['BITCOIN_TRUSTED_HOST_' + str(x)], + settings.__dict__['BITCOIN_TRUSTED_PORT_' + str(x)], + settings.__dict__['BITCOIN_TRUSTED_USER_' + str(x)], + settings.__dict__['BITCOIN_TRUSTED_PASSWORD_' + str(x)]) + + def next_connection(self): + time.sleep(1) + if len(self.conns) <= 1: + log.error("Problem with Pool 0 -- NO ALTERNATE POOLS!!!") + time.sleep(4) + return + log.error("Problem with Pool %i Switching to Next!" % (self.curr_conn) ) + self.curr_conn = self.curr_conn + 1 + if self.curr_conn >= len(self.conns): + self.curr_conn = 0 + + @defer.inlineCallbacks + def check_height(self): + while True: + try: + resp = (yield self.conns[self.curr_conn]._call('getinfo', [])) + break + except: + log.error("Check Height -- Pool %i Down!" % (self.curr_conn) ) + self.next_connection() + curr_height = json.loads(resp)['result']['blocks'] + log.debug("Check Height -- Current Pool %i : %i" % (self.curr_conn,curr_height) ) + for i in self.conns: + if i == self.curr_conn: + continue + + try: + resp = (yield self.conns[i]._call('getinfo', [])) + except: + log.error("Check Height -- Pool %i Down!" % (i,) ) + continue + + height = json.loads(resp)['result']['blocks'] + log.debug("Check Height -- Pool %i : %i" % (i,height) ) + if height > curr_height: + self.curr_conn = i + defer.returnValue(True) + + def _call_raw(self, data): + while True: + try: + return self.conns[self.curr_conn]._call_raw(data) + except: + self.next_connection() + + def _call(self, method, params): + while True: + try: + return self.conns[self.curr_conn]._call(method,params) + except: + self.next_connection() + + def submitblock(self, block_hex): + while True: + try: + return self.conns[self.curr_conn].submitblock(block_hex) + except: + self.next_connection() + + def getinfo(self): + while True: + try: + return self.conns[self.curr_conn].getinfo() + except: + self.next_connection() + + def getblocktemplate(self): + while True: + try: + return self.conns[self.curr_conn].getblocktemplate() + except: + self.next_connection() + + def prevhash(self): + self.check_height() + while True: + try: + return self.conns[self.curr_conn].prevhash() + except: + self.next_connection() + + def validateaddress(self, address): + while True: + try: + return self.conns[self.curr_conn].validateaddress(address) + except: + self.next_connection() + + diff --git a/mining/__init__.py b/mining/__init__.py index b992115..2707c1c 100644 --- a/mining/__init__.py +++ b/mining/__init__.py @@ -26,28 +26,11 @@ def setup(on_startup): from lib.block_updater import BlockUpdater from lib.template_registry import TemplateRegistry - from lib.bitcoin_rpc import BitcoinRPC + from lib.bitcoin_rpc_manager import BitcoinRPCManager from lib.block_template import BlockTemplate from lib.coinbaser import SimpleCoinbaser - bitcoin_rpc = BitcoinRPC(settings.BITCOIN_TRUSTED_HOST, - settings.BITCOIN_TRUSTED_PORT, - settings.BITCOIN_TRUSTED_USER, - settings.BITCOIN_TRUSTED_PASSWORD) - - import stratum.logger - log = stratum.logger.get_logger('mining') - - log.info('Waiting for bitcoin RPC...') - - while True: - try: - result = (yield bitcoin_rpc.getblocktemplate()) - if isinstance(result, dict): - log.info('Response from bitcoin RPC OK') - break - except: - time.sleep(1) + bitcoin_rpc = BitcoinRPCManager() # Check bitcoind # Check we can connect (sleep) From 54e3d1e3f5ac49361e45d9823b125c6107f2bc2c Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Mon, 22 Apr 2013 14:15:05 +0100 Subject: [PATCH 35/56] MySQL Optimisations and SHA1 passwords Optimised MySQL pool and pool_worker tables. Added SHA1'ing of passwords for MySQL, salt is in config.py. Reformatted DB_Mysql.py to use all whitespace. --- conf/config_sample.py | 3 + mining/DB_Mysql.py | 525 +++++++++++++++++++++++------------------- 2 files changed, 287 insertions(+), 241 deletions(-) diff --git a/conf/config_sample.py b/conf/config_sample.py index fa2deba..9facb16 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -82,6 +82,9 @@ IRC_NICK = None +# Salt used when hashing passwords +PASSWORD_SALT = 'some_crazy_string' + # ******************** Database ********************* DATABASE_DRIVER = 'sqlite' # Options: none, sqlite, postgresql or mysql diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 5489795..05daa06 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -1,4 +1,5 @@ import time +import hashlib from stratum import settings import stratum.logger log = stratum.logger.get_logger('DB_Mysql') @@ -7,293 +8,335 @@ class DB_Mysql(): def __init__(self): - log.debug("Connecting to DB") - self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST,settings.DB_MYSQL_USER,settings.DB_MYSQL_PASS,settings.DB_MYSQL_DBNAME) - self.dbc = self.dbh.cursor() + log.debug("Connecting to DB") + self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST,settings.DB_MYSQL_USER,settings.DB_MYSQL_PASS,settings.DB_MYSQL_DBNAME) + self.dbc = self.dbh.cursor() + + if hasattr(settings, 'PASSWORD_SALT'): + self.salt = settings.PASSWORD_SALT + else: + raise Exception("PASSWORD_SALT isn't set, please set in config.py") def updateStats(self,averageOverTime): - log.debug("Updating Stats") - # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0, alive = 0"); - stime = '%.0f' % ( time.time() - averageOverTime ); - self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) - total_speed = 0 - for name,shares in self.dbc.fetchall(): - speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) - total_speed += speed - self.dbc.execute("update pool_worker set speed = %s, alive = 1 where username = %s", (speed,name)) - self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) - self.dbh.commit() + log.debug("Updating Stats") + # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. + self.dbc.execute("update pool_worker set speed = 0, alive = 0"); + stime = '%.0f' % ( time.time() - averageOverTime ); + self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) + total_speed = 0 + for name,shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + total_speed += speed + self.dbc.execute("update pool_worker set speed = %s, alive = 1 where username = %s", (speed,name)) + self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) + self.dbh.commit() def archive_check(self): - # Check for found shares to archive - self.dbc.execute("select time from shares where upstream_result = 1 order by time limit 1") - data = self.dbc.fetchone() - if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : - return False - return data[0] + # Check for found shares to archive + self.dbc.execute("select time from shares where upstream_result = 1 order by time limit 1") + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : + return False + return data[0] def archive_found(self,found_time): - self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = 1 and time <= FROM_UNIXTIME(%s)", (found_time,)) - self.dbh.commit() + self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = 1 and time <= FROM_UNIXTIME(%s)", (found_time,)) + self.dbh.commit() def archive_to_db(self,found_time): - self.dbc.execute("insert into shares_archive select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) - self.dbh.commit() + self.dbc.execute("insert into shares_archive select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + self.dbh.commit() def archive_cleanup(self,found_time): - self.dbc.execute("delete from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) - self.dbh.commit() + self.dbc.execute("delete from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + self.dbh.commit() def archive_get_shares(self,found_time): - self.dbc.execute("select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) - return self.dbc + self.dbc.execute("select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + return self.dbc def import_shares(self,data): - log.debug("Importing Shares") -# 0 1 2 3 4 5 6 7 8 9 10 -# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] - checkin_times = {} - total_shares = 0 - best_diff = 0 - for k,v in enumerate(data): - if settings.DATABASE_EXTEND : - total_shares += v[3] - if v[0] in checkin_times: - if v[4] > checkin_times[v[0]] : - checkin_times[v[0]]["time"] = v[4] - else: - checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } - - if v[5] == True : - checkin_times[v[0]]["shares"] += v[3] - else : - checkin_times[v[0]]["rejects"] += v[3] - - if v[10] > best_diff: - best_diff = v[10] - - self.dbc.execute("insert into shares " +\ - "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ - "VALUES (FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) - else : - self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ - "(FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'') ) - - if settings.DATABASE_EXTEND : - self.dbc.execute("select value from pool where parameter = 'round_shares'") - round_shares = int(self.dbc.fetchone()[0]) + total_shares - self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) - - self.dbc.execute("select value from pool where parameter = 'round_best_share'") - round_best_share = int(self.dbc.fetchone()[0]) - if best_diff > round_best_share: - self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) - - self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") - difficulty = float(self.dbc.fetchone()[0]) - - if difficulty == 0: - progress = 0 - else: - progress = (round_shares/difficulty)*100 - self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) - - for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", - (v["time"],v["shares"],v["rejects"],k)) - - self.dbh.commit() + log.debug("Importing Shares") +# 0 1 2 3 4 5 6 7 8 9 10 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] + checkin_times = {} + total_shares = 0 + best_diff = 0 + for k,v in enumerate(data): + if settings.DATABASE_EXTEND : + total_shares += v[3] + if v[0] in checkin_times: + if v[4] > checkin_times[v[0]] : + checkin_times[v[0]]["time"] = v[4] + else: + checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } + + if v[5] == True : + checkin_times[v[0]]["shares"] += v[3] + else : + checkin_times[v[0]]["rejects"] += v[3] + + if v[10] > best_diff: + best_diff = v[10] + + self.dbc.execute("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) + else : + self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ + "(FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],v[5],0,v[9],'') ) + + if settings.DATABASE_EXTEND : + self.dbc.execute("select value from pool where parameter = 'round_shares'") + round_shares = int(self.dbc.fetchone()[0]) + total_shares + self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + + self.dbc.execute("select value from pool where parameter = 'round_best_share'") + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: + self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) + + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + difficulty = float(self.dbc.fetchone()[0]) + + if difficulty == 0: + progress = 0 + else: + progress = (round_shares/difficulty)*100 + self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) + + for k,v in checkin_times.items(): + self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", + (v["time"],v["shares"],v["rejects"],k)) + + self.dbh.commit() def found_block(self,data): - # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = FROM_UNIXTIME(%s) and username = %s limit 1", - (data[5],data[2],data[4],data[0])) - if settings.DATABASE_EXTEND and data[5] == True: - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) - self.dbc.execute("select value from pool where parameter = 'pool_total_found'") - total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), - (0,'round_progress'), - (0,'round_best_share'), - (time.time(),'round_start'), - (total_found,'pool_total_found') - ]) - self.dbh.commit() + # Note: difficulty = -1 here + self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = FROM_UNIXTIME(%s) and username = %s limit 1", + (data[5],data[2],data[4],data[0])) + if settings.DATABASE_EXTEND and data[5] == True: + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) + self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + total_found = int(self.dbc.fetchone()[0]) + 1 + self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), + (0,'round_progress'), + (0,'round_best_share'), + (time.time(),'round_start'), + (total_found,'pool_total_found') + ]) + self.dbh.commit() def delete_user(self,username): - log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = %s", - (username )) - self.dbh.commit() + log.debug("Deleting Username") + self.dbc.execute("delete from pool_worker where username = %s", + (username )) + self.dbh.commit() def insert_user(self,username,password): - log.debug("Adding Username/Password") - self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", - (username, password )) - self.dbh.commit() + log.debug("Adding Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", + (username, m.hexdigest() )) + self.dbh.commit() def update_user(self,username,password): - log.debug("Updating Username/Password") - self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", - (username, password )) - self.dbh.commit() + log.debug("Updating Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", + {"pass": m.hexdigest(), "uname": username}) + self.dbh.commit() def update_worker_diff(self,username,diff): - self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) - self.dbh.commit() + self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) + self.dbh.commit() def clear_worker_diff(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("update pool_worker set difficulty = 0") - self.dbh.commit() + if settings.DATABASE_EXTEND == True : + self.dbc.execute("update pool_worker set difficulty = 0") + self.dbh.commit() def check_password(self,username,password): - log.debug("Checking Username/Password") - self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", - (username, password )) - data = self.dbc.fetchone() - if data[0] > 0 : - return True - return False + log.debug("Checking Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("select COUNT(*) from pool_worker where username = %(uname)s and password = %(pass)s", + {"pass": m.hexdigest(), "uname": username}) + data = self.dbc.fetchone() + if data[0] > 0 : + return True + return False def update_pool_info(self,pi): - self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), - (pi['balance'],"bitcoin_balance"), - (pi['connections'],"bitcoin_connections"), - (pi['difficulty'],"bitcoin_difficulty"), - (time.time(),"bitcoin_infotime") - ]) - self.dbh.commit() + self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), + (pi['balance'],"bitcoin_balance"), + (pi['connections'],"bitcoin_connections"), + (pi['difficulty'],"bitcoin_difficulty"), + (time.time(),"bitcoin_infotime") + ]) + self.dbh.commit() def get_pool_stats(self): - self.dbc.execute("select * from pool") - ret = {} - for data in self.dbc.fetchall(): - ret[data[0]] = data[1] - return ret + self.dbc.execute("select * from pool") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = data[1] + return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") - ret = {} - for data in self.dbc.fetchall(): - ret[data[0]] = { "username" : data[0], - "speed" : data[1], - "last_checkin" : time.mktime(data[2].timetuple()), - "total_shares" : data[3], - "total_rejects" : data[4], - "total_found" : data[5], - "alive" : data[6], - "difficulty" : data[7] } - return ret + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = { "username" : data[0], + "speed" : data[1], + "last_checkin" : time.mktime(data[2].timetuple()), + "total_shares" : data[3], + "total_rejects" : data[4], + "total_found" : data[5], + "alive" : data[6], + "difficulty" : data[7] } + return ret def close(self): - self.dbh.close() + self.dbh.close() def check_tables(self): - log.debug("Checking Tables") - - # Do we have our tables? - shares_exist = False - self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS " +\ - "where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", - {"schema": settings.DB_MYSQL_DBNAME }) - data = self.dbc.fetchone() - if data[0] <= 0 : - self.update_version_1() # no, we don't, so create them - - if settings.DATABASE_EXTEND == True : - self.update_tables() - + log.debug("Checking Tables") + + # Do we have our tables? + shares_exist = False + self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS " +\ + "where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", + {"schema": settings.DB_MYSQL_DBNAME }) + data = self.dbc.fetchone() + if data[0] <= 0 : + self.update_version_1() # no, we don't, so create them + + if settings.DATABASE_EXTEND == True : + self.update_tables() + def update_tables(self): - version = 0 - current_version = 6 - while version < current_version : - self.dbc.execute("select value from pool where parameter = 'DB Version'") - data = self.dbc.fetchone() - version = int(data[0]) - if version < current_version : - log.info("Updating Database from %i to %i" % (version, version +1)) - getattr(self, 'update_version_' + str(version) )() + version = 0 + current_version = 7 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() def update_version_1(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("create table if not exists shares " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("create index shares_username ON shares(username(10))") - - self.dbc.execute("create table if not exists pool_worker" +\ - "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp" +\ - ") ENGINE = MYISAM") - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") - - self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") - self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") - self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") - - else : - self.dbc.execute("create table if not exists shares" + \ - "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") - self.dbc.execute("create index shares_username ON shares(username(10))") - self.dbc.execute("create table if not exists pool_worker(id serial,username TEXT, password TEXT) ENGINE = MYISAM") - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") - self.dbh.commit() - + if settings.DATABASE_EXTEND == True : + self.dbc.execute("create table if not exists shares " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + self.dbc.execute("create index shares_username ON shares(username(10))") + + self.dbc.execute("create table if not exists pool_worker" +\ + "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp" +\ + ") ENGINE = MYISAM") + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + + self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + + else : + self.dbc.execute("create table if not exists shares" + \ + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") + self.dbc.execute("create index shares_username ON shares(username(10))") + self.dbc.execute("create table if not exists pool_worker(id serial,username TEXT, password TEXT) ENGINE = MYISAM") + self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + self.dbh.commit() + def update_version_2(self): - log.info("running update 2") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), - ('bitcoin_balance',0), - ('bitcoin_connections',0), - ('bitcoin_difficulty',0), - ('pool_speed',0), - ('pool_total_found',0), - ('round_shares',0), - ('round_progress',0), - ('round_start',time.time()) - ]) - self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") - self.dbh.commit() - + log.info("running update 2") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), + ('bitcoin_balance',0), + ('bitcoin_connections',0), + ('bitcoin_difficulty',0), + ('pool_speed',0), + ('pool_total_found',0), + ('round_shares',0), + ('round_progress',0), + ('round_start',time.time()) + ]) + self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + self.dbh.commit() + def update_version_3(self): - log.info("running update 3") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ - ('round_best_share',0), - ('bitcoin_infotime',0) - ]) - self.dbc.execute("alter table pool_worker add alive BOOLEAN") - self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") - self.dbh.commit() - + log.info("running update 3") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ + ('round_best_share',0), + ('bitcoin_infotime',0) + ]) + self.dbc.execute("alter table pool_worker add alive BOOLEAN") + self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + self.dbh.commit() + def update_version_4(self): - log.info("running update 4") - self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") - self.dbc.execute("create table if not exists shares_archive " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("create table if not exists shares_archive_found " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") - self.dbh.commit() + log.info("running update 4") + self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") + self.dbc.execute("create table if not exists shares_archive " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + self.dbc.execute("create table if not exists shares_archive_found " +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") + self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + self.dbh.commit() def update_version_5(self): - log.info("running update 5") - # Adding Primary key to table: pool - self.dbc.execute("alter table pool add primary key (parameter(100))") - self.dbh.commit() - # Adjusting indicies on table: shares - self.dbc.execute("DROP INDEX shares_username ON shares") - self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username(10))") - self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") - self.dbh.commit() - - self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") - self.dbh.commit() + log.info("running update 5") + # Adding Primary key to table: pool + self.dbc.execute("alter table pool add primary key (parameter(100))") + self.dbh.commit() + # Adjusting indicies on table: shares + self.dbc.execute("DROP INDEX shares_username ON shares") + self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username(10))") + self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") + self.dbh.commit() + + self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") + self.dbh.commit() + + def update_version_6(self): + log.info("running update 6") + + # Optimising table layout + self.dbc.execute("ALTER TABLE `pool` CHARACTER SET = utf8, COLLATE = utf8_general_ci, ENGINE = InnoDB, " +\ + "CHANGE COLUMN `parameter` `parameter` VARCHAR(128) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, " +\ + "CHANGE COLUMN `value` `value` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NULL, " +\ + "DROP PRIMARY KEY, ADD PRIMARY KEY (`parameter`)") + self.dbh.commit() + + self.dbc.execute("UPDATE pool_worker SET password = SHA1(CONCAT(password, %(salt)s)) WHERE id > 0", {"salt": self.salt}) + self.dbh.commit() + + self.dbc.execute("ALTER TABLE `pool_worker` CHARACTER SET = utf8, COLLATE = utf8_general_ci, ENGINE = InnoDB, " +\ + "CHANGE COLUMN `username` `username` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, " +\ + "CHANGE COLUMN `password` `password` CHAR(40) CHARACTER SET 'utf8' COLLATE 'utf8_bin' NOT NULL, " +\ + "CHANGE COLUMN `speed` `speed` INT(10) UNSIGNED NULL , " +\ + "CHANGE COLUMN `total_shares` `total_shares` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ + "CHANGE COLUMN `total_rejects` `total_rejects` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ + "CHANGE COLUMN `total_found` `total_found` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ + "CHANGE COLUMN `alive` `alive` TINYINT(1) UNSIGNED NULL DEFAULT NULL , " +\ + "CHANGE COLUMN `difficulty` `difficulty` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ + "ADD UNIQUE INDEX `pool_worker-username` (`username`(128) ASC), DROP INDEX `pool_worker_username`, DROP INDEX `id`") + self.dbh.commit() + + self.dbc.execute("update pool set value = 7 where parameter = 'DB Version'") + self.dbh.commit() From a3211aa61f559186009b7ab575c42e2d3ad087e4 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Mon, 22 Apr 2013 14:20:49 +0100 Subject: [PATCH 36/56] Updated readme --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5af2122..24e962c 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,7 @@ stratum-mining Basic implementation of bitcoin mining pool using Stratum mining protocol. -This fork includes a database implementation for: - None - Sqlite - Mysql - Postgresql +This fork includes database optimisations for MySQL and password hashing using a salt. Basic worker stats are provided (and updated) @@ -17,5 +13,6 @@ For more info on Stratum: http://mining.bitcoin.cz/stratum-mining. Original version by Slush +Modified version by GeneralFault -This version by GeneralFault (Tips Welcome: 15Zk7DoFYJ7hESpZzmix1WLkomTMGW81c2 ) +This version by Wade Womersley (Media Skunk Works) ( Tips Welcome: 1FxBTbWR15WZp8vnru8N6zVsVBwigPAcdN ) From b4a2f312853a9c3c082b274c6cad288f84c66188 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Mon, 22 Apr 2013 15:30:20 +0100 Subject: [PATCH 37/56] PostgreSQL Optimisation and SHA1 pass encryption --- .gitignore | 1 + mining/DB_Postgresql.py | 500 ++++++++++++++++++++++------------------ 2 files changed, 272 insertions(+), 229 deletions(-) diff --git a/.gitignore b/.gitignore index 2dd4327..caaa6de 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ NOTES run pooldb.sqlite archives/* +twistd.pid \ No newline at end of file diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 959586c..f01a8ae 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -1,4 +1,5 @@ import time +import hashlib from stratum import settings import stratum.logger log = stratum.logger.get_logger('DB_Postgresql') @@ -7,288 +8,329 @@ class DB_Postgresql(): def __init__(self): - log.debug("Connecting to DB") - self.dbh = psycopg2.connect("host='"+settings.DB_PGSQL_HOST+"' dbname='"+settings.DB_PGSQL_DBNAME+"' user='"+settings.DB_PGSQL_USER+\ - "' password='"+settings.DB_PGSQL_PASS+"'") - # TODO -- set the schema - self.dbc = self.dbh.cursor() + log.debug("Connecting to DB") + self.dbh = psycopg2.connect("host='"+settings.DB_PGSQL_HOST+"' dbname='"+settings.DB_PGSQL_DBNAME+"' user='"+settings.DB_PGSQL_USER+\ + "' password='"+settings.DB_PGSQL_PASS+"'") + # TODO -- set the schema + self.dbc = self.dbh.cursor() + + if hasattr(settings, 'PASSWORD_SALT'): + self.salt = settings.PASSWORD_SALT + else: + raise Exception("PASSWORD_SALT isn't set, please set in config.py") def updateStats(self,averageOverTime): - log.debug("Updating Stats") - # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0, alive = 'f'"); - stime = '%.2f' % ( time.time() - averageOverTime ); - self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", [stime]) - total_speed = 0 - for name,shares in self.dbc.fetchall(): - speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) - total_speed += speed - self.dbc.execute("update pool_worker set speed = %s, alive = 't' where username = %s", (speed,name)) - self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) - self.dbh.commit() + log.debug("Updating Stats") + # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. + self.dbc.execute("update pool_worker set speed = 0, alive = 'f'"); + stime = '%.2f' % ( time.time() - averageOverTime ); + self.dbc.execute("select username,SUM(difficulty) from shares where time > to_timestamp(%s) group by username", [stime]) + total_speed = 0 + for name,shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + total_speed += speed + self.dbc.execute("update pool_worker set speed = %s, alive = 't' where username = %s", (speed,name)) + self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) + self.dbh.commit() def archive_check(self): - # Check for found shares to archive - self.dbc.execute("select time from shares where upstream_result = true order by time limit 1") - data = self.dbc.fetchone() - if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : - return False - return data[0] + # Check for found shares to archive + self.dbc.execute("select time from shares where upstream_result = true order by time limit 1") + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : + return False + return data[0] def archive_found(self,found_time): - self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = true and time <= to_timestamp(%s)", [found_time]) - self.dbh.commit() + self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = true and time <= to_timestamp(%s)", [found_time]) + self.dbh.commit() def archive_to_db(self,found_time): - self.dbc.execute("insert into shares_archive select * from shares where time <= to_timestamp(%s)",[found_time]) - self.dbh.commit() + self.dbc.execute("insert into shares_archive select * from shares where time <= to_timestamp(%s)",[found_time]) + self.dbh.commit() def archive_cleanup(self,found_time): - self.dbc.execute("delete from shares where time <= to_timestamp(%s)",[found_time]) - self.dbh.commit() + self.dbc.execute("delete from shares where time <= to_timestamp(%s)",[found_time]) + self.dbh.commit() def archive_get_shares(self,found_time): - self.dbc.execute("select * from shares where time <= to_timestamp(%s)",[found_time]) - return self.dbc + self.dbc.execute("select * from shares where time <= to_timestamp(%s)",[found_time]) + return self.dbc def import_shares(self,data): - log.debug("Importing Shares") -# 0 1 2 3 4 5 6 7 8 9 10 -# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] - checkin_times = {} - total_shares = 0 - best_diff = 0 - for k,v in enumerate(data): - if settings.DATABASE_EXTEND : - total_shares += v[3] - if v[0] in checkin_times: - if v[4] > checkin_times[v[0]] : - checkin_times[v[0]]["time"] = v[4] - else: - checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } + log.debug("Importing Shares") +# 0 1 2 3 4 5 6 7 8 9 10 +# data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] + checkin_times = {} + total_shares = 0 + best_diff = 0 + for k,v in enumerate(data): + if settings.DATABASE_EXTEND : + total_shares += v[3] + if v[0] in checkin_times: + if v[4] > checkin_times[v[0]] : + checkin_times[v[0]]["time"] = v[4] + else: + checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } - if v[5] == True : - checkin_times[v[0]]["shares"] += v[3] - else : - checkin_times[v[0]]["rejects"] += v[3] + if v[5] == True : + checkin_times[v[0]]["shares"] += v[3] + else : + checkin_times[v[0]]["rejects"] += v[3] - if v[10] > best_diff: - best_diff = v[10] + if v[10] > best_diff: + best_diff = v[10] - self.dbc.execute("insert into shares " +\ - "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ - "VALUES (to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],bool(v[5]),False,v[9],'',v[7],v[8],'',v[3]) ) - else : - self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ - "(to_timestamp(%s),%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],bool(v[5]),False,v[9],'') ) + self.dbc.execute("insert into shares " +\ + "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ + "VALUES (to_timestamp(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],bool(v[5]),False,v[9],'',v[7],v[8],'',v[3]) ) + else : + self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ + "(to_timestamp(%s),%s,%s,%s,%s,%s,%s)", + (v[4],v[6],v[0],bool(v[5]),False,v[9],'') ) - if settings.DATABASE_EXTEND : - self.dbc.execute("select value from pool where parameter = 'round_shares'") - round_shares = int(self.dbc.fetchone()[0]) + total_shares - self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + if settings.DATABASE_EXTEND : + self.dbc.execute("select value from pool where parameter = 'round_shares'") + round_shares = int(self.dbc.fetchone()[0]) + total_shares + self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) - self.dbc.execute("select value from pool where parameter = 'round_best_share'") - round_best_share = int(self.dbc.fetchone()[0]) - if best_diff > round_best_share: - self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) + self.dbc.execute("select value from pool where parameter = 'round_best_share'") + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: + self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) - self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") - difficulty = float(self.dbc.fetchone()[0]) + self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + difficulty = float(self.dbc.fetchone()[0]) - if difficulty == 0: - progress = 0 - else: - progress = (round_shares/difficulty)*100 - self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) - - for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = to_timestamp(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", - (v["time"],v["shares"],v["rejects"],k)) + if difficulty == 0: + progress = 0 + else: + progress = (round_shares/difficulty)*100 + self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) + + for k,v in checkin_times.items(): + self.dbc.execute("update pool_worker set last_checkin = to_timestamp(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", + (v["time"],v["shares"],v["rejects"],k)) - self.dbh.commit() + self.dbh.commit() def found_block(self,data): - # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = %s, solution = %s where id in (select id from shares where time = to_timestamp(%s) and username = %s limit 1)", - (bool(data[5]),data[2],data[4],data[0])) - if settings.DATABASE_EXTEND and data[5] == True : - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0],)) - self.dbc.execute("select value from pool where parameter = 'pool_total_found'") - total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), - (0,'round_progress'), - (0,'round_best_share'), - (time.time(),'round_start'), - (total_found,'pool_total_found') - ]) - self.dbh.commit() + # Note: difficulty = -1 here + self.dbc.execute("update shares set upstream_result = %s, solution = %s where id in (select id from shares where time = to_timestamp(%s) and username = %s limit 1)", + (bool(data[5]),data[2],data[4],data[0])) + if settings.DATABASE_EXTEND and data[5] == True : + self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0],)) + self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + total_found = int(self.dbc.fetchone()[0]) + 1 + self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), + (0,'round_progress'), + (0,'round_best_share'), + (time.time(),'round_start'), + (total_found,'pool_total_found') + ]) + self.dbh.commit() def delete_user(self,username): - log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = %s", - (username )) - self.dbh.commit() + log.debug("Deleting Username") + self.dbc.execute("delete from pool_worker where username = %s", [username]) + self.dbh.commit() def insert_user(self,username,password): - log.debug("Adding Username/Password") - self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", - (username, password )) - self.dbh.commit() + log.debug("Adding Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", + (username, m.hexdigest() )) + self.dbh.commit() def update_user(self,username,password): - log.debug("Updating Username/Password") - self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", - (username, password )) - self.dbh.commit() + log.debug("Updating Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("update pool_worker set password = %s where username = %s", + (m.hexdigest(), username )) + self.dbh.commit() def update_worker_diff(self,username,diff): - self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) - self.dbh.commit() + self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) + self.dbh.commit() def clear_worker_diff(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("update pool_worker set difficulty = 0") - self.dbh.commit() + if settings.DATABASE_EXTEND == True : + self.dbc.execute("update pool_worker set difficulty = 0") + self.dbh.commit() def check_password(self,username,password): - log.debug("Checking Username/Password") - self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", - (username, password )) - data = self.dbc.fetchone() - if data[0] > 0 : - return True - return False + log.debug("Checking Username/Password") + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + self.dbc.execute("select COUNT(*) from pool_worker where username = %s and password = %s", + (username, m.hexdigest() )) + data = self.dbc.fetchone() + if data[0] > 0 : + return True + return False def update_pool_info(self,pi): - self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), - (pi['balance'],"bitcoin_balance"), - (pi['connections'],"bitcoin_connections"), - (pi['difficulty'],"bitcoin_difficulty"), - (time.time(),"bitcoin_infotime") - ]) - self.dbh.commit() + self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), + (pi['balance'],"bitcoin_balance"), + (pi['connections'],"bitcoin_connections"), + (pi['difficulty'],"bitcoin_difficulty"), + (time.time(),"bitcoin_infotime") + ]) + self.dbh.commit() def get_pool_stats(self): - self.dbc.execute("select * from pool") - ret = {} - for data in self.dbc.fetchall(): - ret[data[0]] = data[1] - return ret + self.dbc.execute("select * from pool") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = data[1] + return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") - ret = {} - for data in self.dbc.fetchall(): - ret[data[0]] = { "username" : data[0], - "speed" : data[1], - "last_checkin" : time.mktime(data[2].timetuple()), - "total_shares" : data[3], - "total_rejects" : data[4], - "total_found" : data[5], - "alive" : data[6], - "difficulty" : data[7] } - return ret + self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") + ret = {} + for data in self.dbc.fetchall(): + ret[data[0]] = { "username" : data[0], + "speed" : data[1], + "last_checkin" : time.mktime(data[2].timetuple()), + "total_shares" : data[3], + "total_rejects" : data[4], + "total_found" : data[5], + "alive" : data[6], + "difficulty" : data[7] } + return ret def close(self): - self.dbh.close() + self.dbh.close() def check_tables(self): - log.debug("Checking Tables") + log.debug("Checking Tables") - shares_exist = False - self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'shares'", - {"schema": settings.DB_PGSQL_SCHEMA }) - data = self.dbc.fetchone() - if data[0] <= 0 : - self.update_version_1() - - if settings.DATABASE_EXTEND == True : - self.update_tables() + shares_exist = False + self.dbc.execute("select COUNT(*) from pg_catalog.pg_tables where schemaname = %(schema)s and tablename = 'shares'", + {"schema": settings.DB_PGSQL_SCHEMA }) + data = self.dbc.fetchone() + if data[0] <= 0 : + self.update_version_1() + + if settings.DATABASE_EXTEND == True : + self.update_tables() - + def update_tables(self): - version = 0 - current_version = 6 - while version < current_version : - self.dbc.execute("select value from pool where parameter = 'DB Version'") - data = self.dbc.fetchone() - version = int(data[0]) - if version < current_version : - log.info("Updating Database from %i to %i" % (version, version +1)) - getattr(self, 'update_version_' + str(version) )() - + version = 0 + current_version = 7 + while version < current_version : + self.dbc.execute("select value from pool where parameter = 'DB Version'") + data = self.dbc.fetchone() + version = int(data[0]) + if version < current_version : + log.info("Updating Database from %i to %i" % (version, version +1)) + getattr(self, 'update_version_' + str(version) )() + def update_version_1(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("create table shares" +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") - self.dbc.execute("create index shares_username ON shares(username)") - self.dbc.execute("create table pool_worker" +\ - "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") - self.dbc.execute("create index pool_worker_username ON pool_worker(username)") - self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") - self.dbc.execute("create table pool(parameter TEXT, value TEXT)") - self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") - else : - self.dbc.execute("create table shares" + \ - "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT)") - self.dbc.execute("create index shares_username ON shares(username)") - self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") - self.dbc.execute("create index pool_worker_username ON pool_worker(username)") - self.dbh.commit() + if settings.DATABASE_EXTEND == True : + self.dbc.execute("create table shares" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create index shares_username ON shares(username)") + self.dbc.execute("create table pool_worker" +\ + "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") + self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") + self.dbc.execute("create table pool(parameter TEXT, value TEXT)") + self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") + else : + self.dbc.execute("create table shares" + \ + "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT)") + self.dbc.execute("create index shares_username ON shares(username)") + self.dbc.execute("create table pool_worker(id serial,username TEXT, password TEXT)") + self.dbc.execute("create index pool_worker_username ON pool_worker(username)") + self.dbh.commit() def update_version_2(self): - log.info("running update 2") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), - ('bitcoin_balance',0), - ('bitcoin_connections',0), - ('bitcoin_difficulty',0), - ('pool_speed',0), - ('pool_total_found',0), - ('round_shares',0), - ('round_progress',0), - ('round_start',time.time()) - ]) - self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") - self.dbh.commit() - + log.info("running update 2") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), + ('bitcoin_balance',0), + ('bitcoin_connections',0), + ('bitcoin_difficulty',0), + ('pool_speed',0), + ('pool_total_found',0), + ('round_shares',0), + ('round_progress',0), + ('round_start',time.time()) + ]) + self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + self.dbh.commit() + def update_version_3(self): - log.info("running update 3") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ - ('round_best_share',0), - ('bitcoin_infotime',0) - ]) - self.dbc.execute("alter table pool_worker add alive BOOLEAN") - self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") - self.dbh.commit() - + log.info("running update 3") + self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ + ('round_best_share',0), + ('bitcoin_infotime',0) + ]) + self.dbc.execute("alter table pool_worker add alive BOOLEAN") + self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + self.dbh.commit() + def update_version_4(self): - log.info("running update 4") - self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") - self.dbc.execute("create table shares_archive" +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") - self.dbc.execute("create table shares_archive_found" +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") - self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") - self.dbh.commit() - + log.info("running update 4") + self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") + self.dbc.execute("create table shares_archive" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("create table shares_archive_found" +\ + "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ + "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER)") + self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + self.dbh.commit() + def update_version_5(self): - log.info("running update 5") - # Adding Primary key to table: pool - self.dbc.execute("alter table pool add primary key (parameter)") - self.dbh.commit() - # Adjusting indicies on table: shares - self.dbc.execute("DROP INDEX shares_username") - self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username)") - self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") - self.dbh.commit() - - self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") - self.dbh.commit() + log.info("running update 5") + # Adding Primary key to table: pool + self.dbc.execute("alter table pool add primary key (parameter)") + self.dbh.commit() + # Adjusting indicies on table: shares + self.dbc.execute("DROP INDEX shares_username") + self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username)") + self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") + self.dbh.commit() + + self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") + self.dbh.commit() + + def update_version_6(self): + log.info("running update 6") + + try: + self.dbc.execute("CREATE EXTENSION pgcrypto") + except psycopg2.ProgrammingError: + log.info("pgcrypto already added to database") + except psycopg2.OperationalError: + raise Exception("Could not add pgcrypto extension to database. Have you got it installed? Ubuntu is postgresql-contrib") + self.dbh.commit() + + # Optimising table layout + self.dbc.execute("ALTER TABLE pool " +\ + "ALTER COLUMN parameter TYPE character varying(128), ALTER COLUMN value TYPE character varying(512);") + self.dbh.commit() + + self.dbc.execute("UPDATE pool_worker SET password = encode(digest(concat(password, %s), 'sha1'), 'hex') WHERE id > 0", [self.salt]) + self.dbh.commit() + + self.dbc.execute("ALTER TABLE pool_worker " +\ + "ALTER COLUMN username TYPE character varying(512), ALTER COLUMN password TYPE character(40), " +\ + "ADD CONSTRAINT username UNIQUE (username)") + self.dbh.commit() + + self.dbc.execute("update pool set value = 7 where parameter = 'DB Version'") + self.dbh.commit() + From 708e14192d40fc0f8a333dced6909cf62d538c9f Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Mon, 22 Apr 2013 17:18:24 +0100 Subject: [PATCH 38/56] Ignore on .project and .pydevproject --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index caaa6de..3065190 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ NOTES run pooldb.sqlite archives/* -twistd.pid \ No newline at end of file +twistd.pid +.project +.pydevproject \ No newline at end of file From fae32548782c1c4272903767b812fc7ffe598aa0 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Mon, 22 Apr 2013 17:30:28 +0100 Subject: [PATCH 39/56] Raise ValueError if salt is missing from config --- mining/DB_Mysql.py | 2 +- mining/DB_Postgresql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 05daa06..27495f0 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -15,7 +15,7 @@ def __init__(self): if hasattr(settings, 'PASSWORD_SALT'): self.salt = settings.PASSWORD_SALT else: - raise Exception("PASSWORD_SALT isn't set, please set in config.py") + raise ValueError("PASSWORD_SALT isn't set, please set in config.py") def updateStats(self,averageOverTime): log.debug("Updating Stats") diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index f01a8ae..996c093 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -17,7 +17,7 @@ def __init__(self): if hasattr(settings, 'PASSWORD_SALT'): self.salt = settings.PASSWORD_SALT else: - raise Exception("PASSWORD_SALT isn't set, please set in config.py") + raise ValueError("PASSWORD_SALT isn't set, please set in config.py") def updateStats(self,averageOverTime): log.debug("Updating Stats") From 7f205606727ecc2990acdb3fcebf34bb42d9aed3 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Wed, 24 Apr 2013 13:57:33 +0100 Subject: [PATCH 40/56] MySQL foreign keys. Tokenized MySQL calls. Moving from "username" (string) to "worker" (int) in shares tables - foreign key enabled. MySQL calls now use dictionary's for values rather than just %s. Started code reformatting of all classes. --- lib/basic_stats.py | 2 +- mining/DBInterface.py | 373 ++++++------ mining/DB_Mysql.py | 1050 +++++++++++++++++++++++++++------ mining/basic_share_limiter.py | 6 +- mining/service.py | 40 +- mining/subscription.py | 8 +- 6 files changed, 1087 insertions(+), 392 deletions(-) diff --git a/lib/basic_stats.py b/lib/basic_stats.py index 8bc06a3..e92a53d 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -96,7 +96,7 @@ def render_GET(self, request): if wd["speed"] > 100: wc = "#0A0" r+="%s%s/%s%s/%s%s"%( - wc,wi,format(int(wd["speed"]),"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), + wc,wi,format(wd["speed"],"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) r+="" else : diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 4152e6e..c219d45 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -10,213 +10,232 @@ class DBInterface(): def __init__(self): - self.dbi = self.connectDB() + self.dbi = self.connectDB() def init_main(self): - self.dbi.check_tables() + self.dbi.check_tables() - self.q = Queue.Queue() + self.q = Queue.Queue() self.queueclock = None - self.usercache = {} + self.usercache = {} self.clearusercache() - self.nextStatsUpdate = 0 + self.nextStatsUpdate = 0 self.scheduleImport() - def set_bitcoinrpc(self,bitcoinrpc): - self.bitcoinrpc=bitcoinrpc + def set_bitcoinrpc(self, bitcoinrpc): + self.bitcoinrpc = bitcoinrpc def connectDB(self): - # Choose our database driver and put it in self.dbi - if settings.DATABASE_DRIVER == "sqlite": - log.debug('DB_Sqlite INIT') - import DB_Sqlite - return DB_Sqlite.DB_Sqlite() - elif settings.DATABASE_DRIVER == "mysql": - log.debug('DB_Mysql INIT') - import DB_Mysql - return DB_Mysql.DB_Mysql() - elif settings.DATABASE_DRIVER == "postgresql": - log.debug('DB_Postgresql INIT') - import DB_Postgresql - return DB_Postgresql.DB_Postgresql() - elif settings.DATABASE_DRIVER == "none": - log.debug('DB_None INIT') - import DB_None - return DB_None.DB_None() - else: - log.error('Invalid DATABASE_DRIVER -- using NONE') - log.debug('DB_None INIT') - import DB_None - return DB_None.DB_None() + # Choose our database driver and put it in self.dbi + if settings.DATABASE_DRIVER == "sqlite": + log.debug('DB_Sqlite INIT') + import DB_Sqlite + return DB_Sqlite.DB_Sqlite() + elif settings.DATABASE_DRIVER == "mysql": + log.debug('DB_Mysql INIT') + import DB_Mysql + return DB_Mysql.DB_Mysql() + elif settings.DATABASE_DRIVER == "postgresql": + log.debug('DB_Postgresql INIT') + import DB_Postgresql + return DB_Postgresql.DB_Postgresql() + elif settings.DATABASE_DRIVER == "none": + log.debug('DB_None INIT') + import DB_None + return DB_None.DB_None() + else: + log.error('Invalid DATABASE_DRIVER -- using NONE') + log.debug('DB_None INIT') + import DB_None + return DB_None.DB_None() def clearusercache(self): - self.usercache = {} - self.usercacheclock = reactor.callLater( settings.DB_USERCACHE_TIME , self.clearusercache) + self.usercache = {} + self.usercacheclock = reactor.callLater(settings.DB_USERCACHE_TIME , self.clearusercache) def scheduleImport(self): - # This schedule's the Import - use_thread = True - if settings.DATABASE_DRIVER == "sqlite": - use_thread = False - - if use_thread: - self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import_thread) - else: - self.queueclock = reactor.callLater( settings.DB_LOADER_CHECKTIME , self.run_import) + # This schedule's the Import + use_thread = True + if settings.DATABASE_DRIVER == "sqlite": + use_thread = False + + if use_thread: + self.queueclock = reactor.callLater(settings.DB_LOADER_CHECKTIME , self.run_import_thread) + else: + self.queueclock = reactor.callLater(settings.DB_LOADER_CHECKTIME , self.run_import) def run_import_thread(self): - if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run - reactor.callInThread(self.import_thread) - self.scheduleImport() + if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run + reactor.callInThread(self.import_thread) + + self.scheduleImport() def run_import(self): - self.do_import(self.dbi,False) - if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : - self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME - self.dbi.updateStats(settings.DB_STATS_AVG_TIME) + self.do_import(self.dbi, False) + + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + self.dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) - if settings.ARCHIVE_SHARES : - self.archive_shares(self.dbi) - self.scheduleImport() + + if settings.ARCHIVE_SHARES : + self.archive_shares(self.dbi) + + self.scheduleImport() def import_thread(self): - # Here we are in the thread. - dbi = self.connectDB() - self.do_import(dbi,False) - if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : - self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME - dbi.updateStats(settings.DB_STATS_AVG_TIME) + # Here we are in the thread. + dbi = self.connectDB() + self.do_import(dbi, False) + + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME + dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) - if settings.ARCHIVE_SHARES : - self.archive_shares(dbi) - dbi.close() - - def _update_pool_info(self,data): - self.dbi.update_pool_info({ 'blocks' : data['blocks'], 'balance' : data['balance'], - 'connections' : data['connections'], 'difficulty' : data['difficulty'] }) - - def do_import(self,dbi,force): - # Only run if we have data - while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN: - force = False - # Put together the data we want to import - sqldata = [] - datacnt = 0 - while self.q.empty() == False and datacnt < settings.DB_LOADER_REC_MAX : - datacnt += 1 - data = self.q.get() - sqldata.append(data) - self.q.task_done() - # try to do the import, if we fail, log the error and put the data back in the queue - try: - log.info("Inserting %s Share Records",datacnt) - dbi.import_shares(sqldata) - except Exception as e: - log.error("Insert Share Records Failed: %s", e.args[0]) - for k,v in enumerate(sqldata): - self.q.put(v) - break # Allows us to sleep a little - - def archive_shares(self,dbi): - found_time = dbi.archive_check() - if found_time == 0: - return False - log.info("Archiving shares newer than timestamp %f " % found_time) - dbi.archive_found(found_time) - if settings.ARCHIVE_MODE == 'db': - dbi.archive_to_db(found_time) - dbi.archive_cleanup(found_time) - elif settings.ARCHIVE_MODE == 'file': - shares = dbi.archive_get_shares(found_time) - - filename = settings.ARCHIVE_FILE - if settings.ARCHIVE_FILE_APPEND_TIME : - filename = filename + "-" + datetime.fromtimestamp(found_time).strftime("%Y-%m-%d-%H-%M-%S") - filename = filename + ".csv" - - if settings.ARCHIVE_FILE_COMPRESS == 'gzip' : - import gzip - filename = filename + ".gz" - filehandle = gzip.open(filename, 'a') - elif settings.ARCHIVE_FILE_COMPRESS == 'bzip2' and settings.ARCHIVE_FILE_APPEND_TIME : - import bz2 - filename = filename + ".bz2" - filehandle = bz2.BZFile(filename, mode='wb', buffering=4096 ) - else: - filehandle = open(filename, "a") - - while True: - row = shares.fetchone() - if row == None: - break - str1 = '","'.join([str(x) for x in row]) - filehandle.write('"%s"\n' % str1) - filehandle.close() - - clean = False - while not clean: - try: - dbi.archive_cleanup(found_time) - clean = True - except Exception as e: - clean = False - log.error("Archive Cleanup Failed... will retry to cleanup in 30 seconds") - sleep(30) - - return True - - def queue_share(self,data): - self.q.put( data ) - - def found_block(self,data): - try: - log.info("Updating Found Block Share Record") - self.do_import(self.dbi,True) # We can't Update if the record is not there. - self.dbi.found_block(data) - except Exception as e: - log.error("Update Found Block Share Record Failed: %s", e.args[0]) - - def check_password(self,username,password): - if username == "": - log.info("Rejected worker for blank username") - return False - wid = username+":-:"+password - if wid in self.usercache : - return True - elif self.dbi.check_password(username,password) : - self.usercache[wid] = 1 - return True - elif settings.USERS_AUTOADD == True : - self.insert_user(username,password) - self.usercache[wid] = 1 - return True - return False - - def insert_user(self,username,password): - return self.dbi.insert_user(username,password) - - def delete_user(self,username): - self.usercache = {} - return self.dbi.delete_user(username) - - def update_user(self,username,password): - self.usercache = {} - return self.dbi.update_user(username,password) - - def update_worker_diff(self,username,diff): - return self.dbi.update_worker_diff(username,diff) + + if settings.ARCHIVE_SHARES: + self.archive_shares(dbi) + + dbi.close() + + def _update_pool_info(self, data): + self.dbi.update_pool_info({ 'blocks' : data['blocks'], 'balance' : data['balance'], + 'connections' : data['connections'], 'difficulty' : data['difficulty'] }) + + def do_import(self, dbi, force): + # Only run if we have data + while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN: + force = False + # Put together the data we want to import + sqldata = [] + datacnt = 0 + + while self.q.empty() == False and datacnt < settings.DB_LOADER_REC_MAX : + datacnt += 1 + data = self.q.get() + sqldata.append(data) + self.q.task_done() + + # try to do the import, if we fail, log the error and put the data back in the queue + try: + log.info("Inserting %s Share Records", datacnt) + dbi.import_shares(sqldata) + except Exception as e: + log.error("Insert Share Records Failed: %s", e.args[0]) + for k, v in enumerate(sqldata): + self.q.put(v) + break # Allows us to sleep a little + + def archive_shares(self, dbi): + found_time = dbi.archive_check() + + if found_time == 0: + return False + + log.info("Archiving shares newer than timestamp %f " % found_time) + dbi.archive_found(found_time) + + if settings.ARCHIVE_MODE == 'db': + dbi.archive_to_db(found_time) + dbi.archive_cleanup(found_time) + elif settings.ARCHIVE_MODE == 'file': + shares = dbi.archive_get_shares(found_time) + + filename = settings.ARCHIVE_FILE + + if settings.ARCHIVE_FILE_APPEND_TIME : + filename = filename + "-" + datetime.fromtimestamp(found_time).strftime("%Y-%m-%d-%H-%M-%S") + + filename = filename + ".csv" + + if settings.ARCHIVE_FILE_COMPRESS == 'gzip' : + import gzip + filename = filename + ".gz" + filehandle = gzip.open(filename, 'a') + elif settings.ARCHIVE_FILE_COMPRESS == 'bzip2' and settings.ARCHIVE_FILE_APPEND_TIME : + import bz2 + filename = filename + ".bz2" + filehandle = bz2.BZFile(filename, mode='wb', buffering=4096) + else: + filehandle = open(filename, "a") + + while True: + row = shares.fetchone() + if row == None: + break + str1 = '","'.join([str(x) for x in row]) + filehandle.write('"%s"\n' % str1) + + filehandle.close() + + clean = False + + while not clean: + try: + dbi.archive_cleanup(found_time) + clean = True + except Exception as e: + clean = False + log.error("Archive Cleanup Failed... will retry to cleanup in 30 seconds") + sleep(30) + + return True + + def queue_share(self, data): + self.q.put(data) + + def found_block(self, data): + try: + log.info("Updating Found Block Share Record") + self.do_import(self.dbi, True) # We can't Update if the record is not there. + self.dbi.found_block(data) + except Exception as e: + log.error("Update Found Block Share Record Failed: %s", e.args[0]) + + def check_password(self, username, password): + if username == "": + log.info("Rejected worker for blank username") + return False + + wid = username + ":-:" + password + + if wid in self.usercache : + return True + elif self.dbi.check_password(username, password) : + self.usercache[wid] = 1 + return True + elif settings.USERS_AUTOADD == True : + self.insert_user(username, password) + self.usercache[wid] = 1 + return True + + return False + + def insert_user(self, username, password): + return self.dbi.insert_user(username, password) + + def delete_user(self, username): + self.usercache = {} + return self.dbi.delete_user(username) + + def update_user(self, username, password): + self.usercache = {} + return self.dbi.update_user(username, password) + + def update_worker_diff(self, username, diff): + return self.dbi.update_worker_diff(username, diff) def get_pool_stats(self): - return self.dbi.get_pool_stats() + return self.dbi.get_pool_stats() def get_workers_stats(self): - return self.dbi.get_workers_stats() + return self.dbi.get_workers_stats() def clear_worker_diff(self): - return self.dbi.clear_worker_diff() + return self.dbi.clear_worker_diff() diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 27495f0..0fdc768 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -9,67 +9,190 @@ class DB_Mysql(): def __init__(self): log.debug("Connecting to DB") - self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST,settings.DB_MYSQL_USER,settings.DB_MYSQL_PASS,settings.DB_MYSQL_DBNAME) + + self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST, + settings.DB_MYSQL_USER, settings.DB_MYSQL_PASS, + settings.DB_MYSQL_DBNAME) + self.dbc = self.dbh.cursor() if hasattr(settings, 'PASSWORD_SALT'): self.salt = settings.PASSWORD_SALT else: raise ValueError("PASSWORD_SALT isn't set, please set in config.py") + + def hash_pass(self, password): + m = hashlib.sha1() + m.update(password) + m.update(self.salt) + + return m.hexdigest() - def updateStats(self,averageOverTime): + def updateStats(self, averageOverTime): log.debug("Updating Stats") # Note: we are using transactions... so we can set the speed = 0 and it doesn't take affect until we are commited. - self.dbc.execute("update pool_worker set speed = 0, alive = 0"); - stime = '%.0f' % ( time.time() - averageOverTime ); - self.dbc.execute("select username,SUM(difficulty) from shares where time > FROM_UNIXTIME(%s) group by username", (stime,)) + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `speed` = 0, + `alive` = 0 + """ + ); + + stime = '%.0f' % (time.time() - averageOverTime); + + self.dbc.execute( + """ + SELECT `username`, SUM(`shares`.`difficulty`) + FROM `shares` + LEFT JOIN `pool_worker` + ON `shares`.`worker` = `pool_worker`.`id` + WHERE `time` > FROM_UNIXTIME(%(time)s) + GROUP BY `shares`.`worker` + """, + { + "time": stime + } + ) + total_speed = 0 - for name,shares in self.dbc.fetchall(): - speed = int(int(shares) * pow(2,32)) / ( int(averageOverTime) * 1000 * 1000) + + for name, shares in self.dbc.fetchall(): + speed = int(int(shares) * pow(2, 32)) / ( int(averageOverTime) * 1000 * 1000) total_speed += speed - self.dbc.execute("update pool_worker set speed = %s, alive = 1 where username = %s", (speed,name)) - self.dbc.execute("update pool set value = %s where parameter = 'pool_speed'",[total_speed]) + + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `speed` = %(speed)s, + `alive` = 1 + WHERE `username` = %(uname)s + """, + { + "speed": speed, + "uname": name + } + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = 'pool_speed' + """, + { + "value": total_speed + } + ) + self.dbh.commit() def archive_check(self): # Check for found shares to archive - self.dbc.execute("select time from shares where upstream_result = 1 order by time limit 1") + self.dbc.execute( + """ + SELECT `time` + FROM `shares` + WHERE `upstream_result` = 1 + ORDER BY `time` + LIMIT 1 + """ + ) + data = self.dbc.fetchone() + if data is None or (data[0] + settings.ARCHIVE_DELAY) > time.time() : return False + return data[0] - def archive_found(self,found_time): - self.dbc.execute("insert into shares_archive_found select * from shares where upstream_result = 1 and time <= FROM_UNIXTIME(%s)", (found_time,)) + def archive_found(self, found_time): + self.dbc.execute( + """ + INSERT INTO `shares_archive_found` + SELECT s.`id`, s.`time`, s.`rem_host`, pw.`id`, s.`our_result`, + s.`upstream_result`, s.`reason`, s.`solution`, s.`block_num`, + s.`prev_block_hash`, s.`useragent`, s.`difficulty` + FROM `shares` s + LEFT JOIN `pool_worker` pw + ON s.`worker` = pw.`id` + WHERE `upstream_result` = 1 + AND `time` <= FROM_UNIXTIME(%(time)s) + """, + { + "time": found_time + } + ) + self.dbh.commit() - def archive_to_db(self,found_time): - self.dbc.execute("insert into shares_archive select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + def archive_to_db(self, found_time): + self.dbc.execute( + """ + INSERT INTO `shares_archive` + SELECT s.`id`, s.`time`, s.`rem_host`, pw.`id`, s.`our_result`, + s.`upstream_result`, s.`reason`, s.`solution`, s.`block_num`, + s.`prev_block_hash`, s.`useragent`, s.`difficulty` + FROM `shares` s + LEFT JOIN `pool_worker` pw + ON s.`worker` = pw.`id` + WHERE `time` <= FROM_UNIXTIME(%(time)s) + """, + { + "time": found_time + } + ) + self.dbh.commit() - def archive_cleanup(self,found_time): - self.dbc.execute("delete from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + def archive_cleanup(self, found_time): + self.dbc.execute( + """ + DELETE FROM `shares` + WHERE `time` <= FROM_UNIXTIME(%(time)s) + """, + { + "time": found_time + } + ) + self.dbh.commit() - def archive_get_shares(self,found_time): - self.dbc.execute("select * from shares where time <= FROM_UNIXTIME(%s)",(found_time,)) + def archive_get_shares(self, found_time): + self.dbc.execute( + """ + SELECT * + FROM `shares` + WHERE `time` <= FROM_UNIXTIME(%(time)s) + """, + { + "time": found_time + } + ) + return self.dbc - def import_shares(self,data): + def import_shares(self, data): log.debug("Importing Shares") # 0 1 2 3 4 5 6 7 8 9 10 # data: [worker_name,block_header,block_hash,difficulty,timestamp,is_valid,ip,block_height,prev_hash,invalid_reason,best_diff] checkin_times = {} total_shares = 0 best_diff = 0 - for k,v in enumerate(data): + + for k, v in enumerate(data): if settings.DATABASE_EXTEND : total_shares += v[3] + if v[0] in checkin_times: if v[4] > checkin_times[v[0]] : checkin_times[v[0]]["time"] = v[4] else: - checkin_times[v[0]] = {"time": v[4], "shares": 0, "rejects": 0 } + checkin_times[v[0]] = { + "time": v[4], + "shares": 0, + "rejects": 0 + } if v[5] == True : checkin_times[v[0]]["shares"] += v[3] @@ -79,130 +202,383 @@ def import_shares(self,data): if v[10] > best_diff: best_diff = v[10] - self.dbc.execute("insert into shares " +\ - "(time,rem_host,username,our_result,upstream_result,reason,solution,block_num,prev_block_hash,useragent,difficulty) " +\ - "VALUES (FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'',v[7],v[8],'',v[3]) ) - else : - self.dbc.execute("insert into shares (time,rem_host,username,our_result,upstream_result,reason,solution) VALUES " +\ - "(FROM_UNIXTIME(%s),%s,%s,%s,%s,%s,%s)", - (v[4],v[6],v[0],v[5],0,v[9],'') ) - - if settings.DATABASE_EXTEND : - self.dbc.execute("select value from pool where parameter = 'round_shares'") + self.dbc.execute( + """ + INSERT INTO `shares` + (time, rem_host, worker, our_result, upstream_result, + reason, solution, block_num, prev_block_hash, + useragent, difficulty) + VALUES + (FROM_UNIXTIME(%(time)s), %(host)s, + (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s), + %(lres)s, 0, %(reason)s, '', + %(blocknum)s, %(hash)s, '', %(difficulty)s) + """, + { + "time": v[4], + "host": v[6], + "uname": v[0], + "lres": v[5], + "reason": v[9], + "blocknum": v[7], + "hash": v[8], + "difficulty": v[3] + } + ) + else: + self.dbc.execute( + """ + INSERT INTO `shares` + (time, rem_host, worker, our_result, + upstream_result, reason, solution) + VALUES + (FROM_UNIXTIME(%(time)s), %(host)s, + (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s), + %(lres)s, 0, %(reason)s, '') + """, + { + "time": v[4], + "host": v[6], + "uname": v[0], + "lres": v[5], + "reason": v[9] + } + ) + + if settings.DATABASE_EXTEND: + self.dbc.execute( + """ + SELECT `value` + FROM `pool` + WHERE `parameter` = 'round_shares' + """ + ) + round_shares = int(self.dbc.fetchone()[0]) + total_shares - self.dbc.execute("update pool set value = %s where parameter = 'round_shares'",[round_shares]) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = 'round_shares' + """, + { + "value": round_shares + } + ) - self.dbc.execute("select value from pool where parameter = 'round_best_share'") + self.dbc.execute( + """ + SELECT `value` + FROM `pool` + WHERE `parameter` = 'round_best_share' + """ + ) + round_best_share = int(self.dbc.fetchone()[0]) + if best_diff > round_best_share: - self.dbc.execute("update pool set value = %s where parameter = 'round_best_share'",[best_diff]) + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = 'round_best_share' + """, + { + "value": best_diff + } + ) - self.dbc.execute("select value from pool where parameter = 'bitcoin_difficulty'") + self.dbc.execute( + """ + SELECT `value` + FROM `pool` + WHERE `parameter` = 'bitcoin_difficulty' + """ + ) + difficulty = float(self.dbc.fetchone()[0]) if difficulty == 0: progress = 0 else: - progress = (round_shares/difficulty)*100 - self.dbc.execute("update pool set value = %s where parameter = 'round_progress'",[progress]) + progress = (round_shares / difficulty) * 100 + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = 'round_progress' + """, + { + "value": progress + } + ) - for k,v in checkin_times.items(): - self.dbc.execute("update pool_worker set last_checkin = FROM_UNIXTIME(%s), total_shares = total_shares + %s, total_rejects = total_rejects + %s where username = %s", - (v["time"],v["shares"],v["rejects"],k)) + for k, v in checkin_times.items(): + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `last_checkin` = FROM_UNIXTIME(%(time)s), + `total_shares` = `total_shares` + %(shares)s, + `total_rejects` = `total_rejects` + %(rejects)s + WHERE `username` = (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s) + """, + { + "time": v["time"], + "shares": v["shares"], + "rejects": v["rejects"], + "uname": k + } + ) self.dbh.commit() - def found_block(self,data): + def found_block(self, data): # Note: difficulty = -1 here - self.dbc.execute("update shares set upstream_result = %s, solution = %s where time = FROM_UNIXTIME(%s) and username = %s limit 1", - (data[5],data[2],data[4],data[0])) + self.dbc.execute( + """ + UPDATE `shares` + SET `upstream_result` = %(result)s, + `solution` = %(solution)s + WHERE `time` = FROM_UNIXTIME(%(time)s) + AND `username` = (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s) + LIMIT 1 + """, + { + "result": data[5], + "solution": data[2], + "time": data[4], + "uname": data[0] + } + ) + if settings.DATABASE_EXTEND and data[5] == True: - self.dbc.execute("update pool_worker set total_found = total_found + 1 where username = %s",(data[0])) - self.dbc.execute("select value from pool where parameter = 'pool_total_found'") + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `total_found` = `total_found` + 1 + WHERE `username` = %(uname)s + """, + { + "uname": data[0] + } + ) + self.dbc.execute( + """ + SELECT `value` + FROM `pool` + WHERE `parameter` = 'pool_total_found' + """ + ) total_found = int(self.dbc.fetchone()[0]) + 1 - self.dbc.executemany("update pool set value = %s where parameter = %s",[(0,'round_shares'), - (0,'round_progress'), - (0,'round_best_share'), - (time.time(),'round_start'), - (total_found,'pool_total_found') - ]) + + self.dbc.executemany( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = %(param)s + """, + [ + { + "param": "round_shares", + "value": "0" + }, + { + "param": "round_progress", + "value": "0" + }, + { + "param": "round_best_share", + "value": "0" + }, + { + "param": "round_start", + "value": time.time() + }, + { + "param": "pool_total_found", + "value": total_found + } + ] + ) + self.dbh.commit() - def delete_user(self,username): - log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = %s", - (username )) + def delete_user(self, username): + log.debug("Deleting user %s", username) + + self.dbc.execute( + """ + DELETE FROM `pool_worker` + WHERE `username` = %(uname)s + """, + { + "uname": username + } + ) + self.dbh.commit() - def insert_user(self,username,password): - log.debug("Adding Username/Password") - m = hashlib.sha1() - m.update(password) - m.update(self.salt) - self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", - (username, m.hexdigest() )) + def insert_user(self, username, password): + log.debug("Adding new user %s", username) + + self.dbc.execute( + """ + INSERT INTO `pool_worker` + (`username`, `password`) + VALUES + (%(uname)s, %(pass)s) + """, + { + "uname": username, + "pass": self.hash_pass(password) + } + ) + self.dbh.commit() - def update_user(self,username,password): - log.debug("Updating Username/Password") - m = hashlib.sha1() - m.update(password) - m.update(self.salt) - self.dbc.execute("update pool_worker set password = %(pass)s where username = %(uname)s", - {"pass": m.hexdigest(), "uname": username}) + def update_user(self, username, password): + log.debug("Updating password for user %s", username); + + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `password` = %(pass)s + WHERE `username` = %(uname)s + """, + { + "uname": username, + "pass": self.hash_pass(password) + } + ) + self.dbh.commit() - def update_worker_diff(self,username,diff): - self.dbc.execute("update pool_worker set difficulty = %s where username = %s",(diff,username)) + def update_worker_diff(self, username, diff): + log.debug("Setting difficulty for %s to %s", username, diff) + + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `difficulty` = %(diff)s + WHERE `username` = %(uname)s + """, + { + "uname": username, + "diff": diff + } + ) + self.dbh.commit() def clear_worker_diff(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("update pool_worker set difficulty = 0") + if settings.DATABASE_EXTEND == True: + log.debug("Resetting difficulty for all workers") + + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `difficulty` = 0 + """ + ) + self.dbh.commit() - def check_password(self,username,password): - log.debug("Checking Username/Password") - m = hashlib.sha1() - m.update(password) - m.update(self.salt) - self.dbc.execute("select COUNT(*) from pool_worker where username = %(uname)s and password = %(pass)s", - {"pass": m.hexdigest(), "uname": username}) + def check_password(self, username, password): + log.debug("Checking username/password for %s", username) + + self.dbc.execute( + """ + SELECT COUNT(*) + FROM `pool_worker` + WHERE `username` = %(uname)s + AND `password` = %(pass)s + """, + { + "uname": username, + "pass": self.hash_pass(password) + } + ) + data = self.dbc.fetchone() - if data[0] > 0 : + + if data[0] > 0: return True + return False - def update_pool_info(self,pi): - self.dbc.executemany("update pool set value = %s where parameter = %s",[(pi['blocks'],"bitcoin_blocks"), - (pi['balance'],"bitcoin_balance"), - (pi['connections'],"bitcoin_connections"), - (pi['difficulty'],"bitcoin_difficulty"), - (time.time(),"bitcoin_infotime") - ]) + def update_pool_info(self, pi): + self.dbc.executemany( + """ + UPDATE `pool` + SET `value` = %(value)s + WHERE `parameter` = %(param)s + """, + [ + { + "param": "bitcoin_blocks", + "value": pi['blocks'] + }, + { + "param": "bitcoin_balance", + "value": pi['balance'] + }, + { + "param": "bitcoin_connections", + "value": pi['connections'] + }, + { + "param": "bitcoin_difficulty", + "value": pi['difficulty'] + }, + { + "param": "bitcoin_infotime", + "value": time.time() + } + ] + ) + self.dbh.commit() def get_pool_stats(self): - self.dbc.execute("select * from pool") + self.dbc.execute( + """ + SELECT * FROM `pool` + """ + ) + ret = {} + for data in self.dbc.fetchall(): ret[data[0]] = data[1] + return ret def get_workers_stats(self): - self.dbc.execute("select username,speed,last_checkin,total_shares,total_rejects,total_found,alive,difficulty from pool_worker") + self.dbc.execute( + """ + SELECT `username`, `speed`, `last_checkin`, `total_shares`, + `total_rejects`, `total_found`, `alive`, `difficulty` + FROM `pool_worker` + """ + ) + ret = {} + for data in self.dbc.fetchall(): - ret[data[0]] = { "username" : data[0], - "speed" : data[1], + ret[data[0]] = { + "username" : data[0], + "speed" : int(data[1]), "last_checkin" : time.mktime(data[2].timetuple()), - "total_shares" : data[3], - "total_rejects" : data[4], - "total_found" : data[5], - "alive" : data[6], - "difficulty" : data[7] } + "total_shares" : int(data[3]), + "total_rejects" : int(data[4]), + "total_found" : int(data[5]), + "alive" : True if data[6] is 1 else False, + "difficulty" : int(data[7]) + } + return ret def close(self): @@ -213,10 +589,21 @@ def check_tables(self): # Do we have our tables? shares_exist = False - self.dbc.execute("select COUNT(*) from INFORMATION_SCHEMA.STATISTICS " +\ - "where table_schema = %(schema)s and table_name = 'shares' and index_name = 'shares_username'", - {"schema": settings.DB_MYSQL_DBNAME }) + + self.dbc.execute( + """ + SELECT COUNT(*) + FROM INFORMATION_SCHEMA.STATISTICS + WHERE `table_schema` = %(schema)s + AND `table_name` = 'shares' + """, + { + "schema": settings.DB_MYSQL_DBNAME + } + ) + data = self.dbc.fetchone() + if data[0] <= 0 : self.update_version_1() # no, we don't, so create them @@ -226,117 +613,420 @@ def check_tables(self): def update_tables(self): version = 0 current_version = 7 - while version < current_version : - self.dbc.execute("select value from pool where parameter = 'DB Version'") + + while version < current_version: + self.dbc.execute( + """ + SELECT `value` + FROM `pool` + WHERE parameter = 'DB Version' + """ + ) + data = self.dbc.fetchone() version = int(data[0]) - if version < current_version : + + if version < current_version: log.info("Updating Database from %i to %i" % (version, version +1)) getattr(self, 'update_version_' + str(version) )() def update_version_1(self): - if settings.DATABASE_EXTEND == True : - self.dbc.execute("create table if not exists shares " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("create index shares_username ON shares(username(10))") - - self.dbc.execute("create table if not exists pool_worker" +\ - "(id serial primary key,username TEXT, password TEXT, speed INTEGER, last_checkin timestamp" +\ - ") ENGINE = MYISAM") - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") - - self.dbc.execute("create table if not exists pool(parameter TEXT, value TEXT)") - self.dbc.execute("alter table pool_worker add total_shares INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_rejects INTEGER default 0") - self.dbc.execute("alter table pool_worker add total_found INTEGER default 0") - self.dbc.execute("insert into pool (parameter,value) VALUES ('DB Version',2)") - - else : - self.dbc.execute("create table if not exists shares" + \ - "(id serial,time timestamp,rem_host TEXT, username TEXT, our_result INTEGER, upstream_result INTEGER, reason TEXT, solution TEXT) ENGINE = MYISAM") - self.dbc.execute("create index shares_username ON shares(username(10))") - self.dbc.execute("create table if not exists pool_worker(id serial,username TEXT, password TEXT) ENGINE = MYISAM") - self.dbc.execute("create index pool_worker_username ON pool_worker(username(10))") + if settings.DATABASE_EXTEND == True: + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `shares` + ( + `id` SERIAL PRIMARY KEY, + `time` TIMESTAMP, + `rem_host` TEXT, + `username` TEXT, + `our_result` BOOLEAN, + `upstream_result` BOOLEAN, + `reason` TEXT, + `solution` TEXT, + `block_num` INTEGER, + `prev_block_hash` TEXT, + `useragent` TEXT, + `difficulty` INTEGER + ) + ENGINE=MYISAM + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `shares_username` ON `shares`(`username`(10)) + """ + ) + + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `pool_worker` + ( + `id` SERIAL PRIMARY KEY, + `username` TEXT, + `password` TEXT, + `speed` INTEGER, + `last_checkin` TIMESTAMP + ) + ENGINE=MYISAM + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `pool_worker_username` ON `pool_worker`(`username`(10)) + """ + ) + + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `pool` + ( + `parameter` TEXT, + `value` TEXT + ) + """ + ) + + self.dbc.execute( + """ + ALTER TABLE `pool_worker` ADD `total_shares` INTEGER DEFAULT 0 + """ + ) + + self.dbc.execute( + """ + ALTER TABLE `pool_worker` ADD `total_rejects` INTEGER DEFAULT 0 + """ + ) + + self.dbc.execute( + """ + ALTER TABLE `pool_worker` ADD `total_found` INTEGER DEFAULT 0 + """ + ) + + self.dbc.execute( + """ + INSERT INTO `pool` + (parameter, value) + VALUES + ('DB Version', 2) + """ + ) + else: + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `shares` + ( + `id` SERIAL, + `time` TIMESTAMP, + `rem_host` TEXT, + `username` TEXT, + `our_result` INTEGER, + `upstream_result` INTEGER, + `reason` TEXT, + `solution` TEXT + ) + ENGINE=MYISAM + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `shares_username` ON `shares`(`username`(10)) + """ + ) + + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `pool_worker` + ( + `id` SERIAL, + `username` TEXT, + `password` TEXT + ) + ENGINE=MYISAM + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `pool_worker_username` ON `pool_worker`(`username`(10)) + """ + ) + self.dbh.commit() def update_version_2(self): log.info("running update 2") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[('bitcoin_blocks',0), - ('bitcoin_balance',0), - ('bitcoin_connections',0), - ('bitcoin_difficulty',0), - ('pool_speed',0), - ('pool_total_found',0), - ('round_shares',0), - ('round_progress',0), - ('round_start',time.time()) - ]) - self.dbc.execute("update pool set value = 3 where parameter = 'DB Version'") + + self.dbc.executemany( + """ + INSERT INTO `pool` (`parameter`, `value`) VALUES (%s, %s) + """, + [ + ('bitcoin_blocks', 0), + ('bitcoin_balance', 0), + ('bitcoin_connections', 0), + ('bitcoin_difficulty', 0), + ('pool_speed', 0), + ('pool_total_found', 0), + ('round_shares', 0), + ('round_progress', 0), + ('round_start', time.time()) + ] + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = 3 + WHERE `parameter` = 'DB Version' + """ + ) + self.dbh.commit() def update_version_3(self): log.info("running update 3") - self.dbc.executemany("insert into pool (parameter,value) VALUES (%s,%s)",[ - ('round_best_share',0), - ('bitcoin_infotime',0) - ]) - self.dbc.execute("alter table pool_worker add alive BOOLEAN") - self.dbc.execute("update pool set value = 4 where parameter = 'DB Version'") + + self.dbc.executemany( + """ + INSERT INTO `pool` (`parameter`, `value`) VALUES (%s, %s) + """, + [ + ('round_best_share', 0), + ('bitcoin_infotime', 0) + ] + ) + + self.dbc.execute( + """ + ALTER TABLE `pool_worker` ADD `alive` BOOLEAN + """ + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = 4 + WHERE `parameter` = 'DB Version' + """ + ) + self.dbh.commit() def update_version_4(self): log.info("running update 4") - self.dbc.execute("alter table pool_worker add difficulty INTEGER default 0") - self.dbc.execute("create table if not exists shares_archive " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("create table if not exists shares_archive_found " +\ - "(id serial primary key,time timestamp,rem_host TEXT, username TEXT, our_result BOOLEAN, upstream_result BOOLEAN, reason TEXT, solution TEXT, " +\ - "block_num INTEGER, prev_block_hash TEXT, useragent TEXT, difficulty INTEGER) ENGINE = MYISAM;") - self.dbc.execute("update pool set value = 5 where parameter = 'DB Version'") + + self.dbc.execute( + """ + ALTER TABLE `pool_worker` + ADD `difficulty` INTEGER DEFAULT 0 + """ + ) + + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `shares_archive` + ( + `id` SERIAL PRIMARY KEY, + `time` TIMESTAMP, + `rem_host` TEXT, + `username` TEXT, + `our_result` BOOLEAN, + `upstream_result` BOOLEAN, + `reason` TEXT, + `solution` TEXT, + `block_num` INTEGER, + `prev_block_hash` TEXT, + `useragent` TEXT, + `difficulty` INTEGER + ) + ENGINE = MYISAM + """ + ) + + self.dbc.execute( + """ + CREATE TABLE IF NOT EXISTS `shares_archive_found` + ( + `id` SERIAL PRIMARY KEY, + `time` TIMESTAMP, + `rem_host` TEXT, + `username` TEXT, + `our_result` BOOLEAN, + `upstream_result` BOOLEAN, + `reason` TEXT, + `solution` TEXT, + `block_num` INTEGER, + `prev_block_hash` TEXT, + `useragent` TEXT, + `difficulty` INTEGER + ) + ENGINE = MYISAM + """ + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = 5 + WHERE `parameter` = 'DB Version' + """ + ) + self.dbh.commit() def update_version_5(self): log.info("running update 5") - # Adding Primary key to table: pool - self.dbc.execute("alter table pool add primary key (parameter(100))") - self.dbh.commit() + + self.dbc.execute( + """ + ALTER TABLE `pool` + ADD PRIMARY KEY (`parameter`(100)) + """ + ) + # Adjusting indicies on table: shares - self.dbc.execute("DROP INDEX shares_username ON shares") - self.dbc.execute("CREATE INDEX shares_time_username ON shares(time,username(10))") - self.dbc.execute("CREATE INDEX shares_upstreamresult ON shares(upstream_result)") - self.dbh.commit() + self.dbc.execute( + """ + DROP INDEX `shares_username` ON `shares` + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `shares_time_username` ON `shares`(`time`, `username`(10)) + """ + ) + + self.dbc.execute( + """ + CREATE INDEX `shares_upstreamresult` ON `shares`(`upstream_result`) + """ + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = 6 + WHERE `parameter` = 'DB Version' + """ + ) - self.dbc.execute("update pool set value = 6 where parameter = 'DB Version'") self.dbh.commit() def update_version_6(self): log.info("running update 6") - # Optimising table layout - self.dbc.execute("ALTER TABLE `pool` CHARACTER SET = utf8, COLLATE = utf8_general_ci, ENGINE = InnoDB, " +\ - "CHANGE COLUMN `parameter` `parameter` VARCHAR(128) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, " +\ - "CHANGE COLUMN `value` `value` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NULL, " +\ - "DROP PRIMARY KEY, ADD PRIMARY KEY (`parameter`)") - self.dbh.commit() + self.dbc.execute( + """ + ALTER TABLE `pool` + CHARACTER SET = utf8, + COLLATE = utf8_general_ci, + ENGINE = InnoDB, + CHANGE COLUMN `parameter` `parameter` VARCHAR(128) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, + CHANGE COLUMN `value` `value` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NULL, + DROP PRIMARY KEY, ADD PRIMARY KEY (`parameter`) + """ + ) - self.dbc.execute("UPDATE pool_worker SET password = SHA1(CONCAT(password, %(salt)s)) WHERE id > 0", {"salt": self.salt}) - self.dbh.commit() + self.dbc.execute( + """ + UPDATE `pool_worker` + SET `password` = SHA1(CONCAT(password, %(salt)s)) + WHERE id > 0 + """, + { + "salt": self.salt + } + ) - self.dbc.execute("ALTER TABLE `pool_worker` CHARACTER SET = utf8, COLLATE = utf8_general_ci, ENGINE = InnoDB, " +\ - "CHANGE COLUMN `username` `username` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, " +\ - "CHANGE COLUMN `password` `password` CHAR(40) CHARACTER SET 'utf8' COLLATE 'utf8_bin' NOT NULL, " +\ - "CHANGE COLUMN `speed` `speed` INT(10) UNSIGNED NULL , " +\ - "CHANGE COLUMN `total_shares` `total_shares` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ - "CHANGE COLUMN `total_rejects` `total_rejects` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ - "CHANGE COLUMN `total_found` `total_found` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ - "CHANGE COLUMN `alive` `alive` TINYINT(1) UNSIGNED NULL DEFAULT NULL , " +\ - "CHANGE COLUMN `difficulty` `difficulty` INT(10) UNSIGNED NULL DEFAULT '0' , " +\ - "ADD UNIQUE INDEX `pool_worker-username` (`username`(128) ASC), DROP INDEX `pool_worker_username`, DROP INDEX `id`") - self.dbh.commit() + self.dbc.execute( + """ + ALTER TABLE `pool_worker` + CHARACTER SET = utf8, + COLLATE = utf8_general_ci, + ENGINE = InnoDB, + CHANGE COLUMN `username` `username` VARCHAR(512) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci' NOT NULL, + CHANGE COLUMN `password` `password` CHAR(40) CHARACTER SET 'utf8' COLLATE 'utf8_bin' NOT NULL, + CHANGE COLUMN `speed` `speed` INT(10) UNSIGNED NOT NULL DEFAULT '0', + CHANGE COLUMN `total_shares` `total_shares` INT(10) UNSIGNED NOT NULL DEFAULT '0', + CHANGE COLUMN `total_rejects` `total_rejects` INT(10) UNSIGNED NOT NULL DEFAULT '0', + CHANGE COLUMN `total_found` `total_found` INT(10) UNSIGNED NOT NULL DEFAULT '0', + CHANGE COLUMN `alive` `alive` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + CHANGE COLUMN `difficulty` `difficulty` INT(10) UNSIGNED NOT NULL DEFAULT '0', + ADD UNIQUE INDEX `pool_worker-username` (`username`(128) ASC), + DROP INDEX `pool_worker_username`, + DROP INDEX `id` + """ + ) + + self.dbc.execute( + """ + ALTER TABLE `shares` + ADD COLUMN `worker` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 AFTER `username`, + DROP INDEX `id`, + ENGINE = InnoDB; + """ + ) + + self.dbc.execute( + """ + UPDATE `shares` + JOIN `pool_worker` + ON `pool_worker`.`username` = `shares`.`username` + SET `worker` = `pool_worker`.`id` + """ + ) + + self.dbc.execute( + """ + SET SESSION sql_mode='NO_AUTO_VALUE_ON_ZERO'; + """ + ) + + self.dbc.execute( + """ + INSERT INTO `pool_worker` + (`id`, `username`, `password`) + VALUES + (0, SHA1(RAND(CURRENT_TIMESTAMP)), SHA1(CURRENT_TIMESTAMP)) + """ + ) + + self.dbc.execute( + """ + SET SESSION sql_mode=''; + """ + ) + + self.dbc.execute( + """ + ALTER TABLE `shares` + ADD CONSTRAINT `workerid` + FOREIGN KEY (`worker` ) + REFERENCES `pool_worker` (`id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + DROP INDEX `shares_time_username`, + ADD INDEX `shares_time_worker` (`time` ASC, `worker` ASC), + ADD INDEX `shares_worker` (`worker` ASC), + DROP COLUMN `username` + """ + ) + + self.dbc.execute( + """ + UPDATE `pool` + SET `value` = 7 + WHERE `parameter` = 'DB Version' + """ + ) - self.dbc.execute("update pool set value = 7 where parameter = 'DB Version'") self.dbh.commit() diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 1dc3450..68dd0c7 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -7,7 +7,7 @@ dbi = DBInterface.DBInterface() dbi.clear_worker_diff() -''' This is just a cusomized ring buffer ''' +''' This is just a customized ring buffer ''' class SpeedBuffer: def __init__(self,size_max): self.max = size_max @@ -78,7 +78,8 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n self.worker_stats[worker_name]['last_rtc'] = ts avg = self.worker_stats[worker_name]['buffer'].avg() log.info("Checking Retarget for %s (%i) avg. %i target %i+-%i" % (worker_name,current_difficulty,avg, - self.target,self.variance) ) + self.target,self.variance)) + if avg < 1: log.info("Reseting avg = 1 since it's SOOO low") avg = 1 @@ -107,6 +108,7 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n self.worker_stats[worker_name]['buffer'].clear() session = connection_ref().get_session() + session['prev_diff'] = session['difficulty'] session['prev_jobid'] = job_id session['difficulty'] = new_diff diff --git a/mining/service.py b/mining/service.py index 922358d..f00e67c 100644 --- a/mining/service.py +++ b/mining/service.py @@ -41,7 +41,6 @@ def authorize(self, worker_name, worker_password): if Interfaces.worker_manager.authorize(worker_name, worker_password): session['authorized'][worker_name] = worker_password return True - else: if worker_name in session['authorized']: del session['authorized'][worker_name] @@ -57,24 +56,9 @@ def subscribe(self): session = self.connection_ref().get_session() session['extranonce1'] = extranonce1 - session['difficulty'] = settings.POOL_TARGET # Following protocol specs, default diff is 1 + session['difficulty'] = settings.POOL_TARGET # Following protocol specs, default diff is 1 return Pubsub.subscribe(self.connection_ref(), MiningSubscription()) + (extranonce1_hex, extranonce2_size) - - ''' - def submit(self, worker_name, job_id, extranonce2, ntime, nonce): - import time - start = time.time() - - for x in range(100): - try: - ret = self.submit2(worker_name, job_id, extranonce2, ntime, nonce) - except: - pass - - log.info("LEN %.03f" % (time.time() - start)) - return ret - ''' def submit(self, worker_name, job_id, extranonce2, ntime, nonce): '''Try to solve block candidate using given parameters.''' @@ -83,12 +67,12 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): session.setdefault('authorized', {}) # Check if worker is authorized to submit shares - if not Interfaces.worker_manager.authorize(worker_name, - session['authorized'].get(worker_name)): + if not Interfaces.worker_manager.authorize(worker_name, session['authorized'].get(worker_name)): raise SubmitException("Worker is not authorized") # Check if extranonce1 is in connection session extranonce1_bin = session.get('extranonce1', None) + if not extranonce1_bin: raise SubmitException("Connection is not subscribed for mining") @@ -102,32 +86,32 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): # and it is valid proof of work. try: (block_header, block_hash, share_diff, on_submit) = Interfaces.template_registry.submit_share(job_id, - worker_name, session, extranonce1_bin, extranonce2, ntime, nonce, difficulty) + worker_name, session, extranonce1_bin, extranonce2, ntime, nonce, difficulty) except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted - Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, - submit_time, False, ip, e[0], 0) + Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, + submit_time, False, ip, e[0], 0) raise - Interfaces.share_manager.on_submit_share(worker_name, block_header, block_hash, difficulty, - submit_time, True, ip, '', share_diff) + Interfaces.share_manager.on_submit_share(worker_name, block_header, + block_hash, difficulty, submit_time, True, ip, '', share_diff) if on_submit != None: # Pool performs submitblock() to bitcoind. Let's hook # to result and report it to share manager on_submit.addCallback(Interfaces.share_manager.on_submit_block, - worker_name, block_header, block_hash, submit_time,ip,share_diff) + worker_name, block_header, block_hash, submit_time, ip, share_diff) return True # Service documentation for remote discovery update_block.help_text = "Notify Stratum server about new block on the network." - update_block.params = [('password', 'string', 'Administrator password'),] + update_block.params = [('password', 'string', 'Administrator password'), ] authorize.help_text = "Authorize worker for submitting shares on this connection." authorize.params = [('worker_name', 'string', 'Name of the worker, usually in the form of user_login.worker_id.'), - ('worker_password', 'string', 'Worker password'),] + ('worker_password', 'string', 'Worker password'), ] subscribe.help_text = "Subscribes current connection for receiving new mining jobs." subscribe.params = [] @@ -139,5 +123,5 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): ('job_id', 'string', 'ID of job (received by mining.notify) which the current solution is based on.'), ('extranonce2', 'string', 'hex-encoded big-endian extranonce2, length depends on extranonce2_size from mining.notify.'), ('ntime', 'string', 'UNIX timestamp (32bit integer, big-endian, hex-encoded), must be >= ntime provided by mining,notify and <= current time'), - ('nonce', 'string', '32bit integer, hex-encoded, big-endian'),] + ('nonce', 'string', '32bit integer, hex-encoded, big-endian'), ] diff --git a/mining/subscription.py b/mining/subscription.py index 41c77d0..81df21d 100644 --- a/mining/subscription.py +++ b/mining/subscription.py @@ -17,10 +17,10 @@ def on_template(cls, is_new_block): new block which we have to broadcast clients.''' start = Interfaces.timestamper.time() - clean_jobs = is_new_block + (job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, _) = \ - Interfaces.template_registry.get_last_broadcast_args() + Interfaces.template_registry.get_last_broadcast_args() # Push new job to subscribed clients cls.emit(job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) @@ -38,8 +38,8 @@ def _finish_after_subscribe(self, result): return result # Force set higher difficulty - self.connection_ref().rpc('mining.set_difficulty', [settings.POOL_TARGET,], is_notification=True) - #self.connection_ref().rpc('client.get_version', []) + self.connection_ref().rpc('mining.set_difficulty', [settings.POOL_TARGET, ], is_notification=True) + # self.connection_ref().rpc('client.get_version', []) # Force client to remove previous jobs if any (eg. from previous connection) clean_jobs = True From bdddd517da32023e4140b9a931e4a58e7b801890 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Wed, 24 Apr 2013 14:17:08 +0100 Subject: [PATCH 41/56] More debug output in DBInterface --- mining/DBInterface.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index c219d45..7d0b2c4 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -53,6 +53,7 @@ def connectDB(self): return DB_None.DB_None() def clearusercache(self): + log.debug("DBInterface.clearusercache called"); self.usercache = {} self.usercacheclock = reactor.callLater(settings.DB_USERCACHE_TIME , self.clearusercache) @@ -74,6 +75,8 @@ def run_import_thread(self): self.scheduleImport() def run_import(self): + log.debug("DBInterface.run_import called"); + self.do_import(self.dbi, False) if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : @@ -108,6 +111,8 @@ def _update_pool_info(self, data): 'connections' : data['connections'], 'difficulty' : data['difficulty'] }) def do_import(self, dbi, force): + log.debug("DBInterface.do_import called. force: %s, queue size: %s", 'yes' if force == True else 'no', self.q.qsize()) + # Only run if we have data while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN: force = False @@ -132,6 +137,7 @@ def do_import(self, dbi, force): break # Allows us to sleep a little def archive_shares(self, dbi): + log.debug("DBInterface.archive_shares called") found_time = dbi.archive_check() if found_time == 0: From 2ddff0ba4ac0b23cc176dd4d7de2c157ffe35813 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Wed, 24 Apr 2013 15:51:28 +0100 Subject: [PATCH 42/56] Reduced SQL queries in updateStats --- mining/DB_Mysql.py | 50 ++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 0fdc768..9dce994 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -43,46 +43,34 @@ def updateStats(self, averageOverTime): self.dbc.execute( """ - SELECT `username`, SUM(`shares`.`difficulty`) - FROM `shares` - LEFT JOIN `pool_worker` - ON `shares`.`worker` = `pool_worker`.`id` - WHERE `time` > FROM_UNIXTIME(%(time)s) - GROUP BY `shares`.`worker` + UPDATE `pool_worker` pw + LEFT JOIN ( + SELECT `worker`, ROUND(ROUND(SUM(`difficulty`)) * 4294967296) / %(average)s AS 'speed' + FROM `shares` + WHERE `time` > FROM_UNIXTIME(%(time)s) + GROUP BY `worker` + ) AS leJoin + ON leJoin.`worker` = pw.`id` + SET pw.`alive` = 1, + pw.`speed` = leJoin.`speed` + WHERE pw.`id` = leJoin.`worker` """, { - "time": stime + "time": stime, + "average": int(averageOverTime) * 1000000 } ) - - total_speed = 0 - - for name, shares in self.dbc.fetchall(): - speed = int(int(shares) * pow(2, 32)) / ( int(averageOverTime) * 1000 * 1000) - total_speed += speed - - self.dbc.execute( - """ - UPDATE `pool_worker` - SET `speed` = %(speed)s, - `alive` = 1 - WHERE `username` = %(uname)s - """, - { - "speed": speed, - "uname": name - } - ) self.dbc.execute( """ UPDATE `pool` - SET `value` = %(value)s + SET `value` = ( + SELECT SUM(`speed`) + FROM `pool_worker` + WHERE `alive` = 1 + ) WHERE `parameter` = 'pool_speed' - """, - { - "value": total_speed - } + """ ) self.dbh.commit() From 9f16a03f75ef153d613410ec33927d4a5e92a8df Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Wed, 24 Apr 2013 15:52:52 +0100 Subject: [PATCH 43/56] get_worker_stats in MySQL ignores the 0 user --- mining/DB_Mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 9dce994..5ccc3fe 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -550,6 +550,7 @@ def get_workers_stats(self): SELECT `username`, `speed`, `last_checkin`, `total_shares`, `total_rejects`, `total_found`, `alive`, `difficulty` FROM `pool_worker` + WHERE `id` > 0 """ ) From 74929e9fab1172d87ae7abe57b3263225cebd905 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Wed, 24 Apr 2013 15:56:39 +0100 Subject: [PATCH 44/56] Added `alive` index to pool_worker table --- mining/DB_Mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 5ccc3fe..c328363 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -950,6 +950,7 @@ def update_version_6(self): CHANGE COLUMN `alive` `alive` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', CHANGE COLUMN `difficulty` `difficulty` INT(10) UNSIGNED NOT NULL DEFAULT '0', ADD UNIQUE INDEX `pool_worker-username` (`username`(128) ASC), + ADD INDEX `pool_worker-alive` (`alive`), DROP INDEX `pool_worker_username`, DROP INDEX `id` """ From 7c030bf74c02f945280a704a7a5c7b8694f9d223 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 11:39:55 +0100 Subject: [PATCH 45/56] MySQL bug in share updates + extra debug info --- mining/DBInterface.py | 2 ++ mining/DB_Mysql.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 7d0b2c4..e4a6979 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -69,6 +69,8 @@ def scheduleImport(self): self.queueclock = reactor.callLater(settings.DB_LOADER_CHECKTIME , self.run_import) def run_import_thread(self): + log.debug("run_import_thread current size: %d", self.q.qsize()); + if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run reactor.callInThread(self.import_thread) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index c328363..4ddc952 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -254,7 +254,7 @@ def import_shares(self, data): "value": round_shares } ) - + self.dbc.execute( """ SELECT `value` @@ -292,6 +292,7 @@ def import_shares(self, data): else: progress = (round_shares / difficulty) * 100 + self.dbc.execute( """ UPDATE `pool` @@ -310,7 +311,7 @@ def import_shares(self, data): SET `last_checkin` = FROM_UNIXTIME(%(time)s), `total_shares` = `total_shares` + %(shares)s, `total_rejects` = `total_rejects` + %(rejects)s - WHERE `username` = (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s) + WHERE `username` = %(uname)s """, { "time": v["time"], @@ -319,7 +320,7 @@ def import_shares(self, data): "uname": k } ) - + self.dbh.commit() From 9875dc0b2c098c76d0cbe7efbc72fcd864e3ad31 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 14:32:46 +0100 Subject: [PATCH 46/56] Share/Pool MySQL DB Commit optimisations --- mining/DB_Mysql.py | 79 ++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 4ddc952..7f7ded5 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -236,72 +236,47 @@ def import_shares(self, data): if settings.DATABASE_EXTEND: self.dbc.execute( """ - SELECT `value` + SELECT `parameter`, `value` FROM `pool` - WHERE `parameter` = 'round_shares' + WHERE `parameter` = 'round_best_share' + OR `parameter` = 'round_shares' + OR `parameter` = 'bitcoin_difficulty' + OR `parameter` = 'round_progress' """ ) - round_shares = int(self.dbc.fetchone()[0]) + total_shares + current_parameters = {} - self.dbc.execute( - """ - UPDATE `pool` - SET `value` = %(value)s - WHERE `parameter` = 'round_shares' - """, + for data in self.dbc.fetchall(): + current_parameters[data[0]] = data[1] + + round_best_share = int(current_parameters['round_best_share']) + difficulty = float(current_parameters['bitcoin_difficulty']) + + updates = [ { - "value": round_shares + "param": "round_shares", + "value": int(current_parameters['round_shares']) + total_shares + }, + { + "param": "round_progress", + "value": 0 if difficulty == 0 else (round_shares / difficulty) * 100 } - ) - - self.dbc.execute( - """ - SELECT `value` - FROM `pool` - WHERE `parameter` = 'round_best_share' - """ - ) - - round_best_share = int(self.dbc.fetchone()[0]) + ] if best_diff > round_best_share: - self.dbc.execute( - """ - UPDATE `pool` - SET `value` = %(value)s - WHERE `parameter` = 'round_best_share' - """, - { - "value": best_diff - } - ) - - self.dbc.execute( - """ - SELECT `value` - FROM `pool` - WHERE `parameter` = 'bitcoin_difficulty' - """ - ) - - difficulty = float(self.dbc.fetchone()[0]) - - if difficulty == 0: - progress = 0 - else: - progress = (round_shares / difficulty) * 100 - + updates.append({ + "param": "round_best_share", + "value": best_diff + }) - self.dbc.execute( + self.dbc.executemany( """ UPDATE `pool` SET `value` = %(value)s - WHERE `parameter` = 'round_progress' + WHERE `parameter` = %(param)s """, - { - "value": progress - } + updates ) for k, v in checkin_times.items(): From c8064485e08b3e118b79262a0adacb2760e9bfd2 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 14:50:16 +0100 Subject: [PATCH 47/56] Added DB_LOADER_FORCE_TIME config option --- mining/DBInterface.py | 8 ++++++-- mining/DB_Mysql.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index e4a6979..268e62e 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -24,6 +24,8 @@ def init_main(self): self.nextStatsUpdate = 0 self.scheduleImport() + + self.next_force_import_time = time.time() + settings.DB_LOADER_FORCE_TIME def set_bitcoinrpc(self, bitcoinrpc): self.bitcoinrpc = bitcoinrpc @@ -71,7 +73,7 @@ def scheduleImport(self): def run_import_thread(self): log.debug("run_import_thread current size: %d", self.q.qsize()); - if self.q.qsize() >= settings.DB_LOADER_REC_MIN: # Don't incur thread overhead if we're not going to run + if self.q.qsize() >= settings.DB_LOADER_REC_MIN or time.time() >= self.next_force_import_time: # Don't incur thread overhead if we're not going to run reactor.callInThread(self.import_thread) self.scheduleImport() @@ -116,7 +118,9 @@ def do_import(self, dbi, force): log.debug("DBInterface.do_import called. force: %s, queue size: %s", 'yes' if force == True else 'no', self.q.qsize()) # Only run if we have data - while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN: + while force == True or self.q.qsize() >= settings.DB_LOADER_REC_MIN or time.time() >= self.next_force_import_time: + self.next_force_import_time = time.time() + settings.DB_LOADER_FORCE_TIME + force = False # Put together the data we want to import sqldata = [] diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 7f7ded5..5705d6b 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -252,11 +252,12 @@ def import_shares(self, data): round_best_share = int(current_parameters['round_best_share']) difficulty = float(current_parameters['bitcoin_difficulty']) + round_shares = int(current_parameters['round_shares']) + total_shares updates = [ { "param": "round_shares", - "value": int(current_parameters['round_shares']) + total_shares + "value": round_shares }, { "param": "round_progress", From 321396057d3408b2563fc291086d25a14658d510 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 15:42:04 +0100 Subject: [PATCH 48/56] Reformatting --- mining/DBInterface.py | 16 +-- mining/__init__.py | 43 ++++---- mining/basic_share_limiter.py | 190 ++++++++++++++++++---------------- mining/interfaces.py | 44 ++++---- mining/service.py | 4 +- 5 files changed, 154 insertions(+), 143 deletions(-) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 268e62e..e4e6016 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -83,13 +83,13 @@ def run_import(self): self.do_import(self.dbi, False) - if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate: self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME self.dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() d.addCallback(self._update_pool_info) - if settings.ARCHIVE_SHARES : + if settings.ARCHIVE_SHARES: self.archive_shares(self.dbi) self.scheduleImport() @@ -99,7 +99,7 @@ def import_thread(self): dbi = self.connectDB() self.do_import(dbi, False) - if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate : + if settings.DATABASE_EXTEND and time.time() > self.nextStatsUpdate: self.nextStatsUpdate = time.time() + settings.DB_STATS_AVG_TIME dbi.updateStats(settings.DB_STATS_AVG_TIME) d = self.bitcoinrpc.getinfo() @@ -126,7 +126,7 @@ def do_import(self, dbi, force): sqldata = [] datacnt = 0 - while self.q.empty() == False and datacnt < settings.DB_LOADER_REC_MAX : + while self.q.empty() == False and datacnt < settings.DB_LOADER_REC_MAX: datacnt += 1 data = self.q.get() sqldata.append(data) @@ -165,7 +165,7 @@ def archive_shares(self, dbi): filename = filename + ".csv" - if settings.ARCHIVE_FILE_COMPRESS == 'gzip' : + if settings.ARCHIVE_FILE_COMPRESS == 'gzip': import gzip filename = filename + ".gz" filehandle = gzip.open(filename, 'a') @@ -216,12 +216,12 @@ def check_password(self, username, password): wid = username + ":-:" + password - if wid in self.usercache : + if wid in self.usercache: return True - elif self.dbi.check_password(username, password) : + elif self.dbi.check_password(username, password): self.usercache[wid] = 1 return True - elif settings.USERS_AUTOADD == True : + elif settings.USERS_AUTOADD == True: self.insert_user(username, password) self.usercache[wid] = 1 return True diff --git a/mining/__init__.py b/mining/__init__.py index 2707c1c..c528bdb 100644 --- a/mining/__init__.py +++ b/mining/__init__.py @@ -33,30 +33,31 @@ def setup(on_startup): bitcoin_rpc = BitcoinRPCManager() # Check bitcoind - # Check we can connect (sleep) + # Check we can connect (sleep) # Check the results: - # - getblocktemplate is avalible (Die if not) - # - we are not still downloading the blockchain (Sleep) + # - getblocktemplate is avalible (Die if not) + # - we are not still downloading the blockchain (Sleep) log.info("Connecting to bitcoind...") while True: - try: - result = (yield bitcoin_rpc.getblocktemplate()) - if isinstance(result, dict): - if result['version'] == 2: - break - except Exception, e: - if isinstance(e[2], str): - if isinstance(json.loads(e[2])['error']['message'],str): - error = json.loads(e[2])['error']['message'] - if error == "Method not found": - log.error("Bitcoind does not support getblocktemplate!!! (time to upgrade.)") - reactor.stop() - elif error == "Bitcoin is downloading blocks...": - log.error("Bitcoind downloading blockchain... will check back in 30 sec") - time.sleep(29) - else: - log.error("Bitcoind Error: %s", error) - time.sleep(1) # If we didn't get a result or the connect failed + try: + result = (yield bitcoin_rpc.getblocktemplate()) + if isinstance(result, dict): + if result['version'] == 2: + break + except Exception, e: + if isinstance(e[2], str): + if isinstance(json.loads(e[2])['error']['message'], str): + error = json.loads(e[2])['error']['message'] + if error == "Method not found": + log.error("Bitcoind does not support getblocktemplate!!! (time to upgrade.)") + reactor.stop() + elif error == "Bitcoin is downloading blocks...": + log.error("Bitcoind downloading blockchain... will check back in 30 sec") + time.sleep(29) + else: + log.error("Bitcoind Error: %s", error) + time.sleep(1) # If we didn't get a result or the connect failed + log.info('Connected to bitcoind - Ready to GO!') # Start the coinbaser diff --git a/mining/basic_share_limiter.py b/mining/basic_share_limiter.py index 68dd0c7..7e2de1c 100644 --- a/mining/basic_share_limiter.py +++ b/mining/basic_share_limiter.py @@ -9,109 +9,119 @@ ''' This is just a customized ring buffer ''' class SpeedBuffer: - def __init__(self,size_max): - self.max = size_max - self.data = [] - self.cur = 0 - def append(self,x): - self.data.append(x) - self.cur += 1 - if len(self.data) == self.max: - self.cur=0 - self.__class__ = SpeedBufferFull - def avg(self): - return sum(self.data)/self.cur - def pos(self): - return self.cur - def clear(self): - self.data=[] - self.cur=0 - def size(self): - return self.cur + def __init__(self, size_max): + self.max = size_max + self.data = [] + self.cur = 0 + + def append(self, x): + self.data.append(x) + self.cur += 1 + if len(self.data) == self.max: + self.cur = 0 + self.__class__ = SpeedBufferFull + + def avg(self): + return sum(self.data) / self.cur + + def pos(self): + return self.cur + + def clear(self): + self.data = [] + self.cur = 0 + + def size(self): + return self.cur class SpeedBufferFull: - def __init__(self,n): - raise "you should use SpeedBuffer" - def append(self,x): - self.data[self.cur]=x - self.cur=(self.cur+1) % self.max - def avg(self): - return sum(self.data)/self.max - def pos(self): - return self.cur - def clear(self): - self.data=[] - self.cur=0 - self.__class__ = SpeedBuffer - def size(self): - return self.max + def __init__(self, n): + raise "you should use SpeedBuffer" + + def append(self, x): + self.data[self.cur] = x + self.cur = (self.cur + 1) % self.max + + def avg(self): + return sum(self.data) / self.max + + def pos(self): + return self.cur + + def clear(self): + self.data = [] + self.cur = 0 + self.__class__ = SpeedBuffer + + def size(self): + return self.max class BasicShareLimiter(object): def __init__(self): - self.worker_stats = {} - self.target = settings.VDIFF_TARGET - self.retarget = settings.VDIFF_RETARGET - self.variance = self.target * (float(settings.VDIFF_VARIANCE_PERCENT)/float(100)) - self.tmin = self.target - self.variance - self.tmax = self.target + self.variance - self.buffersize = self.retarget / self.target *4 - # TODO: trim the hash of inactive workers + self.worker_stats = {} + self.target = settings.VDIFF_TARGET + self.retarget = settings.VDIFF_RETARGET + self.variance = self.target * (float(settings.VDIFF_VARIANCE_PERCENT) / float(100)) + self.tmin = self.target - self.variance + self.tmax = self.target + self.variance + self.buffersize = self.retarget / self.target * 4 + # TODO: trim the hash of inactive workers def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_name): - ts = int(timestamp) + ts = int(timestamp) - # Init the stats for this worker if it isn't set. - if worker_name not in self.worker_stats : - self.worker_stats[worker_name] = {'last_rtc': (ts - self.retarget/2), 'last_ts': ts, 'buffer': SpeedBuffer(self.buffersize) } - dbi.update_worker_diff(worker_name,settings.POOL_TARGET) - return - - # Standard share update of data - self.worker_stats[worker_name]['buffer'].append(ts - self.worker_stats[worker_name]['last_ts']) - self.worker_stats[worker_name]['last_ts'] = ts + # Init the stats for this worker if it isn't set. + if worker_name not in self.worker_stats : + self.worker_stats[worker_name] = {'last_rtc': (ts - self.retarget / 2), 'last_ts': ts, 'buffer': SpeedBuffer(self.buffersize) } + dbi.update_worker_diff(worker_name, settings.POOL_TARGET) + return + + # Standard share update of data + self.worker_stats[worker_name]['buffer'].append(ts - self.worker_stats[worker_name]['last_ts']) + self.worker_stats[worker_name]['last_ts'] = ts - # Do We retarget? If not, we're done. - if ts - self.worker_stats[worker_name]['last_rtc'] < self.retarget and self.worker_stats[worker_name]['buffer'].size() > 0: - return + # Do We retarget? If not, we're done. + if ts - self.worker_stats[worker_name]['last_rtc'] < self.retarget and self.worker_stats[worker_name]['buffer'].size() > 0: + return - # Set up and log our check - self.worker_stats[worker_name]['last_rtc'] = ts - avg = self.worker_stats[worker_name]['buffer'].avg() - log.info("Checking Retarget for %s (%i) avg. %i target %i+-%i" % (worker_name,current_difficulty,avg, - self.target,self.variance)) - - if avg < 1: - log.info("Reseting avg = 1 since it's SOOO low") - avg = 1 + # Set up and log our check + self.worker_stats[worker_name]['last_rtc'] = ts + avg = self.worker_stats[worker_name]['buffer'].avg() + log.info("Checking Retarget for %s (%i) avg. %i target %i+-%i" % (worker_name, current_difficulty, avg, + self.target, self.variance)) + + if avg < 1: + log.info("Reseting avg = 1 since it's SOOO low") + avg = 1 - # Figure out our Delta-Diff - ddiff = int( (float(current_difficulty) * (float(self.target) / float(avg))) - current_difficulty ) - if (avg > self.tmax and current_difficulty > settings.POOL_TARGET): - # For fractional -0.1 ddiff's just drop by 1 - if ddiff > -1: - ddiff = -1 - # Don't drop below POOL_TARGET - if (ddiff + current_difficulty) < settings.POOL_TARGET: - ddiff = settings.POOL_TARGET - current_difficulty - elif avg < self.tmin: - # For fractional 0.1 ddiff's just up by 1 - if ddiff < 1: - ddiff = 1 - # Don't go above BITCOIN_DIFF - #TODO - else: # If we are here, then we should not be retargeting. - return + # Figure out our Delta-Diff + ddiff = int((float(current_difficulty) * (float(self.target) / float(avg))) - current_difficulty) + if (avg > self.tmax and current_difficulty > settings.POOL_TARGET): + # For fractional -0.1 ddiff's just drop by 1 + if ddiff > -1: + ddiff = -1 + # Don't drop below POOL_TARGET + if (ddiff + current_difficulty) < settings.POOL_TARGET: + ddiff = settings.POOL_TARGET - current_difficulty + elif avg < self.tmin: + # For fractional 0.1 ddiff's just up by 1 + if ddiff < 1: + ddiff = 1 + # Don't go above BITCOIN_DIFF + # TODO + else: # If we are here, then we should not be retargeting. + return - # At this point we are retargeting this worker - new_diff = current_difficulty + ddiff - log.info("Retarget for %s %i old: %i new: %i" % (worker_name,ddiff,current_difficulty,new_diff)) + # At this point we are retargeting this worker + new_diff = current_difficulty + ddiff + log.info("Retarget for %s %i old: %i new: %i" % (worker_name, ddiff, current_difficulty, new_diff)) - self.worker_stats[worker_name]['buffer'].clear() + self.worker_stats[worker_name]['buffer'].clear() session = connection_ref().get_session() - session['prev_diff'] = session['difficulty'] - session['prev_jobid'] = job_id - session['difficulty'] = new_diff - connection_ref().rpc('mining.set_difficulty', [new_diff,], is_notification=True) - dbi.update_worker_diff(worker_name,new_diff) + session['prev_diff'] = session['difficulty'] + session['prev_jobid'] = job_id + session['difficulty'] = new_diff + connection_ref().rpc('mining.set_difficulty', [new_diff, ], is_notification=True) + dbi.update_worker_diff(worker_name, new_diff) diff --git a/mining/interfaces.py b/mining/interfaces.py index fcabf04..11b8827 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -23,8 +23,8 @@ def __init__(self): self.on_load.callback(True) def authorize(self, worker_name, worker_password): - # Important NOTE: This is called on EVERY submitted share. So you'll need caching!!! - return dbi.check_password(worker_name,worker_password) + # Important NOTE: This is called on EVERY submitted share. So you'll need caching!!! + return dbi.check_password(worker_name, worker_password) class ShareLimiterInterface(object): @@ -37,39 +37,39 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n - raise SubmitException for stop processing this request - call mining.set_difficulty on connection to adjust the difficulty''' - return dbi.update_worker_diff(worker_name,settings.POOL_TARGET) + return dbi.update_worker_diff(worker_name, settings.POOL_TARGET) class ShareManagerInterface(object): def __init__(self): # Fire deferred when manager is ready self.on_load = defer.Deferred() self.on_load.callback(True) - self.block_height = 0 - self.prev_hash = 0 + self.block_height = 0 + self.prev_hash = 0 - # Send out the e-mail saying we are starting. - notify_email = lib.notify_email.NOTIFY_EMAIL() - notify_email.notify_start() + # Send out the e-mail saying we are starting. + notify_email = lib.notify_email.NOTIFY_EMAIL() + notify_email.notify_start() def on_network_block(self, prevhash, block_height): '''Prints when there's new block coming from the network (possibly new round)''' - self.block_height = block_height - self.prev_hash = b58encode(int(prevhash,16)) + self.block_height = block_height + self.prev_hash = b58encode(int(prevhash, 16)) pass - def on_submit_share(self, worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, invalid_reason, share_diff ): + def on_submit_share(self, worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, invalid_reason, share_diff): log.info("%s (%s) %s %s" % (block_hash, share_diff, 'valid' if is_valid else 'INVALID', worker_name)) - dbi.queue_share([worker_name,block_header,block_hash,difficulty,timestamp,is_valid, ip, self.block_height, self.prev_hash, - invalid_reason, share_diff ]) + dbi.queue_share([worker_name, block_header, block_hash, difficulty, timestamp, is_valid, ip, self.block_height, self.prev_hash, + invalid_reason, share_diff ]) - def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip, share_diff ): + def on_submit_block(self, is_accepted, worker_name, block_header, block_hash, timestamp, ip, share_diff): log.info("Block %s %s" % (block_hash, 'ACCEPTED' if is_accepted else 'REJECTED')) - dbi.found_block([worker_name,block_header,block_hash,-1,timestamp,is_accepted,ip,self.block_height, self.prev_hash, share_diff ]) - - # Send out the e-mail saying we found a block. - if is_accepted: - notify_email = lib.notify_email.NOTIFY_EMAIL() - notify_email.notify_found_block(worker_name) + dbi.found_block([worker_name, block_header, block_hash, -1, timestamp, is_accepted, ip, self.block_height, self.prev_hash, share_diff ]) + + # Send out the e-mail saying we found a block. + if is_accepted: + notify_email = lib.notify_email.NOTIFY_EMAIL() + notify_email.notify_found_block(worker_name) class TimestamperInterface(object): '''This is the only source for current time in the application. @@ -79,7 +79,7 @@ def time(self): class PredictableTimestamperInterface(TimestamperInterface): '''Predictable timestamper may be useful for unit testing.''' - start_time = 1345678900 # Some day in year 2012 + start_time = 1345678900 # Some day in year 2012 delta = 0 def time(self): @@ -111,5 +111,5 @@ def set_timestamper(cls, manager): @classmethod def set_template_registry(cls, registry): - dbi.set_bitcoinrpc(registry.bitcoin_rpc) + dbi.set_bitcoinrpc(registry.bitcoin_rpc) cls.template_registry = registry diff --git a/mining/service.py b/mining/service.py index f00e67c..1285427 100644 --- a/mining/service.py +++ b/mining/service.py @@ -89,12 +89,12 @@ def submit(self, worker_name, job_id, extranonce2, ntime, nonce): worker_name, session, extranonce1_bin, extranonce2, ntime, nonce, difficulty) except SubmitException as e: # block_header and block_hash are None when submitted data are corrupted - Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, + Interfaces.share_manager.on_submit_share(worker_name, None, None, difficulty, submit_time, False, ip, e[0], 0) raise - Interfaces.share_manager.on_submit_share(worker_name, block_header, + Interfaces.share_manager.on_submit_share(worker_name, block_header, block_hash, difficulty, submit_time, True, ip, '', share_diff) if on_submit != None: From ad957bc4f4683985a9eb91ecda3471625d4ec43f Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 15:53:48 +0100 Subject: [PATCH 49/56] Prevent pool_speed being set to null in pool Plus a bit more reformatting --- lib/basic_stats.py | 208 +++++++++++++++++++++++---------------------- mining/DB_Mysql.py | 2 +- 2 files changed, 107 insertions(+), 103 deletions(-) diff --git a/lib/basic_stats.py b/lib/basic_stats.py index e92a53d..9762b23 100644 --- a/lib/basic_stats.py +++ b/lib/basic_stats.py @@ -18,7 +18,7 @@ class Site(server.Site): def log(self, request): - pass + pass class StatsPage(Resource): isLeaf = False @@ -27,109 +27,113 @@ class StatsPage(Resource): last_update = 0 def getChild(self, name, request): - if name == '' or name == 'stats': - return self - return Resource.getChild(self, name, request) + if name == '' or name == 'stats': + return self + return Resource.getChild(self, name, request) def render_GET(self, request): - # Cache the results for 30 seconds, takes care of high volume/ddos - if time.time() - self.last_update < 30 : - if(request.path == "/stats"): - return self.cache_json - return self.cache_html - - # Get info - pool_stats = dbi.get_pool_stats() - workers_stats = dbi.get_workers_stats() - - # Send in JSON? - self.cache_json = json.dumps({"pool":pool_stats,"workers":workers_stats}) - - #Otherwise build webpage - r= "" - r+= "" - r+= "" - r+="" - r+="

Stratum Mining Server

" - r+="" - - pool_color="#A00" - if int(pool_stats['pool_speed']) > 100 and float(pool_stats['round_progress']) < 200: - pool_color="#AA0" - if int(pool_stats['pool_speed']) > 100 and float(pool_stats['round_progress']) < 150: - pool_color="#0A0" - r+="" - - last_update = time.time() - float(pool_stats['bitcoin_infotime']) - bitcoin_color="#A00" - if int(pool_stats['bitcoin_connections']) > 0 and last_update < 660: - bitcoin_color="yellow" - if int(pool_stats['bitcoin_connections']) > 10 and last_update < 330: - bitcoin_color="#0A0" - r+="
Pool Stats:" - r+="Speed: " + pool_stats['pool_speed'] + "Mhs" - r+="Round Shares: " + format(int(pool_stats['round_shares']),"n") + " (%.2f" % float(pool_stats['round_progress']) + "%)" - r+="Round Duration: " + str(timedelta(0,(time.time() - float(pool_stats['round_start'])))).split(".")[0] - r+="Best Share: " + format(int(pool_stats['round_best_share']),"n") - r+="Total Found: " + format(int(pool_stats['pool_total_found']),"n") - r+="
Bitcoin Stats:" - r+="Connections :"+pool_stats['bitcoin_connections'] - r+="Difficulty :"+format(int(float(pool_stats['bitcoin_difficulty'])),"n") - r+="Height :"+format(int(pool_stats['bitcoin_blocks']),"n") - r+="Balance :"+pool_stats['bitcoin_balance'] - r+="" - r+="
" - - r+="
" - - r+="" - size = len(workers_stats) - if size <= 10: - r+="" - else : - colcnt = int(size/3) + 1 - colt = colcnt - r+="" - r+="
" - r+="" - r+="" - for (w, wi) in enumerate(workers_stats): - wd = workers_stats[wi] - wc = "#A00" - if wd["speed"] > 0: - wc = "yellow" - if wd["speed"] > 100: - wc = "#0A0" - r+=""%( - wc,wi,format(wd["speed"],"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), - format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) - r+="
WorkerSpeed/DiffShares/RejFound
%s%s/%s%s/%s%s
" - r+="" - r+="" - for (w, wi) in enumerate(workers_stats): - wd = workers_stats[wi] - wc = "#A00" - if wd["speed"] > 0: - wc = "yellow" - if wd["speed"] > 100: - wc = "#0A0" - r+=""%( - wc,wi,format(int(wd["speed"]),"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), - format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) - colt = colt - 1 - if colt <= 0: - r+="
WorkerSpeedShares/RejFound
%s%s/%s%s/%s%s
" - r+="" - r+="" - colt = colcnt - r+="
WorkerSpeedShares/RejFound
" - self.cache_html = str(r) - self.last_update = time.time() - - # Send Results - if(request.path == "/stats"): - return self.cache_json - return self.cache_html + # Cache the results for 30 seconds, takes care of high volume/ddos + if time.time() - self.last_update < 30 : + if(request.path == "/stats"): + return self.cache_json + return self.cache_html + + # Get info + pool_stats = dbi.get_pool_stats() + workers_stats = dbi.get_workers_stats() + + # Send in JSON? + self.cache_json = json.dumps({"pool":pool_stats,"workers":workers_stats}) + + #Otherwise build webpage + r= "" + r+= "" + r+= "" + r+="" + r+="

Stratum Mining Server

" + r+="" + + pool_speed = pool_stats['pool_speed'] + pool_color="#A00" + + pool_speed = 0 if pool_speed is None else int(pool_speed) + + if pool_speed > 100 and float(pool_stats['round_progress']) < 200: + pool_color="#AA0" + if pool_speed > 100 and float(pool_stats['round_progress']) < 150: + pool_color="#0A0" + r+="" + + last_update = time.time() - float(pool_stats['bitcoin_infotime']) + bitcoin_color="#A00" + if int(pool_stats['bitcoin_connections']) > 0 and last_update < 660: + bitcoin_color="yellow" + if int(pool_stats['bitcoin_connections']) > 10 and last_update < 330: + bitcoin_color="#0A0" + r+="
Pool Stats:" + r+="Speed: " + str(pool_speed) + "Mhs" + r+="Round Shares: " + format(int(pool_stats['round_shares']),"n") + " (%.2f" % float(pool_stats['round_progress']) + "%)" + r+="Round Duration: " + str(timedelta(0,(time.time() - float(pool_stats['round_start'])))).split(".")[0] + r+="Best Share: " + format(int(pool_stats['round_best_share']),"n") + r+="Total Found: " + format(int(pool_stats['pool_total_found']),"n") + r+="
Bitcoin Stats:" + r+="Connections :"+pool_stats['bitcoin_connections'] + r+="Difficulty :"+format(int(float(pool_stats['bitcoin_difficulty'])),"n") + r+="Height :"+format(int(pool_stats['bitcoin_blocks']),"n") + r+="Balance :"+pool_stats['bitcoin_balance'] + r+="" + r+="
" + + r+="
" + + r+="" + size = len(workers_stats) + if size <= 10: + r+="" + else : + colcnt = int(size/3) + 1 + colt = colcnt + r+="" + r+="
" + r+="" + r+="" + for (w, wi) in enumerate(workers_stats): + wd = workers_stats[wi] + wc = "#A00" + if wd["speed"] > 0: + wc = "yellow" + if wd["speed"] > 100: + wc = "#0A0" + r+=""%( + wc,wi,format(wd["speed"],"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), + format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + r+="
WorkerSpeed/DiffShares/RejFound
%s%s/%s%s/%s%s
" + r+="" + r+="" + for (w, wi) in enumerate(workers_stats): + wd = workers_stats[wi] + wc = "#A00" + if wd["speed"] > 0: + wc = "yellow" + if wd["speed"] > 100: + wc = "#0A0" + r+=""%( + wc,wi,format(int(wd["speed"]),"n"),wd["difficulty"],format(int(wd["total_shares"]),"n"), + format(int(wd["total_rejects"]),"n"),format(int(wd["total_found"]),"n")) + colt = colt - 1 + if colt <= 0: + r+="
WorkerSpeedShares/RejFound
%s%s/%s%s/%s%s
" + r+="" + r+="" + colt = colcnt + r+="
WorkerSpeedShares/RejFound
" + self.cache_html = str(r) + self.last_update = time.time() + + # Send Results + if(request.path == "/stats"): + return self.cache_json + return self.cache_html def BasicStats(start_event): diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 5705d6b..fbfad28 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -65,7 +65,7 @@ def updateStats(self, averageOverTime): """ UPDATE `pool` SET `value` = ( - SELECT SUM(`speed`) + SELECT IFNULL(SUM(`speed`), 0) FROM `pool_worker` WHERE `alive` = 1 ) From 61ebbc1305f029c5dc7032cbb0718b09b21f695e Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 16:35:17 +0100 Subject: [PATCH 50/56] Detect SIGINT and flush share queue to DB --- TODO | 2 -- mining/DBInterface.py | 8 ++++++++ mining/__init__.py | 8 ++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/TODO b/TODO index a6318db..c726a7d 100644 --- a/TODO +++ b/TODO @@ -4,8 +4,6 @@ SQL Connection pooling: sqlalchemy Add a "script" to add,list,disable users -Flush all Pending shares on shutdown (I really don't know how to do this.) - Variable difficulty should not be able to go higher than current difficulty Test NON-Local Coinbase with testnet in a box diff --git a/mining/DBInterface.py b/mining/DBInterface.py index e4e6016..7b9a5ba 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -2,6 +2,7 @@ import time from datetime import datetime import Queue +import signal from stratum import settings @@ -26,6 +27,13 @@ def init_main(self): self.scheduleImport() self.next_force_import_time = time.time() + settings.DB_LOADER_FORCE_TIME + + signal.signal(signal.SIGINT, self.signal_handler) + + def signal_handler(self, signal, frame): + print "SIGINT Detected, shutting down" + self.do_import(self.dbi, True) + reactor.stop() def set_bitcoinrpc(self, bitcoinrpc): self.bitcoinrpc = bitcoinrpc diff --git a/mining/__init__.py b/mining/__init__.py index c528bdb..fe52b2f 100644 --- a/mining/__init__.py +++ b/mining/__init__.py @@ -70,7 +70,7 @@ def setup(on_startup): settings.INSTANCE_ID, MiningSubscription.on_template, Interfaces.share_manager.on_network_block) - + # Template registry is the main interface between Stratum service # and pool core logic Interfaces.set_template_registry(registry) @@ -79,7 +79,11 @@ def setup(on_startup): # This is just failsafe solution when -blocknotify # mechanism is not working properly BlockUpdater(registry, bitcoin_rpc) - + log.info("MINING SERVICE IS READY") on_startup.callback(True) + + + + From 4ca7093fbcea73884ede0f9596ee52a42374becd Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Thu, 25 Apr 2013 16:40:51 +0100 Subject: [PATCH 51/56] Removed some semi colons --- mining/DBInterface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 7b9a5ba..6e2ab93 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -63,7 +63,7 @@ def connectDB(self): return DB_None.DB_None() def clearusercache(self): - log.debug("DBInterface.clearusercache called"); + log.debug("DBInterface.clearusercache called") self.usercache = {} self.usercacheclock = reactor.callLater(settings.DB_USERCACHE_TIME , self.clearusercache) @@ -79,7 +79,7 @@ def scheduleImport(self): self.queueclock = reactor.callLater(settings.DB_LOADER_CHECKTIME , self.run_import) def run_import_thread(self): - log.debug("run_import_thread current size: %d", self.q.qsize()); + log.debug("run_import_thread current size: %d", self.q.qsize()) if self.q.qsize() >= settings.DB_LOADER_REC_MIN or time.time() >= self.next_force_import_time: # Don't incur thread overhead if we're not going to run reactor.callInThread(self.import_thread) @@ -87,7 +87,7 @@ def run_import_thread(self): self.scheduleImport() def run_import(self): - log.debug("DBInterface.run_import called"); + log.debug("DBInterface.run_import called") self.do_import(self.dbi, False) From 2e45805a2dd49ea2fdbe7e4f6c8b729a695a4148 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Fri, 26 Apr 2013 07:28:47 +0200 Subject: [PATCH 52/56] Update config_sample.py --- conf/config_sample.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/config_sample.py b/conf/config_sample.py index 9facb16..0303f69 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -111,6 +111,8 @@ DB_LOADER_REC_MIN = 10 # Min Records before the bulk loader fires DB_LOADER_REC_MAX = 50 # Max Records the bulk loader will commit at a time +DB_LOADER_FORCE_TIME = 300 # How often the cache should be flushed into the DB regardless of size. + DB_STATS_AVG_TIME = 300 # When using the DATABASE_EXTEND option, average speed over X sec # Note: this is also how often it updates DB_USERCACHE_TIME = 600 # How long the usercache is good for before we refresh From d63e45b0af106abf50644c6e04de05651d4252ab Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Fri, 26 Apr 2013 13:18:43 +0100 Subject: [PATCH 53/56] Local JSON admin interface initial commit GET /users and GET /users/{id/username} done. --- conf/config_sample.py | 25 ++++--- launcher.tac | 2 + lib/admin_interface.py | 136 +++++++++++++++++++++++++++++++++++ mining/DBInterface.py | 6 ++ mining/DB_Mysql.py | 43 ++++++++++- mining/DB_None.py | 6 ++ mining/DB_Postgresql.py | 40 +++++++++++ mining/DB_Sqlite.py | 6 ++ mining/__init__.py | 5 +- mining/interfaces.py | 9 +-- scripts/generateAdminHash.sh | 3 +- statics/bitcoin.ico | Bin 0 -> 42583 bytes 12 files changed, 258 insertions(+), 23 deletions(-) create mode 100644 lib/admin_interface.py create mode 100644 statics/bitcoin.ico diff --git a/conf/config_sample.py b/conf/config_sample.py index 0303f69..5570b9d 100644 --- a/conf/config_sample.py +++ b/conf/config_sample.py @@ -58,16 +58,12 @@ # Port used for Socket transport. Use 'None' for disabling the transport. LISTEN_SOCKET_TRANSPORT = 3333 - # Port used for HTTP Poll transport. Use 'None' for disabling the transport LISTEN_HTTP_TRANSPORT = None - # Port used for HTTPS Poll transport LISTEN_HTTPS_TRANSPORT = None - # Port used for WebSocket transport, 'None' for disabling WS LISTEN_WS_TRANSPORT = None - # Port used for secure WebSocket, 'None' for disabling WSS LISTEN_WSS_TRANSPORT = None @@ -75,11 +71,6 @@ # Stratum uses both P2P port (which is 8333 already) and RPC port # BITCOIN_TRUSTED_* -- in basic settings above -# Use scripts/generateAdminHash.sh to generate the hash -# for calculating SHA256 of your preferred password -ADMIN_PASSWORD_SHA256 = None -#ADMIN_PASSWORD_SHA256 = '9e6c0c1db1e0dfb3fa5159deb4ecd9715b3c8cd6b06bd4a3ad77e9a8c5694219' # SHA256 of the password - IRC_NICK = None # Salt used when hashing passwords @@ -194,3 +185,19 @@ NOTIFY_EMAIL_PASSWORD = '' NOTIFY_EMAIL_USETLS = True + + +# ******************** Admin settings ********************* + +# Use scripts/generateAdminHash.sh to generate the hash +# for calculating SHA256 of your preferred password +ADMIN_PASSWORD_SHA256 = '9e6c0c1db1e0dfb3fa5159deb4ecd9715b3c8cd6b06bd4a3ad77e9a8c5694219' # SHA256 of the password + +# If ADMIN_PORT is set, you can issue commands to that port to interact with +# the system for things such as user management. It's a JSON interface following +# REST principles, so '/users' returns a list of users, '/users/1' or '/users/username' +# returns a single user. POSTs are done to lists (so /users), PUTs are done to +# items (so /users/1) +ADMIN_PORT = 8085 #Port for JSON admin commands, None to disable + + diff --git a/launcher.tac b/launcher.tac index 3c2e448..41f8593 100644 --- a/launcher.tac +++ b/launcher.tac @@ -35,6 +35,8 @@ Interfaces.set_timestamper(TimestamperInterface()) mining.setup(on_startup) +from lib.admin_interface import AdminInterface + if settings.DATABASE_EXTEND == True and settings.BASIC_STATS == True : from lib.basic_stats import BasicStats BasicStats(on_startup) diff --git a/lib/admin_interface.py b/lib/admin_interface.py new file mode 100644 index 0000000..1ffdb28 --- /dev/null +++ b/lib/admin_interface.py @@ -0,0 +1,136 @@ +from zope.interface import implements + +from twisted.cred.portal import IRealm, Portal +from twisted.web import server, static +from twisted.web.resource import Resource +from twisted.internet import reactor +import twisted.web.error as weberror + +import json +import datetime +import hashlib +from pprint import pprint + +from stratum import settings + +import mining.DBInterface +import sha +dbi = mining.DBInterface.DBInterface() + + +class JSONDateTimeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime.date, datetime.datetime)): + return obj.isoformat() + else: + return json.JSONEncoder.default(self, obj) + + + +class RestResource(Resource): + def __init__(self): + Resource.__init__(self) + self.putChild("", self) + self.putChild('favicon.ico', static.File('statics/bitcoin.ico', defaultType='image/vnd.microsoft.icon') ) + + + def render(self, request): + user = request.getUser() + passwd = request.getPassword() + + m = hashlib.sha256() + m.update(passwd) + + if m.hexdigest() != settings.ADMIN_PASSWORD_SHA256: + request.setResponseCode(401) + return '"Authorisation required!"' + + links = [] + + for c in self.children: + if isinstance(self.children[c], RestResource) and c != '': + links.append('; rel="%s"' % (c, c)) + + request.setHeader('Link', ", ".join(links)) + + return Resource.render(self, request) + + + def get_path(self, request): + return request.path.replace('/users', '').strip('/') + + + def output_item(self, request, item): + if item is None: + request.setResponseCode(404) + request.write('"The resource %s was not found"' % request.path) + return '' + + request.setHeader('Allow', 'GET PUT DELETE') + + output = json.dumps(item, cls=JSONDateTimeEncoder) + request.write(output) + return '' + + + def output_list(self, request, callback): + request.setHeader('Allow', 'GET POST') + + request.write('[') + + isFirst = True + for item in callback(): + if isFirst == False: + request.write(',') + isFirst = False + + item = json.dumps(item, cls=JSONDateTimeEncoder) + request.write(item) + + request.write(']') + + return '' + + + +class AdminInterface(RestResource): + isLeaf = False + + def __init__(self): + RestResource.__init__(self) + self.putChild("users", UsersResource()) + + + def render_GET(self, request): + request.setHeader('Content-Type', 'application/json; charset=utf8') + + return '' + + + +class UsersResource(RestResource): + isLeaf = True + + + def render_GET(self, request): + request.setHeader('Content-Type', 'application/json; charset=utf8') + + path = request.path.replace('/users', '').strip('/') + + if len(path) == 0: + return self.output_list(request, dbi.list_users) + else: + user = dbi.get_user(path) + return self.output_item(request, user) + + + +if settings.ADMIN_PORT is not None: + root = AdminInterface() + factory = server.Site(root) + reactor.listenTCP(settings.ADMIN_PORT, factory) + + + + + \ No newline at end of file diff --git a/mining/DBInterface.py b/mining/DBInterface.py index 6e2ab93..05a5063 100644 --- a/mining/DBInterface.py +++ b/mining/DBInterface.py @@ -235,6 +235,12 @@ def check_password(self, username, password): return True return False + + def list_users(self): + return self.dbi.list_users() + + def get_user(self, id): + return self.dbi.get_user(id) def insert_user(self, username, password): return self.dbi.insert_user(username, password) diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index fbfad28..402fb43 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -13,7 +13,6 @@ def __init__(self): self.dbh = MySQLdb.connect(settings.DB_MYSQL_HOST, settings.DB_MYSQL_USER, settings.DB_MYSQL_PASS, settings.DB_MYSQL_DBNAME) - self.dbc = self.dbh.cursor() if hasattr(settings, 'PASSWORD_SALT'): @@ -370,6 +369,48 @@ def found_block(self, data): ) self.dbh.commit() + + def list_users(self): + cursor = self.dbh.cursor(MySQLdb.cursors.DictCursor) + cursor.execute( + """ + SELECT * + FROM `pool_worker` + WHERE `id`> 0 + """ + ) + + while True: + results = cursor.fetchmany() + if not results: + break + + for result in results: + yield result + + cursor.close() + + def get_user(self, id_or_username): + log.debug("Finding user with id or username of %s", id_or_username) + cursor = self.dbh.cursor(MySQLdb.cursors.DictCursor) + + cursor.execute( + """ + SELECT * + FROM `pool_worker` + WHERE `id` = %(id)s + OR `username` = %(uname)s + """, + { + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username + } + ) + + user = cursor.fetchone() + cursor.close() + return user + def delete_user(self, username): log.debug("Deleting user %s", username) diff --git a/mining/DB_None.py b/mining/DB_None.py index eda5121..d4e8df5 100644 --- a/mining/DB_None.py +++ b/mining/DB_None.py @@ -13,6 +13,12 @@ def import_shares(self,data): def found_block(self,data): log.debug("Found Block") + + def get_user(self, id_or_username): + log.debug("Get User") + + def list_users(self): + log.debug("List Users") def delete_user(self,username): log.debug("Deleting Username") diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 996c093..263cb8b 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -5,6 +5,7 @@ log = stratum.logger.get_logger('DB_Postgresql') import psycopg2 +from psycopg2 import extras class DB_Postgresql(): def __init__(self): @@ -131,6 +132,45 @@ def found_block(self,data): (total_found,'pool_total_found') ]) self.dbh.commit() + + def get_user(self, id_or_username): + log.debug("Finding user with id or username of %s", id_or_username) + cursor = self.dbh.cursor(cursor_factory=extras.DictCursor) + + cursor.execute( + """ + SELECT * + FROM pool_worker + WHERE id = %(id)s + OR username = %(uname)s + """, + { + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username + } + ) + + user = cursor.fetchone() + cursor.close() + return user + + def list_users(self): + cursor = self.dbh.cursor(cursor_factory=extras.DictCursor) + cursor.execute( + """ + SELECT * + FROM pool_worker + WHERE id > 0 + """ + ) + + while True: + results = cursor.fetchmany() + if not results: + break + + for result in results: + yield result def delete_user(self,username): log.debug("Deleting Username") diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 9e1d1d9..6b89993 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -132,6 +132,12 @@ def found_block(self,data): {'val':total_found,'parm':'pool_total_found'} ]) self.dbh.commit() + + def get_user(self, id_or_username): + raise NotImplementedError('Not implemented for SQLite') + + def list_users(self): + raise NotImplementedError('Not implemented for SQLite') def delete_user(self,username): log.debug("Deleting Username") diff --git a/mining/__init__.py b/mining/__init__.py index fe52b2f..627e801 100644 --- a/mining/__init__.py +++ b/mining/__init__.py @@ -19,10 +19,7 @@ def setup(on_startup): import stratum.logger log = stratum.logger.get_logger('mining') - from interfaces import Interfaces - # Let's wait until share manager and worker manager boot up - (yield Interfaces.share_manager.on_load) - (yield Interfaces.worker_manager.on_load) + from interfaces import Interfaces from lib.block_updater import BlockUpdater from lib.template_registry import TemplateRegistry diff --git a/mining/interfaces.py b/mining/interfaces.py index 11b8827..5496485 100644 --- a/mining/interfaces.py +++ b/mining/interfaces.py @@ -18,9 +18,7 @@ class WorkerManagerInterface(object): def __init__(self): - # Fire deferred when manager is ready - self.on_load = defer.Deferred() - self.on_load.callback(True) + return def authorize(self, worker_name, worker_password): # Important NOTE: This is called on EVERY submitted share. So you'll need caching!!! @@ -41,9 +39,6 @@ def submit(self, connection_ref, job_id, current_difficulty, timestamp, worker_n class ShareManagerInterface(object): def __init__(self): - # Fire deferred when manager is ready - self.on_load = defer.Deferred() - self.on_load.callback(True) self.block_height = 0 self.prev_hash = 0 @@ -85,7 +80,7 @@ class PredictableTimestamperInterface(TimestamperInterface): def time(self): self.delta += 1 return self.start_time + self.delta - + class Interfaces(object): worker_manager = None share_manager = None diff --git a/scripts/generateAdminHash.sh b/scripts/generateAdminHash.sh index a790673..9fe6666 100755 --- a/scripts/generateAdminHash.sh +++ b/scripts/generateAdminHash.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash if [ "x$1" == "x" ]; then echo " Usage: $0 " @@ -6,4 +6,3 @@ if [ "x$1" == "x" ]; then fi echo -n "$1" | sha256sum | cut -f1 -d' ' - diff --git a/statics/bitcoin.ico b/statics/bitcoin.ico new file mode 100644 index 0000000000000000000000000000000000000000..8b12f825727a1e04e93ce39af9cc79095a8af1e9 GIT binary patch literal 42583 zcmXtf2O!k{|Nq%DgmOe?NH!ts?3Iy~y;o%KP4z%znJ3IV*ttAR!83}ssi9?~0w-DmT!t2G{+_4gE2INW0 z#g^Bju_6{+e=zuteFC|bmiB!jHqh*3X1TvcLCD?bQm z|79_@L{GHr8|OX0?RK1JRfy|z?k(PQ4Z32a#2>#1J^au6EVHpM>@b&U+9UAhK1c7r zu=hP1ASxBh_u7@#pg3eWMC;?uyO{2!Q8FHbJQsC&$w{6<ZJ^5rcx@eZzOjeNxJrd$Ad^?B+&pB~F zs;C)!fDPo+LpgOr7G*K$>IpUm&uyv(7qAM z>}y0>1ytUkAS`%ktp9^6>Yu1gLwD6A(57j>HWb8oifL` zHY!(34y|d+aN&C%8Iv|e?~*k`7t|cXUk^&9=_h%n^)Dz{P22nsd025)vrsfsyxCV$ zFvW&&wkQs?^_@0;fYN}l2!rFw?O#lC#7 z+1c>4*$a2NiF#(|S!Z~SJ8j!N`%(ADL}F+U$5@}#?YwlGe-vZwWpDFSo1a2r|&+|q*!yjAs{^zWhAw{b9dUW6PX4B zSk9uqjZ}P+pDwDrLT2V?EzV{wWGy`LAx|WKa`drIMxA)gpTWN83&(2Ton*jUN!FB|UnO zl9e_&AyzlMOZ=*=VkBxL>f1#J;=Ur(La6Vl>ftSBz|6h>o*g(3K~!C0&UsWRJHao+ z`=4TfpDzvwSwTvL`mUj+gv;cm-n46}Xt}H8`xd#p!J4_N0feV!g3ILRk@4~O@FDPv zUY4J`o_@@Es&8lQ)2(>3-c@gpR_fIJ2H%xvJk3^SwnZ$JM$3dSrzrL{?B*7-#yHF& zyG*|e5cYUbvy`*O)HXB-K!TA-BxW(clitJUDsHsF$h>|30pyMiY+L+S0(@0hLX{6~ zsdZ*%rT{MF#?0ffjxSr20o(D+&@9)Y&l$Dm%~IB%;4cO5sWKs2W2+ydp#*`XhM;3Y zaOr=!P~%7pd|{vSFcL{pn9t11Tc~$<)Zoy%>p2G{#IDjPcOPij?ds}!_uspUTA}X; zfLlF>2nY!Lvq%(6H`dozB9r2WXeO-v!3ph3uC_g%wHI*EfBiuanu=>dDW?!al8o7D zKR3lbJd|BhuAAPM=|C3C)0BV#DR6&-McNNlCr|#)Z=hBzT&^{w&T17#GGX;e9(Pk! zMTLfmG+MQCatqRwGid2~g$$f?$#rB-skN)u-xpcBRNYVIY4Za6v>5LFH_PUWo10so zC9fVeA+}q>TB^~s7+jfo!+tK`mAce^&T~pYrD|?}!nm;#TB684?0Svtc&qo2eIjgE z$e9|adRw@9jeELb1>H2RD`+q5;=DL4m<^w`7{55RELBQV`%Mdx%ETMqET-%VhVtz( zEoKWAj|EjG(mZ!IkN4kC8`gwcH{E?>c{~eje}6E8jyg6Pp#HGHW_^ShePUcd)DTlH z-JUgSL$C&1tm|fXlFgvc8f>e%rN)F4_#6ygcX7vAHqI4Sy^d{|bFlfoJ-g7=SzTpo2gs7y3~bv=Ew zbRoB)_0rpGITcM5^JaAEMuihTPC;owQC4BheTDQG{*vCZ;8?!2Z_~O^{nK9HhaG?J zt*PlL4#u=6bq3*6Dm){c;?y_U5th7ovg!~>c8TN!KZ#T)6yq>TC`sXBpitsz_;!o4 z{i8UNV4kLG0&eW#2D2>YWuwyR?XCs~5Esf5n2@FfY20F_Bn8%J8L44QUL~FkM$OAM zv9!^KlA}wJX2ZRPLRPTOR~!@kHE=szVhWIkXe*Z+aQL=oYwYFbW-PulSjjq&b+uD( z{)kkiPrc!vw=OcfM+hp%@RL+tZFOCZX`x(Jer7{Tm;e%WpTN#SfnAyEk)X=)-hlX7ZRi@!rKZdZ zC>wMg*9-gA5ZzjbYaVTF_|B=XP4=ku2mPw8$jC^gy4kPT$c|C!f!F+_DoiQgEP45p z6eOlSm{JtsP$S3z?tI<$iqKe;7E73fIyp(Af{&4`dO-!7miB4l@JO)9^cQy2RgXc- zIX6WhS0K#lpWs0jHkT6>w(@YdDYr!9oX7P7XDy99S^K<~!gA8~#RB|tT>6O`C?e_@A4piS`eKncM@vkz01_so@5RG>yj ziMc*S8P^O@#bf&}aPp{V(ubA#vdI!OQ$HOaqgr=uejy^#nx$0SQOP<~IX5>a2s_Ri z&m=y2(P08gMFQG?SvbM3LmJ&1iAztrbAC_HY@t=(Drm{kt2JRbGewz6S0Kk~ zw)|=GsCAN3aRDp7{al3Y+-E|J28ZjgkG51=+TqpPw=5}xaU^0a+;a_RrZT=F32U?) zy#u2dL`s`c{2L=7FFp25Na}G8e@B>^aznwRU}6(|y|@MUh4eezJ`(5Y2RGNUz`5-% z{H9Ug^7}s7oX4akgXHmL8?T7qw1KYfOior-bNZlV;5X>?p*-yBM=SB)@(vidl|77c z(PL$SjNt$w_&CbGbRqDQaShkr?!HaK{=7@hXbGOwjy+qryTHpYnwlW-zkAShzNc#@ z_n^i})nFpU`#521S*HiS!e=~;A`4la+h)WvdUG$DWd33fMjP^zc`p=faM+<03Ul#d zEsCp5$U!xDvreM3L(k*+!maC~jcYa1}$wpipWwb)vMKInwXdX+4V|WTPw&co0)pxT}1`Rip=b6sdBCE zU%$xRjcA{aXuS3Lwjr>c6mEs*oe zik;{A^WxFbagmoVUq%uT5a?)@|2j9CMozK7E(iO0#23#W0N8+eG1b`NQKH3$lq&jh zpp(g4MAyr${xcRbiC6Yq!DU$H+_XMdOk zLQk0)`CY$tL9TS-u=s;*ws_C-GUm4@Uw7&^tSq}TXd;g4Xay-`m^+^nsJAYkYid6j zwESsuG6YTU%Q>S;~J!X6?`BN>zIXU4|h2rWO{`$XPgY_SvQX?M6%0R3vf5Wc}#qXz$n< z(e|mYwbFE6mMdj(7G<((spQks=!Uu7?gj_2uKwbQuFY{8c%@fvfq#smlD#U-=+OG@cyOzb#T3%P$1SX_vDaNGXYC)x@wm4d@#m=UQ{6l96$8(`DFei7-Es^X z`reri63Gl)Bz8mo`1P9c-xwDXOIhN#<)!}okb0ik1%mVFQdFay9BT1~pK4}lX<%iA z{t77$xe&)undpQ{SI%1cwYHW`*#Z2g?`Rh@!o~S}V`_@%B_H4ToHQug1#f!K^3RpT=Y0uG z7#AOUlDc!9R7dH*DEF`fW}NXiBkQiLX-eRAKryM_UR_>>`1$#jO>cv?*s;Wvri2z& zFkxrr^$j=5D%(yt@AX3piukn2$;OO~41_GM&)zS2k_1g`y5(6LILCI&?e?I|D#iJ; zG`{IYZ|Qco>xMO6>+$h%i$F6p&1P!cWxu9tq&T^(WUX%R&!42$*472v!oqn(Y-#vy zGbk|`fbZlfH_Lm(?Qc3F45dE;}L^LV(eBxX72{clgcQ% z?u;77mu_^qWuee?T{Qvv-t8#l%5MnjFD;~lxWJI}E*!(9&>9?yG|HE!stn4s*i7R9 z%D?)N4sv>7*I|eDaW^+%l`cD5<+Eq~SK$Os<0B(C@THr-f(V~2T5p_vKkS2fw{fFP zcpj?ejQAXN7~i-wW{e12aP$&BySooPW9Gh1Th@x0F4N%GmFr};tE&uCQ&X>4$-i&9 z`{&e8+FjP}aTjLnUT|LWaO+fxwNXAKw(c+45r4zIyWLl3rM<2uW zPQaYT|GTlF&YAIn6(4LKSXpOhZbR|+pu!EWOW;bj&h~=-yW<=L7m_e<(bAZHt&_7c zJ&jtryu5^6cH)RTKXZmG+ITPV1|x2U5xE>0!>DkFIa)-0^9!%t-A3E>gYVfx>oZ43 zyw*cj;K~RJ@oYSbOr@PwRCM@(x?BC+Uz}sEA9}Yp&!J4a_U>R zAH%rTgecKSlrIXgnHp5wsfQ@TvrEkE?C?R>WWwR{Y-!yfWd#HU)yuUYgp`s3dDyJ7 zb+f&BB3`dvh<>anvE}3A8@sx?I{Vq(?NxeoSrc$lL|fHI^&YO##zonAJPt#tow#2$ z5}%!owO?&j4vn2PsAx|b85{S5)dk(tw>=-F$efaylXHxRFs@nKTWAG)`|s2j(p}t6 zU6es_;EZXDW|rOcF)Ro%F61afmmo*dt1A?WDL2Pv&36>8B$}6{5o5?1IXkPz&cQ*S zrF=z&W`il3qoJ+6v%gzw(CC!n*jn}Rr)|rZ7CVzB=apTJ@{@h>!)UcO{>!o(_vL^- zNPfWCtZS|AjYCgE;AXsz$xDRCZuVPfgaKOy$cG4f0fHWG|Bd{L3J^Vg6{TC#+ovD+ z%eDMV^K;k6EKYgRzEy7iabxKBIrGUX*rtBcj04^8Ou-XWe)N90-wM<0Hw4EfR}>@p z`W|cXUGs*+2Y&ucuDL(o=0jHh`&APKJ{Fi)2(dAC_I6CYyfSnfRF3)>#De}RZ;#B@ zje}X13qhL~w*LtGz$@t73$DOSiaMIwo!jO@+$sVC5PaNS7N%Y`46r*Z=1p z;dWR2wzs!I(E`P{Dd1WFL?B3bTzc`YW5Y8#TUyzZ8&Ua8MMy6D%5Y}b{oZ&eMBW0^ zEZ`n5F)ls0N?l#uP7qn{MgxnBkE=}(3D$LZ+S=;S>Nc6WADvTE5A7z|aErIVKj9R; z`nNqiJZy4vF#x+Q`i6G#v`0-0!~1Hp^b~xcx}J}!A+G-BWC`5cH8*;^=nulBC)E9V`q2Qj2l`lvU0h*s?_-=zB)gw2ly37Sk~M|74EFE9=~nU4}}>hwN+`ef?kLkam@ zUmp%$7aH--T{zyn7PlYTu)o-sH`h2@uvFT&x!DWeek-xM2DOCxQ7V4A&meNE83wn* zKDYtV%ab;Evp2T5h!b@3^QEPY*YpSlzL4M9C)$R!XOsLYJwrqIV{jO;sgUj7V!Md> z+Hu$e)r(b`bZG-gC`Av>s?~0^RpF3qy-hN!1UVzFIl8(B^CbQsSrs#jek8_?G_K@_ zSDoNRK$!sj-ud{i3d|fJq4!F~pyqt8Z;tL!!pbgrlAfNv0~T_hBDOCG^$cNL{R>-zm0p!HLV#hHe9fJo#Y`(T0sde;Gd z{|311fMWlkkHcKGBd%$zz3BJ(YJH*krOO`e2`ujYi-f{^GLx}NtSu%J!Edpux23eT zQ^jc2z*d23a2E-xrJrY096@5n;C(dFd5Gq zu>`m<_5r{oCBy4SL356cs5M&TKl#DYM}o@FlT1N`f^49z<~;t`QN0b8mVGSu1z#$r zvGOecXwvEak*7XG_s$J>w@AWQqswe{kM5!VNyy`ej8RIg_-^Fr-JX`owxB&eeE9Hl zLj$&H*@}acQ%{2f08#)>{jz&*@8IxZ);`&iAxT%nufcC`zPW~{zvE`2h4F{l1B{E? zb3~A__9W^OG!@jpNUrujT5q4f_4G{tR8zy4r+TgDpKfL5>G?2G{EC$~i_meQCETFV z@sdETDSq7IQqnUpwO5>T<1u22>F`F#^R~(6aXJQ_IcZ%mtw@X>Nt!YfAG*2-7@gCt z;xnt=wyvH2N}5Z!fTIuYI?d0D&1}@UpX04 zy{&6t&~tgb@x!sv<#bD1t5SPOv9IwrX=4v5OAYH`A0AYG)eUDxedoqyeQoA4MSOd0 z3R|du{r&+D=`f8pBMB1S{GVrF;A5~que2XHwVyr&D~ADLO;-Z+!Ayr6j-b<`WrP`P zu{n>+<KDkL_HU|)frES-$la7?Zf3RuZ*4$|z-}XhG6PR1by^iw+_=Qg zpjn^o0NkX85JSQILUB^+EH76_dQ)rfPkm^{Om z{4JMzl@ftAV2ee}W=_JA6(#L2UZ~fpSzlUGa`25Y(bUnA2!wc(?ZAn zLIIcu+mKXlNHi*k5#R6p#68g~y&XPCZ{6UEME_~PZ(mQZ*5?H8!lea?>IeS5jFkv#YZ_3OLV4`|oIS^Fzhan4KGaxR9b3o8bE zhKeHJ)1sls1wK_RI?k10A$&&Y0A|P*b=rN*$)o_x&h~A>3xSPvt9K+JU3NQK} ztDN|@Cy8x`oQ>cV3J#JFW6~$+!2Vsi;F+9B_BQK4ySsx|eJ+iu&19&xZRVdB{_f5? z5Gr9>GVpI`L-=#anhMD+I8uNN19VGeWu-!~#v7kUDH=*Vf`XAU4DUQ!9khvw(~KLh zXS6s4-Ol$GrnwB8zCW7#;q`RUjc~aJbyWjPRb{%}{W~=UyShE^5acH8@x7@X_;z(A zdz{^T)+c2Uv@~AiHWP z%~Jr3q=i%H^Hc{a1gJhPIGV8kaiusTCUVS&1$B_l4uUJzBwTLZ$iG&QnhpxO9?OeG zEiIt}2Fym9k9x^0Jv<(aT9e`6;F!j(+*}?v`CsyaiPYF<{Q)&KbufwRIKguv*5VJf z{(*)!-Xe&L|jA6ileVp6@0%1NnW4YXqNMA)qmr(sj@!h+Ea zpp+%+W=}M|`*dugp)9*SL(v8nIFm4<=in+6^}XE@f%&Wmh1?_8_98nhv(s*%?vEs^xN-upeahr&h1fvR$mZ@EpQb5+kp?6ih^r>ob^NOis*u6Uz}2iwt?^eEyUPr{4(U^4CT zWBc^`8~*nSOwNBtC;`zKbeyqz!mm_J%A-6{rOoD9_1a)eRV8n!HQ@Rx{AI=>pOaAF zG+(e&;BgMEy7uXky`w`F;$k(H#%OwBID;z%^kpr!2fwyGK(`US{F4GkSMRlWL1*NZ zhfo691(6uhVWYCQda@wf1}c#0KwZh6+2gm~}PPdjfu1A z_cK~#dPr!CQN}nU)!GTvxZ}qyomr(wg;?qhkLa%A5>dH9zA+=5=wis z9C>tkgXr*C>z}=gKu@+jY?i>sYjDt*BE?VWA)U`^+Co4{PP*v0`KbiC{d?afDv_4h zZSU|c=6xyhRT3w|?!;|V5j^TXJ*d`xpfbiSyU#*_?@gAxEK%?F(HC=?;V<9q1psLW zzzH`WU#NB6Cf~cwOw!VX ze@BnG+Ahpl%C~o-pXn$}X9fVx=aOrW+`8t`jJVWh3 z?O{>d)lq$+RM1g<5%UdG7|Kyn_ePA95x0URKe5FLia}(7>2ibV`X&?gw^i;QHpU1b z9y~p#ydIYDQG!Zr4#nQ8&z>lo*3sBJVFWGcH^9e4j3P%Qht(F42(u9u(rDB=7}F?B zd{@F9d9y5a^1VYqg8&!<@Izn(TPZKimqIKJ409pk?CQ4=S{h7&rdm>vuEhZL(L{4X z$z;EIrUxuW@AGK2VQk5CYCpX&3UOVj`J#^CcWWv8r*M{MOq68fK7H?!Ov%LNg;TIy z<6PO~Hb^)CmHIom=Y6aG@@J0%UbX&W=vmDTQd3Rrv|9O5pbk&4O}-Da0V{c+Q&13! zk?Up|2SZzXVOXd7JxB|H2LXX~*D<46@&P^Ab+^ZIMLbeB=0Z476*QXCzw(NQiib@QVyC|}yPO<+qM)=~J`(>; zNMIbTd6qWB)d5C~bldDR#zfIor=C$VzG}nf5P-Ddp!x+y$xWXoaWq?nm0x!L%W>}> z>U$FSf|6C)H*)e`s5gx`nk=nM=X1qhkvsxFY(`FR1Rg(`eY(eMeK^zPanCj2l2Wez zW#Cj>+y%a%_)^6r+g+FC?svXCvQk!4D>AN;*4F-1U7cKRKV>gKHTHKj2SK!q@B{K9 zY0|wQT<>Y_;K|9!r7ibXIM{WHdC)?|TvyParVZa-uL0G^Wqs%YG-8G2;$O?&z(o(K z{~yJt;-;!MF1Rp!e<``yG&2n`_E6`LZRf(j1JSyw{II=qMiWiDaRm{}Ivdh{%IC)C ze4cA(x+LL}uk^+MsoqhpSM~bvYP%*>BL;vl&;bGUbFfgNU~M<_X>$8yw{huKWr9D* z!op(BNZJiTaud>sbEngsO?w!j)J%L2Wq0=Xb0}y7QbA09nAj|sw4=`$z6;KfLRRm~ z^^70(0s(~^!`^*HmhYDF$eB)Mzok4M!%VFibs<-z)#3`C*?+NLUiPb1;5dP-*^s3F z1#Re%FCt4P@(8tfk@CI_mu&K!n}r6)ro_Cp_o|TZc@$bA-@3QarFa}qkQffvGyI1< z4w)Kvd=j7@1sA%LQk(mtt}ES- zgS|#e7>E~e4o+)=khB}emrK__5(BlGe2^`*Us|@fcFHI16qr-Ix5`VopFa=JFbSjt zZT_77-~DKq0!KnlGmBaSH}#Y0{OO#J3C<%rjq3ug4~+f|ME%}FmKbL#Xq$)<4{Fg# z$3(h2%3!?DN?5v}ua(Us*zwEdUj0blTi_=*nWCcfgSay*7_h_c!k^g*0GI;s66kc? zDsD|!&D+UI#T$Q~mZ}EbQox5|5l22*q1N8##C4Y9^ za?DhrlE)$%qKd14mABA(a(-jFUytGRlTo z7Mn1#1N4r#90LQ|?;c1mw40g_g5DC|Z>T!m#>LKSeA^dGjr~`!KrR}#x#MLiW5>&3 z02$883X6i%u=DR<=EcBk2NGVxv0<^#X7eIjGcF%@`rG$;IKI>K9uoYVX0hkYGn#&q zuy#SZbY1d=c{wfG{mf@0zyXqvw=&z;5wU!U-jv0?im`lZ+5ZmWMyFhrF~E@^K=)Xr zLXWOd?-zOqls?j74lu!>THkTNjdQX1()I9HXY<7vy?ecPg*ZNSJAj2ci zZf{n?<+mo+ziUb~%7ZS~GE7tfai?c94nMD}@DAxCQxkm$1_i@b&ry?^o@m-8FER!7 zFK*_P8=0|1dpL?qWmo6rd$CZwTAw9xX=kI1r~%Q!Vh}I=(DmLu8kXv9jieV%)h|QE z)2-A8T*{Phq?Y_sAR%X6b0c_$46K19t6tytKU*@muqW6(RPM&L?0z_^a+zC3!_$95 zG4c=}i=CbQLPP^_N9n7VB4_Y+KgYdo|jD6-i(aj5Zcf6CkEYEfC1wz*gDiezJex;?S z9s4pevKl88%{bsxTPPtEs8;QHGyp{2*Pb9v{8WRO+H=3X9uGXLBYvXz?_EEWpnw2r zU0ofbG2r^_2#79Le`96YYd&>0IDFOTD680V+Pdt2w6-nXp ziz)xJu6yp8EWN^(U+Kzy*)M(-ySau{p#(dE9GJaUX~74^2*@-1nB%P$z0Ien3cWZP&Dyd^O?|6+rOa-ISf{$o!v@;{p>NaoTGTRsMNL#I&N1AaMVq*v;e8MpdQxkphd+&P z$Te;xNB$%J*?tQrBJrPQK^%|Cqh%^If@p!LHFsKO4Z2l3%(Z;&Fxx-yjQyT^NPcp@ zGIJNS=j`l`P#((@iIn=l?axzQZQ)^sMOEzC{(9Jcf9K}!bjMkJ+P4Hakq9mWSH%mdCRAlUr z5c7B-6Voak@1a5Z^VP16-Zyg%=6dN2O6%P6X}CQ3G}^Dl|1RXbr8?lhXEOKR4JWKa zh4W}2PLqV$471Ju)vU|;pVL2AhR#n&@pRL}M@Lmot^@A4y)0#-ys$oo5*zDGhqZ{vMrwo)Q_`Mt)?m_$?p;r51foTd*H{rI~jJ1I; zv^7)rR+n+|D8~Ef|0(W{lnI}|yFWM>J@|G&p)kQF<@&K7-Mlo1H1iFe9LZ~w6K}u2 z5#1^zAL0Y&`oyUP_ag6OB8k{orYVYE`rC()hF^HPXO{r10#?GUm-hSi>F1`;!2px( zdbBpx*PptXT>Z=t7wecFh$ORtn4qqvVV|CmBF?7j-$q;4+1BK~0HXf-9P;hkw|9IR zV8Mj*5@4p1Fa1L4#B-Zsy?TAsJKhxx_OQ(3kpvdljXYt__7D561#3MlcD0*itG?pR9|LW|!u!t8i@bb6Niv2BYh)sP%>2-Af?M7#zE~ z?cMCK1FjA{N&t+!z44iE@gM*QgD&iX9znNC{AITfubg#ACLhHuQ+{l)r~lr{WB+RNR8t1m}=9GsD2aV6Etnn;4n+$n>E5%sp)7&6llP`K2WjbQ${LF;hD zf&h{}7U+e?8{>>QqlFWjbQ$9oC5G1N4s(9j*Kdh`k_)M0bBl;v!zm*U-@To#egVun zdPaKmM2{Rx3T&w!2b`#b`2kSMT@AZQmbB`p8=;Sie-96P6-5OZeuClN7)$bOSzzM5 z=VYAq%SVpOtP*B_wQLejyx*+}iRk1OAMYk{H*QNmY2IikT;Q@M%TL8432!Q)<$B~9 z&mHE#!!^A!9Qj-&)Hb^VzP&SB_lg_~aD{blZcOlTk#@(7g@wi2<{JO?XNbqXYZkrH zJi)NrHJI_;yzRZ#9}k2XZk>#b==?!ox#{Fr$rbZ^cBxL{GfISP<>y}opEvG+FUv!piVk2L9J%6>Wj#r%bubdi*tx#a@Y@BCr4a4r&Iv*m_8#7Pcs3Vfp?3x-;159V z-wf7>0cDG5sdcaXR@9JTZ;MgbUM7TS^>Z5$7nzMkR8z%DC`2uL#bNa)tZ?U>KCYQG zxsEz{x&o7HOEm>BssO3-FIlt7A5hW`au>Bg}W%f_r@3viPW740^DfX z%tL&9-%}J@C+$m!iHT90I*6;u@Mz%|^r~TKmTt{K9j&dcU+U^C(uS;n;Uu#JuZR1l zu^PzXG*z#Mm%j5QDa0w$xp0R*j`$Qac#B47SRY$?1t$+_+jpH4#q%3ypc;Rn)Z{?K z1z)s)2o&>6;;I`>dl*jFQ(6#NjG!$C9WSlQi*nJXz*04Ef>m%g?@O_h(g}zm`V@rt zM&j_P5^>(qjR$-eO&%W}R}#Po*^1exT+XTjaMf#XzH4NJ5ZE3-0er8-%p)c?3boiV zb=ps|tG~)h`6n_^%NGno9m7x_17F0i&f*`%vBlJV;f({kc@KDJKzRqVqN$S;aq7<` z@1Xa89?Q>s@e)u*!g~+hlA|)0<6g>2y3T9PysdloH=iIu_?1#hs)lS^#!UbMKu1qX zi-#yKaTwdKrby-?lI$mC0rQ4B_Q#ak{gX^yUJX=900ZM&% zMa3QKP1tkr1W{tkm{6sN=a!6f{8UhzP#8(*HGQuD{&+Ip080!Ko+`kN*QKh#sX&0sHeN3@h6#!)NUx;vjoP+0!ig^h zWo#fSIMH_O>RXz0s&4f^MbMxP3_*Zf1@>n4Y}Ozr$5Uf^A-qo(bV|TJBdjm)t3E0T%%tfJ;Z_g@X>D z$O&(3$s?MZo%PunHn}|_lixKVPl!jm8HC>`s%lS~%yjs9wP;l77i-`R%An_|;>7Xf zig*zH-^*h3>}hhC0LsmH*a<9=2Nb!=_spucV?v z?x~8m%uon_xR3@c;>0G7XJP^!efqJt!-Mhr#sUg;(Y$znc$!S+xsE9cX+CU(euW(R zu*Iyt1$$%;-UtXFeFl#KW&)R-RZ;P}{k6fT>1_mQogdlC3^cgJmKGRaF@T0tovt9X zo<%v+u!|70a&h@F9ztqTO(K{TLL2KV;iCCYP5)}(iGQ=c7MoM%*T2l~3RyYnnLsjK zh=p|YRMSl2Tn6ShnE50~u}!b7TTKcQ^%n=i6i~tc4pI_vY|$i-x!7eh3B;?h0No2n z-@7`TGuq9sqP*^EX=gXIb?WQxFVF%o@@eGnOR5$OgL#Qw1{)$-Xlgg|>n z^2IosOq4}eg>Y}wuocx(LIi!*Y9CPKk9Y3wnu}ksqQ6pe17hv)UdjLVHNDo?)me{U>z$-{Kx3cOJ%u-tC z{VLN9a+@Gte-}=39xI!$ZVp_LuyLrLznj~HvJLMTfbBqA11UK_Pi<{u6R;BD{)VKf;#tnay9q7Ws0m@ieQ7^&V3cDJVPWk5(^?5=g( zEJPVsFF?Il0k?Tznyh|I!f{+^V!2ERFVfVI1LAFCcv1S}4UjV+K&}Fv;k zS%o5Ozu2|kuB4@I zY}xzv$luCYk&#H3Bng{iBq|hV{Q_fE%{dS%^)v3(*<*CF==@TYZ-PV`N%FE)_s2T_ zxP4PHZPoW9y>%!KhyeO8RB!`d^9a*loB<&@XX>bD>TKASQm1#h>2Y2e*< z3ovpVESYNOXVK$nKL-Xzpoa(Ydq$cscgXX@$AfS?$r_!O1i&GS*mYi@Zh@-tKi5)i zd;2b5SOJCaw-@`XE(zB(?vw*hkf9&--)GZWPY@*Vx=fKZS9=rNp3-WP_pRee82jWu zqb@U#o&HNQ8cG+zs1PKF%L5ULDSVZsTQowSMH%O&5pfE86PFm{B1J#e7Mnr?Juw+s zU#8LkqEEg4X?-wYJb*9NUSK;jqRX!ScVx#wIF~=2BdUL;1KQc&FUOEGdh1_r%%%Od zUb5pAog!C5mI9OiCZe@<`z7l0LO{2@d)Cvi*;`kmEFT%3h!h3f?C)~->?OWi(uW-|D7}Ap|HFOmOJhk3mU9jzw`g7&?*CVz?3yz>C0{#mC-)?Rd zOSc!vqn5djHKm@d3&giSXpuj%7(Gf-ae?hnt-&*SYm<@vG*ecKYX{kWBMr}zH`k~ z!{+NJtX@ST(r{WYOlDF%FwY~A0o?tcK@;nbXuW5{N14%Fy2YQN{49=0k>7~Y`$V~3!t z7b!CW!n}`MTSD0jT&_~nm_lx0Wt4Wa#jErqi=CtU5p-|y7;4R-f&AaNiGwj0q<3(f z65nj&GY{()KBmCW{c|DIs{+X;SJw|F{6X>E5wQLIuh`Pw4_X3k%C(vLhsVN}yYMBj zejk>?16#OVLYb(u(3nyNhZEL5?b1@~&bm`g^f7>>11eUC4L)!bbi3rKX8+-W6t8y(c%nYp`IS+htr)u_J ztYwdVo-_I~-Ac^!&zO(U&u@W;u_o+qv(Mb>Mp2BC+8RPpQP$DAQ`cv880!sK@ow);=jPuJgXlTU zYo9=LGR)v{T$u=xsRp{ZiELF;|3&BBC;tpz<+Ai9ZE}Di z;xz2|AH{1AxNt71!I`ZgfJY0r*UCcQd!FObY6LXAJBxgs9-uB$ zt0%M=%ua;!?2CRhwbj-pMK!rIF+Z+XX0F@&eHM-k#mK3s=&4IHobCe=0a*qjz@3|I zGN}m>Rb+qTk}BNxc);Z)s_8u*n}XYWX>;|8HfJN)EmwLpgY%^>=1Od|G*R&{h#c=j zjO*nrY-Me2ou7o3CY>^2;)8*BEX7Z#pL|fBLho&(xbKR(SbUeJgjG>dsLPHHNp4B7 z6yAlVgA*f{TFr*WB6dN{=6BjNI473B<=N^sD%bOE!&Pg7AE>6ls+Ows1Xy9d&N=X? ziv5(VJ{sQg{a!#Vd*$}vn#q|uMb(-r1W9}S8=by9u9k`N@BMkVds&&G3Gtf9_A{4j zm@^8N71cb-6^6cV#@06O`g1&UJ7LsSlylzc@4t`w!Jp8a>lD?=0bvW?1-#di|B!W^ zmii=cM;M#M(Y&DI{t?B&!SMhXAyTd*BV%@jJe#$SMD)RV4AoLlM7g1aHzSAwSy<ooW^&uvv(Q&~r96aSZnS(mo&cxNELNFMfFU>$Of2cM=cq;fJA+Ow1xnGJJbSrG9VH8%2^g} z==U@Om=C{VPiBnMupCAzQf29`k|Hwe2Rhd%PJXj%w-3R(-{F6 zl8O?+Tpx@eC8ZuDJiQdZU~0$>{orRZ1iH}U<@4kL9wjiiPzl`1%gz`k*N+G()3KJy zdg8{Q$0dRZ&Y60kwDQhdnu#b+SQGLroop$z{k^4r1Wn(x#x1nFqLvzE`dxJEDIQ{q(kQC$LD#m&he(S8kOq6>V zo!wC~ZLdifNfL)q4`k^fQH88T^u4&px3LZn_>fJ%GA4s$)o)JmU4EU>MI06=quo!* zM*s3T_F0&rQ>>7qTxCU!YAK(8vG<2%g6VR@gt7`dkCp{W>$&Ze^Ec?&ptW?XU=Nq!k`&K-%r+a)_jYf~J)=so?tbL9M)btSaBK~ldHx!R=iPEl%9 z(yC~Yk%a$oMTI#yA@;hGK4p+DWB6VjVWh<_G$QGZm#|KPId<=3GnI)UHg!(^Y?YVV zsVf~;5oqWIuULb5BJ`{Nx>eJin%vSnLCW?j?nJxYvIC+hV$_lC%&_%K4DlSeP=2eeAPpok4HSvKp zt@E{2Rw79~f+&3w7ULAyGgQYMN_4x)BkvvTFQQ5b zvEjp3VVIp|=7SKD1#9&~Q-%-E!g;XNJt&f6x1Liv4$5+Cp&pRF&txQf_3B!tkOCoFva+y*X3D)% zB|AUB$a%2(8@>3~_$`?-dJ=tn7*=R*h?zkuR+cQWG{JipQvG0D@u>*2g1OxzT!@yo ztUw;KmUdp3>HFRDYc=I;a`ITTk&zKg52K)uud7-(Efk6;z?>Dw1MDdzyz^6;ek`&Z zOG2Am#Kwl3wCU$ThK~XYNZ zk#rn<|Gd9bCstx%hGX@0s4J}qB_KM%F<}hI*OkXP31(2G)`dXeZc_b&6V(UVQDy0Y z8UTkvLbd|Qf(4|af&o4bQiwliG}?Hd!^@L&&g5^2%jDN<$xJbhMwU;inS1UV9RJ@7 z;7-(P{oQGQjZ{htLN(sBhD?o?Iirc1CQLq)B$QVtC+nejr}5z)qs#!jxuQZ@oxH@B z>R9R>qYtWE9Gfu!-_9>HjL_Vkn92onw#nCJqr@KjtA5pr2u4XsuZj%R4V)4W1*gSd zJV3DUPzEOtemcEI`nN7LG;ek3Tpc1&*T~zb78Xz5gk7$5oYg1zE+WxuW*nD!uYO+C z4C7bQKDj?gqS&7)7iKR!dq&Lgi%tagk8vK((@!`a$YSORLo^7fhaO=6#)P%FwpEcGv%aKT7pKOckM zSG}l?i=*32q(4mLkL%4$Lhs^*nD*S0AkTVJ!~o&_frETF>^uEc>|StGw$#jX(NZ|q zW&1)*=Ks-jmQhiD@!B3>fFYEjLt+T&4juT@4bmYD5+b02f(#)oDb3KG(jg!qDGgFX zNQp|ff`F7rzngR3vzBYApXQm_&))l2_jM1GD&vsb5f7LJG@(PoJwDMmPf7ARbY?Z1 zrMv~IF`ya%cabJfLeJe3j2();M|f1XR;}|}JP>NzYyNtxTA*OjkDhF~-T%T=HkXhU z3PNK~k-$S&K^hO7F@1u)H%+H#)|aBHA|kKy!|urrkiuU&ycH>?Hwq z593NE3LTE*f}9SkE{y5AAr7{O$OArHsbt^rcsch?#r`(eJNyBvY)CcBNSc6I%_5A$%;wgR0=exP5o-mg<-SW2l}SF>xdz_>jHB(m z#%m1SV2ady4V(SV4pY-$1If#T+;f+=tOMhHBB7FVnUY5sv(Y6-$X50f@SD|3ltw>K z#qxfdd9BJ*PiMpaS5ElsHqL^H&}o<&MD;Z%XSdF0mo?Ux(a7$vu-4T20AxEP{s$(XL(YNOE0m~_ORLUhg~{#;>`g=0TVTk*8%zMypU#@=5(WvD8J-P%2(8FuFV z7YUWSskiFJ*F_aBX^eei!(;NegG+D060D+6dmfQ=hMZjm6S?3P8Xe9kEDU2#?7 zhr=>FR@n?{g#tuqGH4Mmppp4EI^~&da0~rAcl_CM8~%ihAQ;t;ezHeFH3(t$`3=PC z)A(5QRG@uJMlp_CT6?-q$-C{AF-VhAkoP7RIvSds-pk=hH!7Dd8FETwTN!>!?E1-w ztbKyZxmCY2-j~i>@@v=n_dfXw)v$!XO^l~;o|&_`#_h2RL)y>B!U^EiuSs1zL*HKf zYt|^d41SEum)QceTP z@|LeVZsPso7RJV1t!=}Tzr=xp@YDQXv1RBW&9_0sFF7Z@Zs#mkSwzlJ>|_cu(a`CW zyw=H_dJ}VSBP3izwJZ)ryCD{L*E+2M!E%yQ6uf*V;PPL`Z`j(>Rv8`c)1EZ~tAnt5 z1?Dhh_z$|ZHe2F>{$Q==D3Ux&S&0APYtBR*JbCWBdQ{kkosX-?3dobYP!$a5tr1zw zY@W3-qapXRb$a&0WfBJ=5Q1QZyhxh*bV7Lhq|l#*gsaY<9HcK|X-Av6h~WhKK5&>$ zR~$|_bfZxtrA3qy#>f8kBq3p`?E3cpguh*cP$r3+8};L*j8I_>mSUMfEzSY0@E-BL zbmc_0E%Lr;-{8^x)WC!7fnq2NUmqQi-_qFtNmE`3B3bhmJ5EC-L#T+HA0sIkTA*d> z)6BLUv%`qBHm}N0ih~Bi|MaB^5ogJhD?0LwYPT+N)W|AAmfvuxf9s_xWF^PMe!INz z$yz%od`}>R!>zSNSg+l%yk5H^viIWxA8}H;Rr|HGsN-4G0_9&y6C472vK9C4vrh7- zf;vwWrPZ~hbo=X3w(UUe-{1&l48|{>4x`QczO`m!KWoB>v)#e&5SNo_E9M?@1Hgi5 zW!&$s&Kr$QywcxLOgfUgeJrQ$t*lXw2q#n3i&}pjMY|ND zqAZ~B&}+RI_scJ5rcN2Y?cQ_3V}+Lm7etSx?iEdOESTY?+lTYP+71Q%HmM(jdv0U? zFz0oomNMfk_bt|!=E{zITIWRGq5bPz;Tchh!;_`I~1mtX+efaULR(-xb7X9|3^i=coVwT zk4GQrajWlcU1?;bR*+{@AV5Wt7ydq?*xN?x{Fc>qxhm!CX_Ax_-%D*!ILB8!cS% zBO_k-T*nonc=m*t(%hX<%`W=%)erI>r>tfd4St{9VA8z z_sb@mDbo;9NI!ZaWpy|`bQx*s8B}rJQU%msbfF>#h#(_o{i67E1=DtU#yZ4rUvp)R zYvB?jBR?I_6@P}fuY8`DYNHW6%?C2SgpUgNl;&(quu!1}R`qq-bm*%(`QPA0T-yjx zzqn|O!}-~9g=N2rch0*{z(3%{b^&dY3?5~o)<;#K{qU{L z0=}tXYdh(DHrM}SHf#{U7A1|23=c=&!Y2?70A(MLR+n%WMa?^$(q;5RRBPMB{Z=Nrtb?UZaW3`R}aku!!J5 z0t}>TudJMM)eC*`;38DqAq0|##C8Mshd>8YRhn~BeFY`d9H7jRsK5^Dfo+p zP>ufRJo(CcQ(1?p_{t|S6Mr2yIz0}=sY9XGA@NcQj<*d?gFoWcMBgtnmm}IgEaH&^n{|6)qBk=ZSI86iLx6 zGAFg#WRBFM?5usCu!=-s#T`}jx&7m^Wjw^1()Y!pGVyQh5fM{|s&}7o&p)P8W{?%k;vfxFCpwz;t zYX0}mCO*yXDNhLr)-f?Xc-Y*QQG!jlEEFD!B(Fr9IVRC`cJ+_^u^d$*qW4C(5m&8p zF!m3As=7l@1Y;Zoy*c9<_sA< z7TxEuJVjv3z780|Z>7YIX7EIdY9}1izFj?NQ2ic-?u^jH%o1v|1)?x72^1Bb1hv(( z4aykv|hf^YXwiU+af<(NjC0d>N)I3_^S$OV%Ik~nj;aL$p>i=yKy_7;g0PqUt# zJ4xVoZ!|U;(b3&%223b1<2Mqc@{Iiv9BG~LJ8n{fb&hHe4^KnL3^dmI{=>NH76cFN z>nLaY>xGM1|65C_yTXXR>y@gd;s|Mq79rll&zoqjnMo#+`^k7c5{gZB)ckclwvgGU zo-*h0q$Awp>BFZMsOf15i+Kx~N!KH1)qHMyk2bbIOGOeTNDWC3=SzB^B_x3QAKu41 zn{WuI4Ou1)%cu{TwvPW4qBWcFbKQF|nf6B!&o*L)A)ayci1Bw$`;nBV+3AYA ztx+p`MY)gnz$sqDu&>Z=`1NIMIQ!=H_K(yH+NusB_9SkT{w^#XjxpNGv0{$dVj=$^ zDtxdf4i{6gS78u991?dFloR1Q=n+)VyyE)wRc1DZNrl`xI^x?dk&OueoIbCAbecCHrxt@)zVD||{2e7bY~ z_KD1y$t=7ye|IULyB(T~KkG*mrLg`WQv&5byeW%JuIS&ChgiB5ZpF_HO#SHMMG|3# z7;aWu9#SP8qUC7^iArTClxWDX&+0spz+Cb~_uiG3QZI7z9;AXh8-rb^M^cAyMsztr zf1dD{o~l6L@vrYu{Q5E$yR;nK`QI{zwmfwenHL(ECejU))Pxob;3{UVypp1aV^2Z4I??hj{?i)P{Z~gF=s*{EBvPgxSXm zUokv0DSlSncvFbPpoW~4V=c>&8ik)fm|tAVr6jT9B5Q#<5SzzOcQ`LT#!XJ2?f*P# zq&g23t{K;oeZeYnRMa0_a=F5&Pp4S=2Nlbz z2XB&}wEh+DzOn77qKmsVHUR<&j;)6930Ts`7>=4lqeVD5AoXpcE5sO~@B}AP+b{E% zaqjr77ypep6>2_G3Hw6Tg(lLb$)YJp+!T9JknXDon`%~1Ses64ezL}7*%|y?aM0x< zFRt)iIYXTLA7y4~{vNHICX`G(CF58*~BnhLO*!GAPM6M7#5>V7=nIfiJN zCPhsVlb@C(>J+bEZTVQVuOd7j=BN!`9vXeCjTX@LqZxy-2Fz6x8dzVYv(S&R1KtI; z_VIIGeu{HrT>X17aQkAOP3r*utrQJDrBFIPA$)pMMOMkP0rLQ73 zFF9tpYnv*Baqon7CB1R6_%Eud1IC;A6caU}`zcdA_MBy=%4I@aD%Kz;RmtvaKIv&^ z8vhrAuvh-)v(W|&rYXrza)~k8-Cq!78G-}K;yTouEL=Q@Jqvms2(q@cS>Da!drJ&c zM7*B#rtj}m4fe~cE{jgIaey$XPlMwm5_9T| z%!1+21hQ;#r0fif>w9|jRZb#sC(Yj0Ef78Mr&8faCuM6Y=YC|;&g()F!O9BZcz}!i zrAU(i>-*}@8E47QVg1t>7W(0;#1bN@Y%J5NxKsDrLN6w&+$^a`)t20HUzVhJ{ZC2P z<>6mu+jZmS4(UFrL3!xo)Am>-rhoRRpTJuOCi#K?McmAWbjtX>TfwfQWob=F3^_(1 zit?exIMQM@&^P(ra9-ISGyWPH&eV_7SJe?xF~pdyK2R+1sDO*)+TXsyD5aw8F{>*` zt~ZPj#nDN`IFLFE{M>mS-tc z;(@`$>1EXGf~^G^-ho8Z!;7n}!}ntJ%5wPZK>XDoVfbYE{OTw^00M{N6eV>VF(3%^ z!-7{n=th^ElZ9tIRdal#;Y1MUz;=dAl9Xl;~_& zZTO!=lUsbiAfBu&(Xw7uct58p6`^SPI~cbvJ6)98?9VJsyqilnDO|$_{-W%SJDj?! z&NI@6;HXnYPrz6??jB|xvd61=#Tdl|s{C?yW*|^v$}%GUBJgvqCy`sOl2w?H)o}DB zo$B3F3kQj(tk%r;wH64YYfPP@wDLmqg-@&Nk1CE!RV9rtQg6KxH8IKE4Ae?myOJB7 zliRjH6%U-Q;Y;I&KoPNbmb2*8ZyhGLHau;{v8PfA^_@zD=xflV>XPW0d%N_zc};Di zPNNpG@4?*FP8h%JP10xxZ`7ur$=UpPV64?6%&7T`yv=MTXH{J}B0ET|?;_-}K9eG@ zjV-0DzsB#)iEH*9%EelS!glOX3B%c115EvBw)>4N@9y=7RCrl5cs!-TZ#iv_tZPQSJW0C#x=paXhvNyF4FN9#y3R8C|7fpSzaLX zg50him8!}hB--b*QISLZ*^>2vV3zCX-EQ4~-cmQAUrl*R7J%KD$o)3I2G6`s#TpMj+QLk_JRjbnN};zYs<&YydzWLhUo~ zu(iYE&wD?IM0Dv2vByanhE6biYiL~&Ab4wb#tuLq7d&_2yC-COH81aWGA3Y{Gwqpnm! zJJ<#iQfzTMCDo|M{p#|8=5AK`_G!o5Dzj%kGDy=VjEtyAY-fVjnYaF|v;ESM55-9ZLS$tys59XV?MRaMGM{#J0)quz{q zUlcoZ7smLaB=65P<$k96tz@|TtY^e3Lr7eN664|A8>vU(!~0@1QX0s%G0A6A1=%{k zzaB~M#14%g9+cVX21k2^i;Nsj1|)`GxwO2`W7)S-Lj({yWUnH5bU4%HaZHOfR13&e zx-0{RQ$aP%l1k+=J)NFHg&cPbPZ)y-nxAbE${SQJ!PEE^w<8(UkA(S4?A|?phwX50 zDe%l`nhbX%=nN^Xb~a9aA*vXz#3PaJMr_Djdgf1WtNELnA7f)R#UB0^vl{7@jFhDe z)gCoeSW;K28-?g8tna#58L<`C-5Ibe^>RO^q9@R$EVd}|+KEdqHGQ=5QZ+D90geb~ z!&Q9#**&vr`!>YW%W=53B{(?vcEBf{B1N1p+|5DZBB8>hlXSLT2~#~JZb^D?ie2w) z0?*r%Wr3629zvNVfqOrjjc1`~D^KxUM#V-EY zG1_o&iA`d?DUY)#6rDy!NB`Y)F8h(ci&N%0n`uc5_K&%ItZIlmD z*1uAbYCpcSHRhEY{Nz+SxzKOH#~+B&Q}Bi)PAIPA57y(u6-(_is1Y%>nUSz0C@w^w z>a9N2XCt#fly?KK^&i_2f zOwDr(?_>06vo)O{YvI&*gV&0zu*TylbYy-wzQPjrFOg)2j!8G}jYVer8L=Z^Dw1dFni4wMU&e^vMvYb_NAJ2q0`J)1wQLZTq z5p0MJlzD=y!;AlOiTnYMC}E)_h2jswddAV_FV`{PRxDW^QfHUby`u?>SOxQ)1G>?H zloaXa?MJ!2XssXwLQ!IQ6@rM*{dF+7Sgwp{W-{ze-Q?HcyQ3KEd8gNhx(k0fnVpA) z#s;aqn`_dLp&D&cWDwZdzedlMbtlE1pkTbzq?jxny7Z&|#qS$R^Cz=1(q~Vf^3LlJ zTEhe2K?oSUU&MR{zsN19g~Z{mkSB_GfF0Ma5F&9cZlH()|8|hlFp{ao>MG(|a}1Y9 zr4|1&(BygWt=;QRHy6W2J4=&Kuqsi9(#pvB%qKk}fi}{7vjCE8N~BL6VP_#vlS<7@-bYuG#E@bq$zfAB z^0DO&0UQGB`4ZM^;qDM{CMQW4ge(8#KlMFWujvP*3)~^$-lfnOwa?NUEju4jL*fqN z5S3#MQPWTIHigz`O+@)h69utGBzf5iD|;D6C({f$dpwp(ZSbzoy2!qJCe{b07|mz2 z(ea-}z?xDCGxdLmIO`SbHbXlt%ntam4xpw9cc6pi!%|ieq)K3q0)9owg8RA>Yw3fECaO9rr#PsrFO5mry%J) z54_+1Ehv|8c{9G&d#$^Q#83x6w7ydtr-QG_LllIF&gZ6bG@&(3Vb#47AoYNd9v_%m z{AXLQ7GSHTHI2~7cucW^oxuc0kFim05ojOTa7Kr+h4K#S#(m+mpCc+p)Ef9akQt-A zw6opJ(pByz=F!1VULJt97?c6J3{VSxsL>9<6E3y(%yat`O$gye+_14?+V^8#%~dx! zEq#0a`LNYZF|3C*pSyc*ZZ3rReb5K*XBvbqo4trZg8=nvcG7aHIQ$YG{DHmR`~Hsy zDEJCOKoI(_VL9^2kF6FsXczFs+=~H!xB9H zYZxnQn-*+>Z!pf;Z?yjzJ;#{gEN)R!FPi_aZQ(*t7{1C%tBwFBvB&-J0JcV(@Ie^z zzdR`<2B5m*LbgD1wcN5BaDMgXdjqYdG{{tTnwy*3wuXm`0JS?9fdF%oNXFvV5c23e zohH*l6dFg04Y8RxX-kfiTrt$UE{=$T!7L2RH`NhwybjovYzTudvMTKWZ z4zrv4dHu7ivD+AyPj2$`(vJOc%BAA(p0b(kKhk79%WwsFCz+Cio`Y#9dd!C3gXE90 zeV72-hwjITBB9JpB;rH` zE|FZ2OpX1Hyjlb$%eBGrIn+*yF)_7uQ~+FYRRqdA4u^472smUvX-wxm4&R5f7~h(T ziJ0)OWHo3_9L+;m(@hAI^4LVH>MMg-cJSfT1ZTAbPwEg4>uzqz5l`wa@oke5>?Ui6 z!#~QcjHRn+5ike}ri1{^iJke;HMvT>0*_sa{Q_~%2P0C5UJwK{t773UHcoG3Hy}($ z=Xrx{Knw%$#>d_=^D!K8iV{x;vZq-aFJ$+=#HqdgDOyUR_E%oLioVOZzE^&fOF;Dr z>Y>VL;#+wsA2pOPDPH?tm^EFW!=2hAznX%YpDn@)^0m=zEz&M$q6rz??BDHq;;iPn z=kvQDi-t*cxHAW2k z)jG}{mObro17{iv_0x&PP-}QUnm0u2Y`ku%Njbn(r(5`l5_=kdc(ir^U7`JsYkNCe z|C!**GxEguXHamBbqjM4S!>2yv31t+>22>I9S7J*TO?BZFQ-L~o*>e;!Qk}@5+JxW zi11=@yl8SFX8!SaOoH+E$oVGJ_Qm`y|I%T(jtVK-k*&8 zIUcOA=HYP0h8H2|pVIz@A&vfOrge%dxfIX=-uOxm>2|1ow0wVpd5#b=Cp3=%j&RFN_j!?;N+#< zr9WWIgUvk#f#Kq6mf4|vvJStB@9?TXK+Ai^eNRwOZ~_sz5}JI3@yq>krIldr=J3=_Xqowv@`D zT;d`E-zR{IE%>T^9f8ce{Q>JqHQ|5Kw95HQ91TPPO#1GuRGpHOa<<3VGVu66;IYXWRE$<0vV zi@OuabcT{c!gone3+{#TbBs~W8{=7~C|5k%R;>IQaV3fdbT@ofD45jB+3dO#16L!j z*AjbgQn3e!90Dg)kEu1u1Ataq1?F7%%$_^`zhoWRr&mlvucQnhuNL^x)V(BrH;WbwXJX*CtIFb#GqfY$N zcR=+_e^EtWd|zmnDM9}P5pI51d`Gq%#G;pyf^&iC2*<6uM{ z5D+360bbn(AS8h+#$oG}1#1-<-q8~a{n8J&4-R1YhDHl$WqV1V^j4yi?4=(QubhAi z5S0=`5pSR0Za4v3wgBRY|cv@bq-o-!} zY{Dt^2MOaPsQM`%pD5qm#tGNJmn5*biwo(LjMbieudk^+zAH^FpNuSqF!zf9U3VRr zU~Ay+C{g7fvCrs7Z0IlfDZ{6 z^W6{wS;&I9x#=JNo)722IJEo2l@Cd>TmeF$TR3`uVNLlK_txVSw5~Cpo7X{M02`9$ zzFveW^QK|$XY??b5g+2(l79b}^K{@}3A1pDLwJ_XC74+eAq`F|*3g>dNSZieU!V_- zze-(vkR@5E7I2{>t;==}3lQ&+|0_(&-e+3+*AR+N^hiEmAYAJ6H=3h( zRXv=n*L}!71y2qrnM+C*i)0MPh(?~#K()Z4pH?t@eOW9No z3S&5y&&nzq_@zmAt!yv&R<2fh7d4B5fXWca2*9foAQBy_#N1!AgB%?}V)m%-S>hSS z*Xoqi=~q5nMAo&w3~AT(X=B#H25ou)@djYge(vs8WXLJ~k6RgzJH1j?@L?r=7jwkV z{(#2L_^{NzIIy>tUm9X=`GEpATuF?J=AI2d=r(_JoK2Z6WY-a`Wi-j=87g7R&eA=3 zq@8#oSgkGp+Pv+D<)8{fKblc}PUdaMc?qvVenyhzV$0vB_0?LI$6hoxaoc)#ExybA z2LKY_`2l3G^JJM2=K_Jy5dOXmjuoNt5q4o}oPWUlY4$$pYK&^ZxEN`Vy+r1mIH{f{ z+SwU0DhUwD9~~fZd*?z1_!!rcC5_J)t3lRu2L#+T!GL2Z8T)d!Dy`Ulpv9RP#^$0Z zA^*sUURg*<{$NNims;ax%i_!5dl(AbJc~x6iIOMR8XPQ4?1L|=hX(6l<$^Jq>TlgR zFz5M6Pk=Hf%N5umD5ImhoCkV@-OOhk|HB*x*UtLK9=UFjCbc(zdp=h=HuAmg4GTCh z-7UakfImQ*Zd5US)R4eGnvx_jewrBm*v|c9SgQxZ!Z9d6YCuwIVX#kfa}J$VZlCe- z_rr~ke{A?S*jBBXaVrk<6MKsloFsMWi4@MXA%8$L(r2pdx@#JD?!&;FTcvHMaNp9! z%PY_-2m&%rKvn^t20oKCOi8#8+#*KM6spnNDn9A4j~Q+5qF*G(Y?1ci9YmR?{V71M+u-- zPQ~?T#@EiKuFJU))vfkK-t;<|2x*YNg|DXBc|9(4f1NJ_v7;YAtN1Vd%mFh*&=cA| z_Zjz^jt(J?7rQ<=D!15QYaPZ|T~1rgXIF(8_!o~jA8$>!+hq7v|1SmslxC+4UX6b$4APg z>u(_>mzNw-s0P;3vGw+WV;;v*j?R{klOIWtD;p0ws0YK;^m~D0K*C9SazqAYx=`=Dn_G)-^ z>=A%+fUWkw@Iyci0?8wQ5DQXUy9NG-N-V6yA0RQb2=R&s5>I=iwE}9TkRe~zWYUEw zp`*q=0}PLILM&C4Wy(Slo}V?FQ4NgH-CvNMvL4a9098t;NSj8>nh_I=!6?*r#BNVwawk8?yP}z zE-x?p0m9Mh_qu02!kCe#b%O%41gua%F+ha;4mXy z)wU0*6(CK^AQ1!!*gFRm7Hj4clfxHj5r5{M-(`1~`T6yoCzrg+V{1st1e2n4`PlY6E6O_Z>YB(Jg{UZ%S^rlG+ut>F^T$JkmW_8uo6^Hb3pP z{eA5b;PT%>!Wl+cTL;gJLY{_V6K&Ewk6Q83r?*q!jZy*gIP^X5k3y}TBjRjtB{f)= zyc5-_&@5diUd{ccYgjRT1FHvz6hJeA9Ms%ZVN3chDkW+^t&@^>G_iKwGk{1a zKoOp1O$RuTf`68z1KGZ?{uK8cti>s{8(QDmkoVP9dXD{Ar=GxW>r3lJ7y2e;UswK# ziS=OE_8A_dyZ=VL1|ge2Gv&~hzhzeB?=!rB&!1Jm%L#;~n6b-4KUsztElu?`)34hQ zr;U3bp^TZ3iwU7igGPh0i_gu|(}G1)PWLhyAwVz(p#Pu*|Iox_8L3nLsnkH8*=|}z zkGe~w_n%E_d}8hkSx6@U=vIoF@NRzmg*da_ zeh|Gv6w=d&ek=QtbvH)mKSI0k$Uj+LIfSzfhO1U}-=Pq2Pl58R-S@va_Ot$8=8GMq zT%uuynv3SLK7ZQ)Q|QyoHX5z|6C@abUP&_WbJ#K^DC~iHwnYJfPdbweSGFG?XhbPhX%GaHui(D9mhKW0|5@$&9PkF zh@dSBD24*Zf?u9I$r!&jM|YcsBa!Ah3HDP~!;Am|90*mSnv~}ai^ihZrXu>bUZHm4 zCbwI0Pzs-%$~Qz`-m zEsG#<$c{dxMji_{T2S$PUSEcX+5JRJHTVESDTCuGc z&iD%P^WUgkL34-hb{M4Sk8iGUAL$I*iJ{Xfwfv*jCmplS8&|0ln$68?CqS_d+)M{1 zfuBaO4x|RkVC2<%l$ks9W10^=^1>cL6-A`_*9TGgO;ujkv zIg8J=JGL;r@>>kE&p-meK610k-@mn^|4=#_sG`M3fYzY6^M%!Zuw`iPqkGM$PJ}|ESl;niN#yB3F@cbD0Od zDN_Pf#n|#a+ySk(wupr z0j^hVMMXs&(msD*iZhb_2Kj?i1}BhA15XzaDuDeyS%=%p{A@lbvFGk#Aw;s87oTcp zY|LA>k_s_2c&?`jS{5H1KZTO={Y6J~eiV-3&{MV3i}`?B{_M5B_A%n-+?U;y)`11Q zw9@Z4YnRg-*4NAn{6*JUuyPf|^n$j{fO&cYwGCrK@4ul|C# zTIch>)W4hJ#LeEv4n}20?KQ7==XSi?oc8v1;&mGTYi}=|>FMjc!LI|>Cno3-7`IqOMayr!jYSFUa?o0VVO6zJO1y% z`!P+q5ualeV?RF(H#U0jIy1c0sDDnFG)H;8=(?zRt);$byY5WqO%p+kzC}QlHQQj<_CWhkw;sxPd|8B5O6N!(An0^klHBCFcR@5D_4DCK1zQK=fNevLR0-S%0Fh)P&S3rGg>U2L zW8tN%=eC&*GH!gya@WUJ*T*({XV@UadB#0(MwkP~981TieMo8PLg1?f8XE=UFu)7x zOBZ`K(F7SgoEXLC+vEs!gtHHZp>g|TA^a57?)&GwZ0b9!DRv`tq1MbL0{yqjMk@GN zwSIhbXlJXjOrCRNv5-3muX}D)du>GrvGc)_OMr%1vM~JJ?Qj?O6q9-;lxuZyB0Xjb z6LBO}oeX(*67?+&LmqOCO`z=noy7=PGYsIuJtxh(;z1zyX zF@9P^Ha0X|EqvOW$llrcH&~K$g9QNXZNSJ1y&(>Me+kljw`%F+dwBX%&Bg|#`ABP1 z>iG3%D-eum-q!-Qw`a0K)wiMKc}Zppa8jx4(aOSXvuGOG7pEW1t$!QV{fX`ohzc*> z+8pz~e{0~hP*QVjGj#&6Tn7Jy#|XKg6o=GDfyuA;kmP2_^Es~sj<_mTlto3aRzE6)HjqS29dOpBneL>Fxh~dER(G84u&^HW7;L-5Mk0r&f2r5sI?n)&v+IoUp zb`Wib+Ou6@4>tYT;*oJcWW!GyByT=aIeRaQ0+Jk1Lr7~e^s9suVi%qQ@#-2<&-y&F z{wDlU;*=YIv%2wjE#(9sl8Fb>%KnLNW%9|4RD8r)!rh!_{zI4Xzk78Gi7cs_OSe79 z*YK|fph_mi^1kGU*MO><2MnEVJPwhjg0L@p$J{Vhr=kOlVdYiW#H~yUolD>My8dUA zjVTXJu3zq~eAcH@+334G&k_p-Ss);!08AUOieLihQTxYnThs%yheBjcqc(BwvH`p4 zqcm~Jf-lhFs~-;gluMf++^oHb5IVZpHc!>Te@TycC+_A(xV1Le!Y&Tf^;a6g!k@dU`FcEHC)qHoLr;?=P{g@%6g>z!hyi zsW0JY>Aw!730h9No%G$V0t78A{aQAaB`I+{5yf14pPzNR2sJf)nXyyW9kNkxFnN$A zM`N!wX-iBVH}>L7^X*t+$}+O%7wQHQ7-5)xtXVPaAGxKlCJ*=u014-`&`P8j-n19% zGTOY|QhmMg>S|*~jBA@}=^gxk3Nbz4V(@qUHBrPYq3!eHS(sSgO~mR@?|gmbcN0=P z8_mM8A?;ZfmLMP=3&^W|VqF$4B&Yiy`{n2l?NZ8fx5VnJu|$MXjcnQZzxEcwO@3@;`h?;F$$#kpx9Q8)jko10S>xfeA@ib zHVk-Z4=~#fEcKB7SJIHeOx9kR=b{5oGRld=HywVo`alod#F<#XBq^12b#MZ2H@%8vq-iR5t z!z@LLSqjb^zoth9S2Wl-+5NOv+$CSB6DF*vHH*IYqW;Hq&TOY)P?#z>i)i00)3&<; zPlZ=mR*y)!g+K1jeIj(m0E>v79Y+uY5dt1R6}X|fF7glbS5%T3UXctA!g%U_9&BONJ@W6oGNxYQ&{KvPWd=YLpdtVGqYe)KYi1TnAv!1_Pfh63OY&2W z*8ytY8|QTlLL3|M(QnKAU{4%&4PagUph zNap&a{#!^1s@z;Yt41wWB+7*$NX&_93Sb`_`p8pK>6ICW@txX?7RhN^5uF?uJCr?0 z1F|djJ_S@OEzqwfMc|5&2B#Mff}I%wHiY=FxV8kTXOR2@s2y*OgU&Y=H z7FI5qW{Y-BG-$C#eZ6*h-zCQ>?S>11-1r!D_yDu0S2_QAjFPRUEqrn%l-9@37`Zu_ zB7nb{8_UNy=+vXPwsAHDm|!?6s~pxPuFRPV`w0s)OHs`qkHorosir~g^6(~iG)>MNeCE5i2q_x?ca5}(btf5x!8j34U2vw3cS_cI_dx7|BduNN;eE8 zfZTx4fn&&m*X9jw#d)-*<5N_)NECd<_}SB|wW3tZ7GnoJT_R>8@b+M9uRho5%3b!T z7FfGwXxFyY!YWzHqA{my;g`_3G=K^SSm>yJ;s4b) zrTMn@C|@hP;d|x%|CCbYyK=cca6Ocaldt7>f{|37FAD@o-U&TjU z;WE?XfB>kx(R+6;H5ZUr{jlzppFH+Ju=4~23(+-394)}J226EC1o#HUIVr6F%EFBt zeKAQNbudwJIur!gHXhe+eYl+zx2}E%%ci}cc9*T6r@&THP5Sl_n1UoDW4r$7K3I-R z&(hfU7AtJ?Qf%|yi`$>i^O=f8$7vAKsFkE#FsmnheQ^i^k-heFLC_rpnS0(0JLtm9 z)-G^`_{O`w@or2{fv(QH*_mNR*?}eb=JZ<1c52%%-`(^1?a&@iv{$-O;_TaCL}$^1 zRC!oeS65(@@%>cIiaX|zHg!%RB#JCWq33Pv#aA5GvITQ9T5`J~$K{FUla{l)zrsq# zSW03woj;2)9dKO^`f+Zil3RQugyKwTu%K4cO0EWvsgvKe)3$WLFNs(^_dnC3f%VS+ zckp=}ZT;$5*m?UaJ;m0c;o&{stHa!D>+kih8O~dofssorAZD@Jsh<@BzQ4MAyE`@K z9=jj7a(yX4Yz)#_z@fQuSpw8FutxV~smRs2?vjw>KjC97`}OLyS!?=jc*!o5D$gl& za2AHh5@T=w!9 zn-#l2+k;a1#vXkK6T$8wMl0h(s$Rag{|@$~w>wH740m9DuCcaj8YwH=!J#1Y|7G2D zV~Ww{bMoive6Oe0@%P?t{`{}FG!dU-120!3<5gY$8yPK6+j|gt+2b-@$Li|p>#xA> z#r^U3hPLMB2!MtIjXnru!FyW(JT|SRCk_Z0|40kX3n4JN2?{x+)U0ma@1)WX)<_gx z4SFZ05}gdG5feCPadU|To6@`p2FMW?^|O%`P)DFd% zUCrW&A1JE>a5SxzFZs^vJFhw}@7*wrXH`08@N9x{ZV&> zh=LUBRlS+O4;E6{1&4bhpr}c+u_E-r>`dQnoc&l1?`n}+xC z{_ORTsN~lN6X1&|f&%`yS?+70L^<7c_#b2`1oxm%K`l##IlEnuBj~*^%Ka(&P5h|% z$VYQ`LAz|(ijkG{R;&TzF6;X%@Gj>5>*WGAW$h&-@-n3)r1 z2fc2Vzsdjl4Fku~(WEXOB(%qxLGV30uITZo8C7|E<)eWo@h~|tVGTIc_H#IG7y9$O zwBny8`GEZM7LzOTcdScpTyLC#xj^gbMkX*=1+9TF zLP9V%D;{Z6U{HC5Z}S@)YwN7pNmz^vGqti}VP<4(b<{h%F^QOVYP9*way3R%jMA{E+X4_QLSR?3ou>{-h)md4oU zckb`c%gf7Ox63`(b*{5~&UxD>YHp?i7;iv|_w`Jw;Rnh4x^WyjCVIyEyM5Wm3*Cce zzsojwb{jnGHr#8!{4=|)N=4dD#J&L6LJtxlP9d;BATv5uZ>{vh>rg|BBXDM471A!+ zqcNf7BT`c3~&v<17@iTwY|OlQS(Fp95oUPsSemW$hd5XF;*<8f+l_xIb>2V4MValwgwv#qyg?f5=`lX`N=kiYa{`7!lhMlJFSGo=Z zJVqOZY#V^68ZP#*aOb$UlizZ@p{V&X`eFqI6Z9(-XZ|HZ$|?+_gMRTsl~muLRPCar+G%OW;eLp4SO}xxJ1QHKcdz;m$01l|$n6Gk(2`z}aUiFnG z_VBd6PqW{kbj{{ctEXW6NY?FUDhgE>P_vv7x&ff+TWfbq73SA!d^Xw+{Fw8<^v_6? z`X)j@SKD5UAA`~3uUB)Ph$q~B`zLbpHLpdCb~$$9>@mlV-^p%4K?z5BbLYBCEpAt< zn4NmqQ%34knF$rHqMQm%6gPHGHJ(z=>iv_MA)n(3w$?6k1YUqgG{y)m%Be5Q zAxwC_&{e2X$}EwfdREx!^>0qneYkba70b$MwtrZ8>I7m&i;&0;;;2YqVpPFRou->R zNG|JhU+VP2w|}2;1jhyQhKQq~{bHg$UNaUj*m+MMS_s9} zQOK;M*S#!&u*7z)s`2viu6hxDiBr@WqOC(ZYsho!ho|3-9pfC-X6Q(CknjRjQ#n*o zQqBy{`+U6|6VERpb4Do&W*q@$2QJeQ(2zht0k6~^?{=&bW3jvZPHyg++4Z-RX3JCF zXKjW^5@JI$cQh`GPD}+oeiV(h?WuViqO-RAha`bO%`l9P^S0 z=5Q$ME2Jd=BXp~*&uy{OWlxW@^Cy`14BOH_AONHci{lTB!m;5%rygdq5%%gdvPjw8 z-dN)llo@CH?&ZBmXxa9tTg0ZOYnNQxuc4rZ9Q5MyQtIMZZh}rnH%o=X6L%?r7Cdp* zl}Zzb%2%`6%4zGG5Md;(c7|Tfp@n82ZU)lce%`3-gy!%5>7h>eCUusKnEJv)0hFCS zCk$C1PS4D=1#G$bqdY-fn!Ram53)vT7H-8qz#h;2BxLEI-ZE89PE@vvvP{M5(%&FP z+9SiGxIO=H)cH2m7e|Was_FzeWlPc+-K^%l%$WD&D~_3Zf!~+R7O%`Xq>NsS&BU(V z&KX`Nf#`AMCHc=wB#jBK$h`p1p-3nE0)ND?oy@BSZ})#w7zzMJqtn7S>!K(q)N7Rm z|Ex|I-!mD`mS2y|t=vp>edJiwse#q$HJCROaNg~!lJR$e%WV9T&~x6t#(+NOr7BWA zJ0Y3-vzTq)l*LeZb+yx<4u<{DO{ecM#oWO7ja_vUDV>+97i#v7UWI(HCKe&@A`VAk zVIc+BiaXv1&_(oV8U24#xD*Mi*MHYl;o;%HKHY!>3?sN)mW_ zkkjHD<&Q#(7Gdv;(LXw@m~YeA69rd4nLJ*(7cd!q-pr+$gjR*L`ca0JD_G~?PxIBCd0F{;JxF=8Vm{aJA zm)k^!B)b!uYd432>07B*R#s-sW#^`(t;GN^2194S8Gtbdm@6TRI;gP{E})5`P%sV@ zzRNsl#c$;)=eUc+#tn3@#yTD_W?BbSCQz!E925ASeR;H>@$~a^+veCfs)x<(}80uc&zl+j9ak<~KI?V7Ts2stlbdZQ6 zn)u3*YOA!gHNBF)%?HxAeqVS}MvGStnMR{f*0tn}{JgxasQ}O6?PSP^?>mk0oS3zE zo0lgZ2S6%SkS7AajSI2~2+I~MZIom*GyuSKe)oQ=N5AC>xda3B_#!{uS84W@Yd2Hk z-sZ`(BBzaKK3zyP3%LA5V!T!F#C9g<;sjmd^*OS1l}*X#OPaG|LYK)nlljV1Psr=8 zNZkP_KM47H4=1XrXbVaTHT0?m8D1XRYDN#Bm-heMrFLlH*&!7B_C#vUUn3<*;co;p z>C)Qv(3@KR!eJY&TuRJv)8S!jZ!~vESsM4lZ(f$DNiB7x#Sb( zP@`XHJ44Sl@8hVOUr}7AiQlU)y!vP|WsYbY0uD({&fG#2TJ1Yndw?7jY-K%G^4$JB zcqZV-tzo{Y$*D1aNwv3(hN95ORL-RFIN%>Q6=b{3cCpJmC4OXl*o z(2L`k#}DF2$4gNUpUg~U+`br@UAg3F?vXfZm_AR!hk!!!QnOFEwKixz1IutqMm?Ll3Ket@v<}LT;}GDbhyW3BLi)ineY8V0!^}BG z(2#%{<0^A7Kr$rgKe{$Q5dom?8}h3YqnM%ijQlTpi)Dw>~^z=#Ij~9d_uPB zxuD?>vO_Q%8xl-0u}@C(?Ku;8ajFgpu$>ZbcTgwak3iHaAvub=vm$F{7n@+cBX#m>Jj#vqkuEr>?KTk?^|#fBMVyt@mft`BF4YBue~pXoesO|Dv^Ld zzCf&p7v&MvaSsYh~ro6F`35Dv9%m;b_9UiHhI8g=G1OUbphA4(bO?(Wy~ z`F2F~7R{{=tSxVX_X+C%|F7mja!Vcv#)VZ?F-V%I+bzsTZS+`viR*SdP z-$0o0b8Q)6bMMgsyZ@TbgjWi`Ac2;@zrRVFm?^tBa<|L=n!3w!vt$G%W@ywXoafOs z>3^P*_>z5(xSEV`WLESd`{(r~zh5=;6^2|Faki7|-h?d6JcLeL2R`yBkrzoc!sqgW z|0AT(po~l!E+;yCm-%`UGLsWRKu0LJwqv>(Lw>uHLMb3Kax({p1~xtEgSDYK{c|S$ z#q|?%&cC{o_g<|$#l59gFVnEJB#VSyAj?G#6BrbTX6<63T8Q6uEax_Vm>&AOvn=|P zrdU$okX*_!ps<5Oa0;XsEZ8BH+=uH4s z70mu}sh6c{u^rbkJh8XM>B)bAMZ1pj^nE?+IOtQVul%3wdAtfo=AR!hX%ton#e%DU zxXG+7W)VDz7&rQ_;LpMNRYkYq@~cg+G=JAP^|Msa;jFm`*fqv9BMcjMAqQ?zQCQlh zl2w1d`OU%nYZv#meQhsEE^#)_?KW8~x}0kdMN04iz-0!d57<6sEi~up1f`tuuv5hs zx4DbEQ5F&jhq)BuEuTjt(~oP{l!m0cF#NvNnNIc_3TWO}*N!al(#;`~G z(xcH~)>I68(D%CMt`f{~`Cn(+@4QSA-_K{(PeI?fkF)jv-B$+E;IPut(!0uiy}flV z@_MbpyiwKtzY|8iT;$pe^jd0MEkYdxmOQWAWOUO-g&$_htlm9>sv`!vfLc2h<^nh~ z0|GMVS8gh@g+u$6ookSAX_~N>f!qWU*+1{gs5!}dVP8Vl5mY^3EpyO^cJTL=-5QV6 zDmtda71r@4UwY^hYhtSMZzZWxUgaXL=KMfr(;M6I!&v5-md;>_gmavWh=c?*O`x-9 zU)ek!787s<9pLjKQ`I-FvX816RS>Ubepr%)v3;ZL$M12MkvnBkD{9&b4I;Z z(_pClryrxEjoV%>495b|bPcycq7-F7hiXSN!@CtvnT=BbnYtiQ2N%O;$=c-GM``H(>B-#Jh!wUWZ zF?dA^%1ZSzur0jVT*|vN%ljv?I$L(tcZerZq2bDpFxBr(2;FKv7RQnwImj;d5lrUq zzQ)EWQWN<%cs-)>e0)8m_Jb)-Yp_D#tDh@1VZ(4}O5nbH{c5kUcnEwVASS>og+$X! z^RxS@iQEErJ$wAvNW{J8bxFdSB`XWL&ujEBOR+Js;}vYJ@285NuM}sa%_eX?l{+1U z%GH{^F+F|=dAr29{Wg`!93elTz@l9Q2NxVCCd+n z=p5JH*MQUk910|vCMPGM4H+U)prwEe9q2U)H{`_kw&19L7+1qD{Q1sx;D(XU5zWhA zC={u)8KkT-3+ar-_`vLK$ajKOhIkyR?N}WgH2xru>vfmFgqyN2 zojW+RAJgI@FH4hoCn?UI^H)(JHgt;EmRSp#rvP=WP50G1sBz?;`r6pom>EM`$ef*@ z`v9YU+c`DuO!Q!ILYf_fP^C8yK#N$L@^d2hVa}4$eJSRb!O?}GPv5vgVCf)eg(lG< zTd@mVS~?d!E54ct6&p(2uYIZfkh7zCI5m;e*D3dM$g?gm2)+7Jty-{Zh;TtD2(TdN zF|S2JWP3dg9pzZ0?O=$`XlY@32sSJ3(4xus+w^Q{5}H|4Lba=nN@{>xhczcrq(j$Q zufObCb8YHXCy|J&Mk;eFYtN?olDo7Jk!dJIQzC5jPJ!^f^Bfx7fR{{Y^wGip<-pl8Fhyep_!9dG*Y90y{ z?pHvk8i#W2KK~P`2WJ-Q1COpBpJDt`&e@Ycg9H};)0p!@Bcm$rJ#_u_xdWO7T9;ln zXeFH{VBXV%29|GiK`0A%=htV@wTGdtGQ9~b284^=8&<>XKOBPCrp7R$IEusn_Ra0v zw=dY^x2a_wELTk=Q@CRfRy)vDLeOyh4oQWp1>6 z`C{kquc#B);|H1!7kRDneuE7nf%j4=GdpJwb;n0*ZFF(LWd@TSDQz(>!TNI zFs@}-JhFN_F)maVwoW?sO3*~|VSzyM$cFmd#&)~^&$*YnS;h)+V=XeVPfhZ4%be_A zn27tA?mB^+X?!2^KJ4a)C;SocLCHTiv^F>#M}l%gR;Q$2@~4|2eW(m6$^FTap9SvU zyAuq&J1ES6Cg9cOT-QP)aPSy6VXGlOnH$GW?+iytMPs#bppC8<8?nbrpxNc(BpwS~ zee^>4oefj9U*2}d0;GIum;kPN7Bu_M2HFH!F+arLgNg<|jm2-Lo=dJ*T(8=ai}*Ma zf~N{PNJ%6$l{q!O@Mzu{a0r^upM4~i!h}d=lT%K6WtsQf zGVewI{S(Qbwa-g0*yX-`8^8PdiJ8edj$@j=!j7rgo3=^22_Hy4GI;&jmfig0amqVl z9AsQMjPO*c3Q9zix4TS}zM#$#1p~vKJ5P2El-Mv8r#3y|*x25-f6sbA$Ym9Rsb0X2 z0=!+3Vp(eG(Ob54mt4M4Zzo?l8#JVcr^Lt>>R>u??lq43RmZ=U;c8LH87{zq3&k!M@beaxUn!k#?F?E<0hj}~cO z;CFxlG+*}LJCCO(d0+Q#a@Rv+U=)&G6xGSYBE_%2Y)%O+9;$@LSPpLtF)KyAH=_99 zaGm1>@TwBqd&bRM;!D-ex=)x=FaNrF%3Cgz==xYrIr}}vZ9(X^vwE(M`?yO8JE-;- zm2X~rzl%*jQZR;f-Za{j>5Pdq!GD`PbT#$r1cXZLKmXCzG{sYj`!A+ozGvJiX1n-) zReSdA!+{3U5~;zsA&WR^jHP&AhcuuUgt4^jNUtuL^|G;9wm{Ly0a=6P7fu#*KVZIO zb>gp7`Oh894vk#Ev8|#;;AQTi@YI!hY+$A~B`D79?ZF@U!hNn_zeLs0b&VYxya{0m zcjTHRb50BUv?bB3Qj#lwOSOi3;8$7w6_1CGTouFA(R&@7A5gP3-isZ-Y`vOh#xmRp zn10P~nH(>usdNAMpiSr(D1#<*92d%d$M6~IvLvgD1gBDuIB?q)@nBjpo3ww2$$kM! zroJxU8u4MzY_aPivPrAQ4ojjw^|2R<6}!hOe7Tx8r`vXtXSt$$58Yl@f6N-l8+fq)meQ}3xXIx8L z`zdF;vm>^7tjRL?hu@%C>*n(2U*B6dZruM|G&nEGR+vRS8L?v|!)FXA8}J8}-mL3s z5BVu6N$WU1F=JkhJ&0R;bRS32PS%cmy6ct6505LTUc>3ytt^ebyRcRl%G>c8-FPPl zyF0SNT_rvT*!FY%Ff+R)kQhJoQqB^v#kel^j-Dwl{G% zD#OPxE$DZ;6@OHP`={4B`D@*j~ycUkD;(Q3$cGEhfaqM`40A^ms~cRcw3tHCjZET zPyc Date: Fri, 26 Apr 2013 14:04:12 +0100 Subject: [PATCH 54/56] Added DELETE /user/{id/username} to admin iface --- lib/admin_interface.py | 39 ++++++++++++++++++++++++--------------- mining/DB_Mysql.py | 25 +++++++++++++++++++++---- mining/DB_Postgresql.py | 12 ++++++++++-- mining/DB_Sqlite.py | 6 ++---- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/lib/admin_interface.py b/lib/admin_interface.py index 1ffdb28..69f7d87 100644 --- a/lib/admin_interface.py +++ b/lib/admin_interface.py @@ -13,6 +13,9 @@ from stratum import settings +import stratum.logger +log = stratum.logger.get_logger('Admin Interface') + import mining.DBInterface import sha dbi = mining.DBInterface.DBInterface() @@ -28,13 +31,19 @@ def default(self, obj): class RestResource(Resource): + path_or_id = ''; + def __init__(self): Resource.__init__(self) self.putChild("", self) self.putChild('favicon.ico', static.File('statics/bitcoin.ico', defaultType='image/vnd.microsoft.icon') ) - + + def get_path_id(self, request): + return request.path.replace('/' + "/".join(request.prepath), '').strip('/') def render(self, request): + request.setHeader('Content-Type', 'application/json; charset=utf8') + user = request.getUser() passwd = request.getPassword() @@ -51,13 +60,16 @@ def render(self, request): if isinstance(self.children[c], RestResource) and c != '': links.append('; rel="%s"' % (c, c)) - request.setHeader('Link', ", ".join(links)) + if len(links) > 0: + request.setHeader('Link', ", ".join(links)) - return Resource.render(self, request) + self.path_or_id = self.get_path_id(request) - - def get_path(self, request): - return request.path.replace('/users', '').strip('/') + if request.method == 'PUT' or request.method == 'DELETE' and self.path_or_id == '': + request.setResponseCode(405) + return '"Cannot call PUT or DELETE without an identifier"' + + return Resource.render(self, request) def output_item(self, request, item): @@ -102,8 +114,6 @@ def __init__(self): def render_GET(self, request): - request.setHeader('Content-Type', 'application/json; charset=utf8') - return '' @@ -111,18 +121,17 @@ def render_GET(self, request): class UsersResource(RestResource): isLeaf = True - def render_GET(self, request): - request.setHeader('Content-Type', 'application/json; charset=utf8') - - path = request.path.replace('/users', '').strip('/') - - if len(path) == 0: + if self.path_or_id == '': return self.output_list(request, dbi.list_users) else: - user = dbi.get_user(path) + user = dbi.get_user(self.path_or_id) return self.output_item(request, user) + def render_DELETE(self, request): + dbi.delete_user(self.path_or_id) + return '"OK"' + if settings.ADMIN_PORT is not None: diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 402fb43..1f22ae2 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -412,16 +412,33 @@ def get_user(self, id_or_username): return user - def delete_user(self, username): - log.debug("Deleting user %s", username) + def delete_user(self, id_or_username): + if id_or_username.isdigit() and id_or_username == '0': + raise Exception('You cannot delete that user') + + log.debug("Deleting user with id or username of %s", id_or_username) + + self.dbc.execute( + """ + UPDATE `shares` + SET `worker` = 0 + WHERE `worker` = (SELECT `id` FROM `pool_worker` WHERE `id` = %(id)s OR `username` = %(uname)s LIMIT 1) + """, + { + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username + } + ) self.dbc.execute( """ DELETE FROM `pool_worker` - WHERE `username` = %(uname)s + WHERE `id` = %(id)s + OR `username` = %(uname)s """, { - "uname": username + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username } ) diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 263cb8b..34d927d 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -172,9 +172,17 @@ def list_users(self): for result in results: yield result - def delete_user(self,username): + def delete_user(self, id_or_username): log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = %s", [username]) + self.dbc.execute( + """ + delete from pool_worker where id = %(id)s or username = %(uname)s + """, + { + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username + } + ) self.dbh.commit() def insert_user(self,username,password): diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 6b89993..9fee199 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -139,10 +139,8 @@ def get_user(self, id_or_username): def list_users(self): raise NotImplementedError('Not implemented for SQLite') - def delete_user(self,username): - log.debug("Deleting Username") - self.dbc.execute("delete from pool_worker where username = :user", {'user':username}) - self.dbh.commit() + def delete_user(self,id_or_username): + raise NotImplementedError('Not implemented for SQLite') def insert_user(self,username,password): log.debug("Adding Username/Password") From 0a8956178bd501fe84a188b164d7168aef5193ec Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Fri, 26 Apr 2013 14:38:52 +0100 Subject: [PATCH 55/56] Added POST /user to add a user/pass --- lib/admin_interface.py | 38 +++++++++++++++++++++++++++++++++++++- mining/DB_Mysql.py | 26 ++++++++++++++++++++------ mining/DB_Postgresql.py | 16 +++++++++++++--- mining/DB_Sqlite.py | 4 +--- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/lib/admin_interface.py b/lib/admin_interface.py index 69f7d87..e137439 100644 --- a/lib/admin_interface.py +++ b/lib/admin_interface.py @@ -65,7 +65,11 @@ def render(self, request): self.path_or_id = self.get_path_id(request) - if request.method == 'PUT' or request.method == 'DELETE' and self.path_or_id == '': + if request.method == 'POST' and self.path_or_id != '': + request.setResponseCode(405) + return '"Cannot call POST on a resource with an identifier"' + + if (request.method == 'PUT' or request.method == 'DELETE') and self.path_or_id == '': request.setResponseCode(405) return '"Cannot call PUT or DELETE without an identifier"' @@ -128,10 +132,42 @@ def render_GET(self, request): user = dbi.get_user(self.path_or_id) return self.output_item(request, user) + def render_DELETE(self, request): dbi.delete_user(self.path_or_id) return '"OK"' + + def render_POST(self, request): + body = request.content.read() + object = json.loads(body) + + if not 'password' in object or not 'username' in object: + request.setResponseCode(400) + return '"You must specify a username and pasword"' + + try: + username = dbi.insert_user(object['username'], object['password']) + request.setHeader('Location', '/users/%s' % username) + + return '"OK"' + except: + request.setResponseCode(409) + return '"Username taken"' + + def render_PUT(self, request): + user = dbi.get_user(self.path_or_id) + + if user is None: + return self.output_item(request, None) + + body = request.content.read() + object = json.loads(body) + + if 'password' in object: + dbi.update_user(self.path_or_id, object['password']) + + return '"OK"' if settings.ADMIN_PORT is not None: diff --git a/mining/DB_Mysql.py b/mining/DB_Mysql.py index 1f22ae2..5207861 100644 --- a/mining/DB_Mysql.py +++ b/mining/DB_Mysql.py @@ -307,7 +307,11 @@ def found_block(self, data): SET `upstream_result` = %(result)s, `solution` = %(solution)s WHERE `time` = FROM_UNIXTIME(%(time)s) - AND `username` = (SELECT `id` FROM `pool_worker` WHERE `username` = %(uname)s) + AND `username` = ( + SELECT `id` + FROM `pool_worker` + WHERE `username` = %(uname)s + ) LIMIT 1 """, { @@ -422,7 +426,13 @@ def delete_user(self, id_or_username): """ UPDATE `shares` SET `worker` = 0 - WHERE `worker` = (SELECT `id` FROM `pool_worker` WHERE `id` = %(id)s OR `username` = %(uname)s LIMIT 1) + WHERE `worker` = ( + SELECT `id` + FROM `pool_worker` + WHERE `id` = %(id)s + OR `username` = %(uname)s + LIMIT 1 + ) """, { "id": id_or_username if id_or_username.isdigit() else -1, @@ -461,18 +471,22 @@ def insert_user(self, username, password): ) self.dbh.commit() + + return str(username) - def update_user(self, username, password): - log.debug("Updating password for user %s", username); + def update_user(self, id_or_username, password): + log.debug("Updating password for user %s", id_or_username); self.dbc.execute( """ UPDATE `pool_worker` SET `password` = %(pass)s - WHERE `username` = %(uname)s + WHERE `id` = %(id)s + OR `username` = %(uname)s """, { - "uname": username, + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username, "pass": self.hash_pass(password) } ) diff --git a/mining/DB_Postgresql.py b/mining/DB_Postgresql.py index 34d927d..6a3fda9 100644 --- a/mining/DB_Postgresql.py +++ b/mining/DB_Postgresql.py @@ -193,14 +193,24 @@ def insert_user(self,username,password): self.dbc.execute("insert into pool_worker (username,password) VALUES (%s,%s)", (username, m.hexdigest() )) self.dbh.commit() + + return str(username) - def update_user(self,username,password): + def update_user(self, id_or_username, password): log.debug("Updating Username/Password") m = hashlib.sha1() m.update(password) m.update(self.salt) - self.dbc.execute("update pool_worker set password = %s where username = %s", - (m.hexdigest(), username )) + self.dbc.execute( + """ + update pool_worker set password = %(pass)s where id = %(id)s or username = %(uname)s + """, + { + "id": id_or_username if id_or_username.isdigit() else -1, + "uname": id_or_username, + "pass": m.hexdigest() + } + ) self.dbh.commit() def update_worker_diff(self,username,diff): diff --git a/mining/DB_Sqlite.py b/mining/DB_Sqlite.py index 9fee199..5218439 100644 --- a/mining/DB_Sqlite.py +++ b/mining/DB_Sqlite.py @@ -148,9 +148,7 @@ def insert_user(self,username,password): self.dbh.commit() def update_user(self,username,password): - log.debug("Updating Username/Password") - self.dbc.execute("update pool_worker set password = :pass where username = :user", {'pass':password,'user':username}) - self.dbh.commit() + raise NotImplementedError('Not implemented for SQLite') def check_password(self,username,password): log.debug("Checking Username/Password") From 5e41a614236086485dc4a7293a6ae38b4942ff91 Mon Sep 17 00:00:00 2001 From: Wade Womersley Date: Fri, 26 Apr 2013 14:46:27 +0100 Subject: [PATCH 56/56] Readme update for JSON API --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 24e962c..6ad6331 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,30 @@ Basic implementation of bitcoin mining pool using Stratum mining protocol. This fork includes database optimisations for MySQL and password hashing using a salt. +JSON API +-------- + +There is also a JSON API, currently just for users (db: pool_workers). Enabled if you set +ADMIN_PORT to a valid port rather than None. Once enabled, you can perform +the following on http://localhost:ADMIN_PORT/, provided you have a password set (see after commands). + +GET /users - list all users +POST /users - create a user (JSON body: {"username": "username", "password": "password"}). Password will be encrypted using the salt. + +GET /users/{id_or_username} - Get a JSON object of a specific user +DELETE /users/{id_or_username} - Remove a pool_worker. If using MySQL, any shares associated with that user will be associated with the global system account (ID: 0) +PUT /users/{id_or_username} - Update password for user, send as {"password": "password"}, as with POST /users, the password will be encrypted for you. + +### Authentication + +Access to the JSON API requires basic auth. The username does not matter; the password is the same as the ADMIN_PASSWORD_SHA256 password which +can generated using scripts/generateAdminHash.sh . + +I would strongly suggested locking down the port as well using iptables or similar. + +The Rest +-------- + Basic worker stats are provided (and updated) See the INSTALL file for install instructions.